mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-29 05:25:26 +03:00
Compare commits
2 Commits
alert_best
...
improve-ap
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ba0eb6f71d | ||
|
|
f5bd54842a |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -12,7 +12,6 @@
|
||||
/victoria-logs-data
|
||||
/victoria-metrics-data
|
||||
/vmagent-remotewrite-data
|
||||
/vlagent-remotewritewrite
|
||||
/vmstorage-data
|
||||
/vmselect-cache
|
||||
/package/temp-deb-*
|
||||
|
||||
2
Makefile
2
Makefile
@@ -545,7 +545,7 @@ test-full:
|
||||
test-full-386:
|
||||
GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
|
||||
integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore victoria-logs vlagent
|
||||
integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore victoria-logs
|
||||
go test ./apptest/... -skip="^TestCluster.*"
|
||||
|
||||
benchmark:
|
||||
|
||||
@@ -3,12 +3,12 @@
|
||||

|
||||

|
||||

|
||||
[](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml)
|
||||

|
||||

|
||||

|
||||

|
||||
[](https://x.com/VictoriaMetrics/)
|
||||
[](https://www.reddit.com/r/VictoriaMetrics/)
|
||||

|
||||

|
||||
|
||||
<picture>
|
||||
<source srcset="docs/victoriametrics/logo_white.webp" media="(prefers-color-scheme: dark)">
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
# All these commands must run from repository root.
|
||||
|
||||
vlagent:
|
||||
APP_NAME=vlagent $(MAKE) app-local
|
||||
|
||||
vlagent-race:
|
||||
APP_NAME=vlagent RACE=-race $(MAKE) app-local
|
||||
|
||||
vlagent-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker
|
||||
|
||||
vlagent-pure-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-pure
|
||||
|
||||
vlagent-linux-amd64-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-linux-amd64
|
||||
|
||||
vlagent-linux-arm-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-linux-arm
|
||||
|
||||
vlagent-linux-arm64-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-linux-arm64
|
||||
|
||||
vlagent-linux-ppc64le-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-linux-ppc64le
|
||||
|
||||
vlagent-linux-386-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-linux-386
|
||||
|
||||
vlagent-darwin-amd64-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-darwin-amd64
|
||||
|
||||
vlagent-darwin-arm64-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-darwin-arm64
|
||||
|
||||
vlagent-freebsd-amd64-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-freebsd-amd64
|
||||
|
||||
vlagent-openbsd-amd64-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-openbsd-amd64
|
||||
|
||||
vlagent-windows-amd64-prod:
|
||||
APP_NAME=vlagent $(MAKE) app-via-docker-windows-amd64
|
||||
|
||||
package-vlagent:
|
||||
APP_NAME=vlagent $(MAKE) package-via-docker
|
||||
|
||||
package-vlagent-pure:
|
||||
APP_NAME=vlagent $(MAKE) package-via-docker-pure
|
||||
|
||||
package-vlagent-amd64:
|
||||
APP_NAME=vlagent $(MAKE) package-via-docker-amd64
|
||||
|
||||
package-vlagent-arm:
|
||||
APP_NAME=vlagent $(MAKE) package-via-docker-arm
|
||||
|
||||
package-vlagent-arm64:
|
||||
APP_NAME=vlagent $(MAKE) package-via-docker-arm64
|
||||
|
||||
package-vlagent-ppc64le:
|
||||
APP_NAME=vlagent $(MAKE) package-via-docker-ppc64le
|
||||
|
||||
package-vlagent-386:
|
||||
APP_NAME=vlagent $(MAKE) package-via-docker-386
|
||||
|
||||
publish-vlagent:
|
||||
APP_NAME=vlagent $(MAKE) publish-via-docker
|
||||
|
||||
vlagent-linux-amd64:
|
||||
APP_NAME=vlagent CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-linux-arm:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-linux-arm64:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-linux-ppc64le:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-linux-s390x:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-linux-loong64:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-linux-386:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-darwin-amd64:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-darwin-arm64:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-freebsd-amd64:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-openbsd-amd64:
|
||||
APP_NAME=vlagent CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlagent-windows-amd64:
|
||||
GOARCH=amd64 APP_NAME=vlagent $(MAKE) app-local-windows-goarch
|
||||
|
||||
vlagent-pure:
|
||||
APP_NAME=vlagent $(MAKE) app-local-pure
|
||||
@@ -1,3 +0,0 @@
|
||||
See vlagent docs [here](https://docs.victoriametrics.com/victorialogs/vlagent/).
|
||||
|
||||
vlagent docs can be edited at [docs/vlagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victorialogs/vlagent.md).
|
||||
@@ -1,8 +0,0 @@
|
||||
ARG base_image=non-existing
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 9429
|
||||
|
||||
ENTRYPOINT ["/vlagent-prod"]
|
||||
ARG src_binary=non-existing
|
||||
COPY $src_binary ./vlagent-prod
|
||||
@@ -1,97 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlagent/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
|
||||
)
|
||||
|
||||
var (
|
||||
httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
|
||||
"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vlagent instances on the same server. "+
|
||||
"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
|
||||
useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
|
||||
"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
|
||||
"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Write flags and help message to stdout, since it is easier to grep or pipe.
|
||||
flag.CommandLine.SetOutput(os.Stdout)
|
||||
flag.Usage = usage
|
||||
envflag.Parse()
|
||||
buildinfo.Init()
|
||||
remotewrite.InitSecretFlags()
|
||||
logger.Init()
|
||||
|
||||
remotewrite.Init()
|
||||
vlinsert.Init()
|
||||
|
||||
insertutil.SetLogRowsStorage(&remotewrite.Storage{})
|
||||
listenAddrs := *httpListenAddrs
|
||||
if len(listenAddrs) == 0 {
|
||||
listenAddrs = []string{":9429"}
|
||||
}
|
||||
logger.Infof("starting vlagent at %q...", listenAddrs)
|
||||
startTime := time.Now()
|
||||
go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
|
||||
UseProxyProtocol: useProxyProtocol,
|
||||
})
|
||||
logger.Infof("started vlagent in %.3f seconds", time.Since(startTime).Seconds())
|
||||
|
||||
pushmetrics.Init()
|
||||
sig := procutil.WaitForSigterm()
|
||||
logger.Infof("received signal %s", sig)
|
||||
pushmetrics.Stop()
|
||||
|
||||
startTime = time.Now()
|
||||
logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
|
||||
if err := httpserver.Stop(listenAddrs); err != nil {
|
||||
logger.Fatalf("cannot stop the webservice: %s", err)
|
||||
}
|
||||
vlinsert.Stop()
|
||||
remotewrite.Stop()
|
||||
logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
|
||||
logger.Infof("successfully stopped vlagent in %.3f seconds", time.Since(startTime).Seconds())
|
||||
}
|
||||
|
||||
// RequestHandler handles insert requests for VictoriaLogs
|
||||
func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
if r.URL.Path == "/" {
|
||||
if r.Method != http.MethodGet {
|
||||
return false
|
||||
}
|
||||
w.Header().Add("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprintf(w, "<h2>vlagent</h2>")
|
||||
fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victorialogs/vlagent/'>https://docs.victoriametrics.com/victorialogs/vlagent/</a></br>")
|
||||
fmt.Fprintf(w, "Useful endpoints:</br>")
|
||||
httpserver.WriteAPIHelp(w, [][2]string{
|
||||
{"metrics", "available service metrics"},
|
||||
{"flags", "command-line flags"},
|
||||
})
|
||||
return true
|
||||
}
|
||||
return vlinsert.RequestHandler(w, r)
|
||||
}
|
||||
|
||||
func usage() {
|
||||
const s = `
|
||||
vlagent collects logs via popular data ingestion protocols and routes it to VictoriaLogs.
|
||||
|
||||
See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ .
|
||||
`
|
||||
flagutil.Usage(s)
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
FROM $root_image
|
||||
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
EXPOSE 9429
|
||||
ENTRYPOINT ["/vlagent-prod"]
|
||||
ARG TARGETARCH
|
||||
COPY vlagent-linux-${TARGETARCH}-prod ./vlagent-prod
|
||||
@@ -1,462 +0,0 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
)
|
||||
|
||||
var (
|
||||
rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
|
||||
"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data ")
|
||||
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
|
||||
retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxTime")
|
||||
retryMaxTime = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
|
||||
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
|
||||
"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
|
||||
|
||||
tlsHandshakeTimeout = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
|
||||
tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
|
||||
tlsCertFile = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
|
||||
"to the corresponding -remoteWrite.url")
|
||||
tlsKeyFile = flagutil.NewArrayString("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
|
||||
tlsCAFile = flagutil.NewArrayString("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
|
||||
"By default, system CA is used")
|
||||
tlsServerName = flagutil.NewArrayString("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
|
||||
"By default, the server name from -remoteWrite.url is used")
|
||||
|
||||
headers = flagutil.NewArrayString("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
|
||||
"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
|
||||
"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
|
||||
|
||||
basicAuthUsername = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
|
||||
basicAuthPassword = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
|
||||
basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
|
||||
"The file is re-read every second")
|
||||
bearerToken = flagutil.NewArrayString("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
|
||||
bearerTokenFile = flagutil.NewArrayString("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
|
||||
"The token is re-read from the file every second")
|
||||
|
||||
oauth2ClientID = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
|
||||
oauth2ClientSecret = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
|
||||
oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
|
||||
oauth2EndpointParams = flagutil.NewArrayString("remoteWrite.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -remoteWrite.url . "+
|
||||
`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
|
||||
oauth2TokenURL = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
|
||||
oauth2Scopes = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
|
||||
)
|
||||
|
||||
type client struct {
|
||||
sanitizedURL string
|
||||
remoteWriteURL string
|
||||
|
||||
fq *persistentqueue.FastQueue
|
||||
hc *http.Client
|
||||
|
||||
retryMinInterval time.Duration
|
||||
retryMaxTime time.Duration
|
||||
|
||||
sendBlock func(block []byte) bool
|
||||
authCfg *promauth.Config
|
||||
|
||||
rl *ratelimiter.RateLimiter
|
||||
|
||||
bytesSent *metrics.Counter
|
||||
blocksSent *metrics.Counter
|
||||
requestDuration *metrics.Histogram
|
||||
requestsOKCount *metrics.Counter
|
||||
errorsCount *metrics.Counter
|
||||
packetsDropped *metrics.Counter
|
||||
rateLimit *metrics.Gauge
|
||||
retriesCount *metrics.Counter
|
||||
sendDuration *metrics.FloatCounter
|
||||
|
||||
wg sync.WaitGroup
|
||||
stopCh chan struct{}
|
||||
}
|
||||
|
||||
func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
|
||||
authCfg, err := getAuthConfig(argIdx)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
|
||||
}
|
||||
|
||||
tr := httputil.NewTransport(false, "vlagent_remotewrite")
|
||||
tr.TLSHandshakeTimeout = tlsHandshakeTimeout.GetOptionalArg(argIdx)
|
||||
tr.MaxConnsPerHost = 2 * concurrency
|
||||
tr.MaxIdleConnsPerHost = 2 * concurrency
|
||||
tr.IdleConnTimeout = time.Minute
|
||||
tr.WriteBufferSize = 64 * 1024
|
||||
|
||||
pURL := proxyURL.GetOptionalArg(argIdx)
|
||||
if len(pURL) > 0 {
|
||||
if !strings.Contains(pURL, "://") {
|
||||
logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
|
||||
}
|
||||
pu, err := url.Parse(pURL)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
|
||||
}
|
||||
tr.Proxy = http.ProxyURL(pu)
|
||||
}
|
||||
hc := &http.Client{
|
||||
Transport: authCfg.NewRoundTripper(tr),
|
||||
Timeout: sendTimeout.GetOptionalArg(argIdx),
|
||||
}
|
||||
c := &client{
|
||||
sanitizedURL: sanitizedURL,
|
||||
remoteWriteURL: remoteWriteURL,
|
||||
authCfg: authCfg,
|
||||
fq: fq,
|
||||
hc: hc,
|
||||
retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
|
||||
retryMaxTime: retryMaxTime.GetOptionalArg(argIdx),
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
c.sendBlock = c.sendBlockHTTP
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
|
||||
limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
|
||||
if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
|
||||
logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
|
||||
c.rl = ratelimiter.New(int64(bytesPerSec), limitReached, c.stopCh)
|
||||
}
|
||||
c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
|
||||
c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
|
||||
c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
|
||||
return float64(rateLimit.GetOptionalArg(argIdx))
|
||||
})
|
||||
c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vlagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
|
||||
c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
|
||||
c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
|
||||
c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
|
||||
c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
|
||||
c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vlagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
|
||||
metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
|
||||
return float64(*queues)
|
||||
})
|
||||
for i := 0; i < concurrency; i++ {
|
||||
c.wg.Add(1)
|
||||
go func() {
|
||||
defer c.wg.Done()
|
||||
c.runWorker()
|
||||
}()
|
||||
}
|
||||
logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
|
||||
}
|
||||
|
||||
func (c *client) MustStop() {
|
||||
close(c.stopCh)
|
||||
c.wg.Wait()
|
||||
logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
|
||||
}
|
||||
|
||||
func getAuthConfig(argIdx int) (*promauth.Config, error) {
|
||||
headersValue := headers.GetOptionalArg(argIdx)
|
||||
var hdrs []string
|
||||
if headersValue != "" {
|
||||
hdrs = strings.Split(headersValue, "^^")
|
||||
}
|
||||
username := basicAuthUsername.GetOptionalArg(argIdx)
|
||||
password := basicAuthPassword.GetOptionalArg(argIdx)
|
||||
passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
|
||||
var basicAuthCfg *promauth.BasicAuthConfig
|
||||
if username != "" || password != "" || passwordFile != "" {
|
||||
basicAuthCfg = &promauth.BasicAuthConfig{
|
||||
Username: username,
|
||||
Password: promauth.NewSecret(password),
|
||||
PasswordFile: passwordFile,
|
||||
}
|
||||
}
|
||||
|
||||
token := bearerToken.GetOptionalArg(argIdx)
|
||||
tokenFile := bearerTokenFile.GetOptionalArg(argIdx)
|
||||
|
||||
var oauth2Cfg *promauth.OAuth2Config
|
||||
clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
|
||||
clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
|
||||
if clientSecretFile != "" || clientSecret != "" {
|
||||
endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(argIdx)
|
||||
endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
|
||||
}
|
||||
oauth2Cfg = &promauth.OAuth2Config{
|
||||
ClientID: oauth2ClientID.GetOptionalArg(argIdx),
|
||||
ClientSecret: promauth.NewSecret(clientSecret),
|
||||
ClientSecretFile: clientSecretFile,
|
||||
EndpointParams: endpointParams,
|
||||
TokenURL: oauth2TokenURL.GetOptionalArg(argIdx),
|
||||
Scopes: strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
|
||||
}
|
||||
}
|
||||
|
||||
tlsCfg := &promauth.TLSConfig{
|
||||
CAFile: tlsCAFile.GetOptionalArg(argIdx),
|
||||
CertFile: tlsCertFile.GetOptionalArg(argIdx),
|
||||
KeyFile: tlsKeyFile.GetOptionalArg(argIdx),
|
||||
ServerName: tlsServerName.GetOptionalArg(argIdx),
|
||||
InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
|
||||
}
|
||||
|
||||
opts := &promauth.Options{
|
||||
BasicAuth: basicAuthCfg,
|
||||
BearerToken: token,
|
||||
BearerTokenFile: tokenFile,
|
||||
OAuth2: oauth2Cfg,
|
||||
TLSConfig: tlsCfg,
|
||||
Headers: hdrs,
|
||||
}
|
||||
authCfg, err := opts.NewConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot populate auth config for remoteWrite idx: %d, err: %w", argIdx, err)
|
||||
}
|
||||
return authCfg, nil
|
||||
}
|
||||
|
||||
func (c *client) runWorker() {
|
||||
var ok bool
|
||||
var block []byte
|
||||
ch := make(chan bool, 1)
|
||||
for {
|
||||
block, ok = c.fq.MustReadBlock(block[:0])
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if len(block) == 0 {
|
||||
// skip empty data blocks from sending
|
||||
continue
|
||||
}
|
||||
go func() {
|
||||
startTime := time.Now()
|
||||
ch <- c.sendBlock(block)
|
||||
c.sendDuration.Add(time.Since(startTime).Seconds())
|
||||
}()
|
||||
select {
|
||||
case ok := <-ch:
|
||||
if ok {
|
||||
// The block has been sent successfully
|
||||
continue
|
||||
}
|
||||
// Return unsent block to the queue.
|
||||
c.fq.MustWriteBlockIgnoreDisabledPQ(block)
|
||||
return
|
||||
case <-c.stopCh:
|
||||
// c must be stopped. Wait for a while in the hope the block will be sent.
|
||||
graceDuration := 5 * time.Second
|
||||
select {
|
||||
case ok := <-ch:
|
||||
if !ok {
|
||||
// Return unsent block to the queue.
|
||||
c.fq.MustWriteBlockIgnoreDisabledPQ(block)
|
||||
}
|
||||
case <-time.After(graceDuration):
|
||||
// Return unsent block to the queue.
|
||||
c.fq.MustWriteBlockIgnoreDisabledPQ(block)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
|
||||
req, err := c.newRequest(url, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.hc.Do(req)
|
||||
if err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return nil, err
|
||||
}
|
||||
// It is likely connection became stale or timed out during the first request.
|
||||
// Make another attempt in hope request will succeed.
|
||||
// If not, the error should be handled by the caller as usual.
|
||||
// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
|
||||
req, err = c.newRequest(url, body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("second attempt: %w", err)
|
||||
}
|
||||
resp, err = c.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("second attempt: %w", err)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
|
||||
reqBody := bytes.NewBuffer(body)
|
||||
req, err := http.NewRequest(http.MethodPost, url, reqBody)
|
||||
if err != nil {
|
||||
logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
|
||||
}
|
||||
err = c.authCfg.SetHeaders(req, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h := req.Header
|
||||
h.Set("User-Agent", "vlagent")
|
||||
h.Set("Content-Encoding", "zstd")
|
||||
h.Set("Content-Type", "application/octet-stream")
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// sendBlockHTTP sends the given block to c.remoteWriteURL.
|
||||
//
|
||||
// The function returns false only if c.stopCh is closed.
|
||||
// Otherwise, it tries sending the block to remote storage indefinitely.
|
||||
func (c *client) sendBlockHTTP(block []byte) bool {
|
||||
c.rl.Register(len(block))
|
||||
maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
|
||||
retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
|
||||
retriesCount := 0
|
||||
|
||||
again:
|
||||
startTime := time.Now()
|
||||
resp, err := c.doRequest(c.remoteWriteURL, block)
|
||||
c.requestDuration.UpdateDuration(startTime)
|
||||
if err != nil {
|
||||
c.errorsCount.Inc()
|
||||
retryDuration *= 2
|
||||
if retryDuration > maxRetryDuration {
|
||||
retryDuration = maxRetryDuration
|
||||
}
|
||||
remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
|
||||
len(block), c.sanitizedURL, err, retryDuration.Seconds())
|
||||
t := timerpool.Get(retryDuration)
|
||||
select {
|
||||
case <-c.stopCh:
|
||||
timerpool.Put(t)
|
||||
return false
|
||||
case <-t.C:
|
||||
timerpool.Put(t)
|
||||
}
|
||||
c.retriesCount.Inc()
|
||||
goto again
|
||||
}
|
||||
|
||||
statusCode := resp.StatusCode
|
||||
if statusCode/100 == 2 {
|
||||
_ = resp.Body.Close()
|
||||
c.requestsOKCount.Inc()
|
||||
c.bytesSent.Add(len(block))
|
||||
c.blocksSent.Inc()
|
||||
return true
|
||||
}
|
||||
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
|
||||
if statusCode == 400 || statusCode == 404 {
|
||||
logBlockRejected(block, c.sanitizedURL, resp)
|
||||
_ = resp.Body.Close()
|
||||
c.packetsDropped.Inc()
|
||||
return true
|
||||
}
|
||||
// Unexpected status code returned
|
||||
retriesCount++
|
||||
retryAfterHeader := parseRetryAfterHeader(resp.Header.Get("Retry-After"))
|
||||
retryDuration = getRetryDuration(retryAfterHeader, retryDuration, maxRetryDuration)
|
||||
|
||||
// Handle response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
|
||||
} else {
|
||||
logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
|
||||
"re-sending the block in %.3f seconds", len(block), c.sanitizedURL, retriesCount, statusCode, body, retryDuration.Seconds())
|
||||
}
|
||||
t := timerpool.Get(retryDuration)
|
||||
select {
|
||||
case <-c.stopCh:
|
||||
timerpool.Put(t)
|
||||
return false
|
||||
case <-t.C:
|
||||
timerpool.Put(t)
|
||||
}
|
||||
c.retriesCount.Inc()
|
||||
goto again
|
||||
}
|
||||
|
||||
var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
|
||||
var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
|
||||
|
||||
// getRetryDuration returns retry duration.
|
||||
// retryAfterDuration has the highest priority.
|
||||
// If retryAfterDuration is not specified, retryDuration gets doubled.
|
||||
// retryDuration can't exceed maxRetryDuration.
|
||||
//
|
||||
// Also see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6097
|
||||
func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.Duration) time.Duration {
|
||||
// retryAfterDuration has the highest priority duration
|
||||
if retryAfterDuration > 0 {
|
||||
return timeutil.AddJitterToDuration(retryAfterDuration)
|
||||
}
|
||||
|
||||
// default backoff retry policy
|
||||
retryDuration *= 2
|
||||
if retryDuration > maxRetryDuration {
|
||||
retryDuration = maxRetryDuration
|
||||
}
|
||||
|
||||
return retryDuration
|
||||
}
|
||||
|
||||
func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
|
||||
"failed to read response body: %s",
|
||||
len(block), sanitizedURL, resp.StatusCode, err)
|
||||
} else {
|
||||
remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
|
||||
len(block), sanitizedURL, resp.StatusCode, string(body))
|
||||
}
|
||||
}
|
||||
|
||||
// parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
|
||||
// retryAfterString should be in either HTTP-date or a number of seconds.
|
||||
// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
|
||||
func parseRetryAfterHeader(retryAfterString string) (retryAfterDuration time.Duration) {
|
||||
if retryAfterString == "" {
|
||||
return retryAfterDuration
|
||||
}
|
||||
|
||||
defer func() {
|
||||
v := retryAfterDuration.Seconds()
|
||||
logger.Infof("'Retry-After: %s' parsed into %.2f second(s)", retryAfterString, v)
|
||||
}()
|
||||
|
||||
// Retry-After could be in "Mon, 02 Jan 2006 15:04:05 GMT" format.
|
||||
if parsedTime, err := time.Parse(http.TimeFormat, retryAfterString); err == nil {
|
||||
return time.Duration(time.Until(parsedTime).Seconds()) * time.Second
|
||||
}
|
||||
// Retry-After could be in seconds.
|
||||
if seconds, err := strconv.Atoi(retryAfterString); err == nil {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"math"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCalculateRetryDuration(t *testing.T) {
|
||||
// `testFunc` call `calculateRetryDuration` for `n` times
|
||||
// and evaluate if the result of `calculateRetryDuration` is
|
||||
// 1. >= expectMinDuration
|
||||
// 2. <= expectMinDuration + 10% (see timeutil.AddJitterToDuration)
|
||||
f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
|
||||
}
|
||||
|
||||
expectMaxDuration := helper(expectMinDuration)
|
||||
expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
|
||||
|
||||
if !(retryDuration >= expectMinDuration && retryDuration <= expectMaxDuration) {
|
||||
t.Fatalf(
|
||||
"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
|
||||
expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
|
||||
retryDuration.Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Call calculateRetryDuration for 1 time.
|
||||
{
|
||||
// default backoff policy
|
||||
f(0, time.Second, 1, 2*time.Second)
|
||||
// default backoff policy exceed max limit"
|
||||
f(0, 10*time.Minute, 1, time.Minute)
|
||||
|
||||
// retry after > default backoff policy
|
||||
f(10*time.Second, 1*time.Second, 1, 10*time.Second)
|
||||
// retry after < default backoff policy
|
||||
f(1*time.Second, 10*time.Second, 1, 1*time.Second)
|
||||
// retry after invalid and < default backoff policy
|
||||
f(0, time.Second, 1, 2*time.Second)
|
||||
|
||||
}
|
||||
|
||||
// Call calculateRetryDuration for multiple times.
|
||||
{
|
||||
// default backoff policy 2 times
|
||||
f(0, time.Second, 2, 4*time.Second)
|
||||
// default backoff policy 3 times
|
||||
f(0, time.Second, 3, 8*time.Second)
|
||||
// default backoff policy N times exceed max limit
|
||||
f(0, time.Second, 10, time.Minute)
|
||||
|
||||
// retry after 120s 1 times
|
||||
f(120*time.Second, time.Second, 1, 120*time.Second)
|
||||
// retry after 120s 2 times
|
||||
f(120*time.Second, time.Second, 2, 120*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRetryAfterHeader(t *testing.T) {
|
||||
f := func(retryAfterString string, expectResult time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
result := parseRetryAfterHeader(retryAfterString)
|
||||
// expect `expectResult == result` when retryAfterString is in seconds or invalid
|
||||
// expect the difference between result and expectResult to be lower than 10%
|
||||
if !(expectResult == result || math.Abs(float64(expectResult-result))/float64(expectResult) < 0.10) {
|
||||
t.Fatalf(
|
||||
"incorrect retry after duration, want (ms): %d, got (ms): %d",
|
||||
expectResult.Milliseconds(), result.Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// retry after header in seconds
|
||||
f("10", 10*time.Second)
|
||||
// retry after header in date time
|
||||
f(time.Now().Add(30*time.Second).UTC().Format(http.TimeFormat), 30*time.Second)
|
||||
// retry after header invalid
|
||||
f("invalid-retry-after", 0)
|
||||
// retry after header not in GMT
|
||||
f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
|
||||
}
|
||||
|
||||
// helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
|
||||
func helper(d time.Duration) time.Duration {
|
||||
dv := d / 10
|
||||
if dv > 10*time.Second {
|
||||
dv = 10 * time.Second
|
||||
}
|
||||
|
||||
return d + dv
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
)
|
||||
|
||||
var (
|
||||
maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage.")
|
||||
flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
|
||||
"This option takes effect only when less than 2MB of data per second are pushed to -remoteWrite.url")
|
||||
)
|
||||
|
||||
type pendingLogs struct {
|
||||
lastFlushTime atomic.Uint64
|
||||
|
||||
// The queue to send blocks to.
|
||||
fq *persistentqueue.FastQueue
|
||||
|
||||
// mu protects wr
|
||||
mu sync.Mutex
|
||||
wr writeRequest
|
||||
|
||||
stopCh chan struct{}
|
||||
periodicFlusherWG sync.WaitGroup
|
||||
}
|
||||
|
||||
func newPendingLogs(fq *persistentqueue.FastQueue) *pendingLogs {
|
||||
pl := &pendingLogs{
|
||||
fq: fq,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
pl.periodicFlusherWG.Add(1)
|
||||
go func() {
|
||||
defer pl.periodicFlusherWG.Done()
|
||||
pl.periodicFlusher()
|
||||
}()
|
||||
|
||||
return pl
|
||||
}
|
||||
|
||||
func (pl *pendingLogs) add(lr *logstorage.LogRows) {
|
||||
lr.ForEachRow(func(_ uint64, r *logstorage.InsertRow) {
|
||||
pl.addLogRow(r)
|
||||
})
|
||||
}
|
||||
|
||||
func (pl *pendingLogs) addLogRow(r *logstorage.InsertRow) {
|
||||
bb := bbPool.Get()
|
||||
bb.B = r.Marshal(bb.B)
|
||||
|
||||
pl.mu.Lock()
|
||||
_, _ = pl.wr.pendingData.Write(bb.B)
|
||||
pl.wr.pendingLogRowsCount++
|
||||
if len(pl.wr.pendingData.B) > maxUnpackedBlockSize.IntN() {
|
||||
pl.mustFlushLocked()
|
||||
}
|
||||
pl.mu.Unlock()
|
||||
bbPool.Put(bb)
|
||||
}
|
||||
|
||||
func (pl *pendingLogs) mustFlushLocked() {
|
||||
pl.lastFlushTime.Store(fasttime.UnixTimestamp())
|
||||
pl.wr.push(func(b []byte) {
|
||||
if !pl.fq.TryWriteBlock(b) {
|
||||
logger.Fatalf("BUG: TryWriteBlock cannot return false")
|
||||
}
|
||||
})
|
||||
pl.wr.reset()
|
||||
}
|
||||
|
||||
func (pl *pendingLogs) periodicFlusher() {
|
||||
flushSeconds := int64(flushInterval.Seconds())
|
||||
if flushSeconds <= 0 {
|
||||
flushSeconds = 1
|
||||
}
|
||||
d := timeutil.AddJitterToDuration(*flushInterval)
|
||||
ticker := time.NewTicker(d)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-pl.stopCh:
|
||||
pl.mu.Lock()
|
||||
pl.mustFlushOnStop()
|
||||
pl.mu.Unlock()
|
||||
return
|
||||
case <-ticker.C:
|
||||
if fasttime.UnixTimestamp()-pl.lastFlushTime.Load() < uint64(flushSeconds) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
pl.mu.Lock()
|
||||
pl.mustFlushLocked()
|
||||
pl.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// mustFlushOnStop force pushes wr data
|
||||
//
|
||||
// This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
|
||||
func (pl *pendingLogs) mustFlushOnStop() {
|
||||
pl.wr.push(pl.fq.MustWriteBlockIgnoreDisabledPQ)
|
||||
pl.wr.reset()
|
||||
}
|
||||
|
||||
func (pl *pendingLogs) mustStop() {
|
||||
close(pl.stopCh)
|
||||
pl.periodicFlusherWG.Wait()
|
||||
}
|
||||
|
||||
type writeRequest struct {
|
||||
pendingData bytesutil.ByteBuffer
|
||||
pendingLogRowsCount int64
|
||||
}
|
||||
|
||||
func (wr *writeRequest) push(pushBlock func([]byte)) {
|
||||
if len(wr.pendingData.B) == 0 {
|
||||
return
|
||||
}
|
||||
b := wr.pendingData.B
|
||||
|
||||
zb := compressBufPool.Get()
|
||||
zb.B = zstd.CompressLevel(zb.B[:0], b, 1)
|
||||
zbLen := len(zb.B)
|
||||
pushBlock(zb.B)
|
||||
compressBufPool.Put(zb)
|
||||
blockSizeBytes.Update(float64(zbLen))
|
||||
blockSizeLogRows.Update(float64(wr.pendingLogRowsCount))
|
||||
}
|
||||
|
||||
func (wr *writeRequest) reset() {
|
||||
wr.pendingData.Reset()
|
||||
wr.pendingLogRowsCount = 0
|
||||
}
|
||||
|
||||
var (
|
||||
blockSizeBytes = metrics.NewHistogram(`vlagent_remotewrite_block_size_bytes`)
|
||||
blockSizeLogRows = metrics.NewHistogram(`vlagent_remotewrite_block_size_rows`)
|
||||
)
|
||||
|
||||
var (
|
||||
compressBufPool bytesutil.ByteBufferPool
|
||||
bbPool bytesutil.ByteBufferPool
|
||||
)
|
||||
@@ -1,277 +0,0 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/cespare/xxhash/v2"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
|
||||
)
|
||||
|
||||
var (
|
||||
remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support VictoriaLogs native protocol. "+
|
||||
"Example url: http://<victorialogs-host>:9428/internal/insert. "+
|
||||
"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems.")
|
||||
maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
|
||||
"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
|
||||
"Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. "+
|
||||
"Disk usage is unlimited if the value is set to 0")
|
||||
|
||||
tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vlagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
|
||||
"See also -remoteWrite.maxDiskUsagePerURL")
|
||||
queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
|
||||
"isn't enough for sending high volume of collected data to remote storage. "+
|
||||
"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
|
||||
|
||||
showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
|
||||
"It is hidden by default, since it can contain sensitive info such as auth key")
|
||||
)
|
||||
|
||||
// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
|
||||
var rwctxsGlobal []*remoteWriteCtx
|
||||
|
||||
// Storage implements insertutil.LogRowsStorage interface
|
||||
type Storage struct{}
|
||||
|
||||
// MustAddRows implements insertutil.LogRowsStorage interface
|
||||
func (*Storage) MustAddRows(lr *logstorage.LogRows) {
|
||||
pushToRemoteStorages(lr)
|
||||
}
|
||||
|
||||
// CanWriteData implements insertutil.LogRowsStorage interface
|
||||
func (*Storage) CanWriteData() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
|
||||
// since it may lead to high memory usage due to big number of buffers.
|
||||
var maxQueues = cgroup.AvailableCPUs() * 16
|
||||
|
||||
const persistentQueueDirname = "persistent-queue"
|
||||
|
||||
// InitSecretFlags must be called after flag.Parse and before any logging.
|
||||
func InitSecretFlags() {
|
||||
if !*showRemoteWriteURL {
|
||||
// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
|
||||
flagutil.RegisterSecretFlag("remoteWrite.url")
|
||||
}
|
||||
}
|
||||
|
||||
// Init initializes remotewrite.
|
||||
//
|
||||
// It must be called after flag.Parse().
|
||||
//
|
||||
// Stop must be called for graceful shutdown.
|
||||
func Init() {
|
||||
if len(*remoteWriteURLs) == 0 {
|
||||
logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
|
||||
}
|
||||
if *queues > maxQueues {
|
||||
*queues = maxQueues
|
||||
}
|
||||
if *queues <= 0 {
|
||||
*queues = 1
|
||||
}
|
||||
initRemoteWriteCtxs(*remoteWriteURLs)
|
||||
dropDanglingQueues()
|
||||
}
|
||||
|
||||
// Stop stops remotewrite.
|
||||
//
|
||||
// It is expected that nobody calls TryPush during and after the call to this func.
|
||||
func Stop() {
|
||||
for _, rwctx := range rwctxsGlobal {
|
||||
rwctx.mustStop()
|
||||
}
|
||||
rwctxsGlobal = nil
|
||||
}
|
||||
|
||||
func dropDanglingQueues() {
|
||||
// Remove dangling persistent queues, if any.
|
||||
// This is required for the case when the number of queues has been changed or URL have been changed.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
|
||||
//
|
||||
// In case if there were many persistent queues with identical *remoteWriteURLs
|
||||
// the queue with the last index will be dropped.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
|
||||
existingQueues := make(map[string]struct{}, len(rwctxsGlobal))
|
||||
for _, rwctx := range rwctxsGlobal {
|
||||
existingQueues[rwctx.fq.Dirname()] = struct{}{}
|
||||
}
|
||||
|
||||
queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
|
||||
files := fs.MustReadDir(queuesDir)
|
||||
removed := 0
|
||||
for _, f := range files {
|
||||
dirname := f.Name()
|
||||
if _, ok := existingQueues[dirname]; !ok {
|
||||
logger.Infof("removing dangling queue %q", dirname)
|
||||
fullPath := filepath.Join(queuesDir, dirname)
|
||||
fs.MustRemoveAll(fullPath)
|
||||
removed++
|
||||
}
|
||||
}
|
||||
if removed > 0 {
|
||||
logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsGlobal))
|
||||
}
|
||||
}
|
||||
|
||||
func initRemoteWriteCtxs(urls []string) {
|
||||
if len(urls) == 0 {
|
||||
logger.Panicf("BUG: urls must be non-empty")
|
||||
}
|
||||
|
||||
maxInmemoryBlocks := memory.Allowed() / len(urls) / 10000
|
||||
if maxInmemoryBlocks / *queues > 100 {
|
||||
// There is no much sense in keeping higher number of blocks in memory,
|
||||
// since this means that the producer outperforms consumer and the queue
|
||||
// will continue growing. It is better storing the queue to file.
|
||||
maxInmemoryBlocks = 100 * *queues
|
||||
}
|
||||
if maxInmemoryBlocks < 2 {
|
||||
maxInmemoryBlocks = 2
|
||||
}
|
||||
rwctxs := make([]*remoteWriteCtx, len(urls))
|
||||
rwctxIdx := make([]int, len(urls))
|
||||
for i, remoteWriteURLRaw := range urls {
|
||||
remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
|
||||
if err != nil {
|
||||
logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
|
||||
}
|
||||
sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
|
||||
if *showRemoteWriteURL {
|
||||
sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
|
||||
}
|
||||
rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
|
||||
rwctxIdx[i] = i
|
||||
}
|
||||
|
||||
rwctxsGlobal = rwctxs
|
||||
}
|
||||
|
||||
func pushToRemoteStorages(lr *logstorage.LogRows) {
|
||||
rwctxs := rwctxsGlobal
|
||||
if len(rwctxs) == 1 {
|
||||
// fast path
|
||||
rwctxs[0].push(lr)
|
||||
return
|
||||
}
|
||||
// Push samples to remote storage systems in parallel in order to reduce
|
||||
// the time needed for sending the data to multiple remote storage systems.
|
||||
var wg sync.WaitGroup
|
||||
for _, rwctx := range rwctxs {
|
||||
wg.Add(1)
|
||||
go func(rwctx *remoteWriteCtx) {
|
||||
defer wg.Done()
|
||||
rwctx.push(lr)
|
||||
|
||||
}(rwctx)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
type remoteWriteCtx struct {
|
||||
idx int
|
||||
fq *persistentqueue.FastQueue
|
||||
c *client
|
||||
|
||||
pls []*pendingLogs
|
||||
pssNextIdx atomic.Uint64
|
||||
}
|
||||
|
||||
func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
|
||||
// protocol version is required by victoria-logs
|
||||
q := remoteWriteURL.Query()
|
||||
q.Set("version", netinsert.ProtocolVersion)
|
||||
remoteWriteURL.RawQuery = q.Encode()
|
||||
|
||||
// strip query params, otherwise changing params resets pq
|
||||
pqURL := *remoteWriteURL
|
||||
pqURL.RawQuery = ""
|
||||
pqURL.Fragment = ""
|
||||
h := xxhash.Sum64([]byte(pqURL.String()))
|
||||
queuePath := filepath.Join(*tmpDataPath, persistentQueueDirname, fmt.Sprintf("%d_%016X", argIdx+1, h))
|
||||
maxPendingBytes := maxPendingBytesPerURL.GetOptionalArg(argIdx)
|
||||
if maxPendingBytes != 0 && maxPendingBytes < persistentqueue.DefaultChunkFileSize {
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4195
|
||||
logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
|
||||
maxPendingBytes = persistentqueue.DefaultChunkFileSize
|
||||
}
|
||||
|
||||
fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, false)
|
||||
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
|
||||
return float64(fq.GetPendingBytes())
|
||||
})
|
||||
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
|
||||
return float64(fq.GetInmemoryQueueLen())
|
||||
})
|
||||
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_queue_blocked{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
|
||||
if fq.IsWriteBlocked() {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
|
||||
var c *client
|
||||
switch remoteWriteURL.Scheme {
|
||||
case "http", "https":
|
||||
c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
|
||||
default:
|
||||
logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
|
||||
}
|
||||
c.init(argIdx, *queues, sanitizedURL)
|
||||
|
||||
// Initialize pss
|
||||
plsLen := *queues
|
||||
if n := cgroup.AvailableCPUs(); plsLen > n {
|
||||
// There is no sense in running more than availableCPUs concurrent pendingLogs,
|
||||
// since every pendingLogs can saturate up to a single CPU.
|
||||
plsLen = n
|
||||
}
|
||||
pls := make([]*pendingLogs, plsLen)
|
||||
for i := range pls {
|
||||
pls[i] = newPendingLogs(fq)
|
||||
}
|
||||
|
||||
rwctx := &remoteWriteCtx{
|
||||
idx: argIdx,
|
||||
fq: fq,
|
||||
c: c,
|
||||
pls: pls,
|
||||
}
|
||||
|
||||
return rwctx
|
||||
}
|
||||
|
||||
func (rwctx *remoteWriteCtx) push(lr *logstorage.LogRows) {
|
||||
pls := rwctx.pls
|
||||
idx := rwctx.pssNextIdx.Add(1) % uint64(len(pls))
|
||||
pls[idx].add(lr)
|
||||
}
|
||||
|
||||
func (rwctx *remoteWriteCtx) mustStop() {
|
||||
for _, ps := range rwctx.pls {
|
||||
ps.mustStop()
|
||||
}
|
||||
rwctx.idx = 0
|
||||
rwctx.pls = nil
|
||||
rwctx.fq.UnblockAllReaders()
|
||||
rwctx.c.MustStop()
|
||||
rwctx.c = nil
|
||||
|
||||
rwctx.fq.MustClose()
|
||||
rwctx.fq = nil
|
||||
}
|
||||
@@ -101,7 +101,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
|
||||
|
||||
var (
|
||||
v2LogsRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
|
||||
v2LogsRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
|
||||
v2LogsRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
|
||||
)
|
||||
|
||||
// datadog message field has two formats:
|
||||
|
||||
@@ -129,7 +129,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
|
||||
var (
|
||||
bulkRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
|
||||
bulkRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
|
||||
bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
|
||||
)
|
||||
|
||||
func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {
|
||||
|
||||
@@ -84,5 +84,5 @@ var (
|
||||
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
|
||||
|
||||
requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/internal/insert"}`)
|
||||
requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/internal/insert"}`)
|
||||
)
|
||||
|
||||
@@ -28,6 +28,24 @@ import (
|
||||
// See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
|
||||
const maxFieldNameLen = 64
|
||||
|
||||
func isValidJournaldFieldName(s string) bool {
|
||||
if len(s) == 0 {
|
||||
return false
|
||||
}
|
||||
c := s[0]
|
||||
if !(c >= 'A' && c <= 'Z' || c == '_') {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := 1; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var (
|
||||
journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields")
|
||||
@@ -145,7 +163,7 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
|
||||
var (
|
||||
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
|
||||
requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
|
||||
requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
|
||||
)
|
||||
|
||||
func processStreamInternal(streamName string, r io.Reader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
|
||||
@@ -196,18 +214,6 @@ func (fb *fieldsBuf) addField(name, value string) {
|
||||
})
|
||||
}
|
||||
|
||||
func (fb *fieldsBuf) appendNextLineToValue(lr *insertutil.LineReader) error {
|
||||
if !lr.NextLine() {
|
||||
if err := lr.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("unexpected end of stream")
|
||||
}
|
||||
fb.value = append(fb.value, lr.Line...)
|
||||
fb.value = append(fb.value, '\n')
|
||||
return nil
|
||||
}
|
||||
|
||||
func getFieldsBuf() *fieldsBuf {
|
||||
fb := fieldsBufPool.Get()
|
||||
if fb == nil {
|
||||
@@ -253,35 +259,42 @@ func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp inse
|
||||
return nil
|
||||
}
|
||||
|
||||
// line could be either "key=value" or "key"
|
||||
// line could be either "key=value\n" or "key\n<little_endian_size_64>value\n"
|
||||
// according to https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
|
||||
if n := bytes.IndexByte(line, '='); n >= 0 {
|
||||
// line = "key=value"
|
||||
// "key=value\n"
|
||||
fb.name = append(fb.name[:0], line[:n]...)
|
||||
name = bytesutil.ToUnsafeString(fb.name)
|
||||
|
||||
fb.value = append(fb.value[:0], line[n+1:]...)
|
||||
value = bytesutil.ToUnsafeString(fb.value)
|
||||
} else {
|
||||
// line = "key"
|
||||
// Parse the binary-encoded value from the next line according to "key\n<little_endian_size_64>value\n" format
|
||||
// "key\n<little_endian_size_64>value\n"
|
||||
fb.name = append(fb.name[:0], line...)
|
||||
name = bytesutil.ToUnsafeString(fb.name)
|
||||
|
||||
fb.value = fb.value[:0]
|
||||
for len(fb.value) < 8 {
|
||||
if err := fb.appendNextLineToValue(lr); err != nil {
|
||||
return fmt.Errorf("cannot read value size: %w", err)
|
||||
if !lr.NextLine() {
|
||||
if err := lr.Err(); err != nil {
|
||||
return fmt.Errorf("cannot read value size: %w", err)
|
||||
}
|
||||
return fmt.Errorf("unexpected end of stream while reading value size")
|
||||
}
|
||||
fb.value = append(fb.value, lr.Line...)
|
||||
fb.value = append(fb.value, '\n')
|
||||
}
|
||||
size := binary.LittleEndian.Uint64(fb.value[:8])
|
||||
|
||||
// Read the value until its length exceeds the given size - the last char in the read value will always be '\n'
|
||||
// because it is appended by appendNextLineToValue().
|
||||
for uint64(len(fb.value[8:])) <= size {
|
||||
if err := fb.appendNextLineToValue(lr); err != nil {
|
||||
return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
|
||||
for size > uint64(len(fb.value[8:])) {
|
||||
if !lr.NextLine() {
|
||||
if err := lr.Err(); err != nil {
|
||||
return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
|
||||
}
|
||||
return fmt.Errorf("unexpected end of stream while reading %q value with size %d bytes; read only %d bytes", fb.name, size, len(fb.value[8:]))
|
||||
}
|
||||
fb.value = append(fb.value, lr.Line...)
|
||||
fb.value = append(fb.value, '\n')
|
||||
}
|
||||
value = bytesutil.ToUnsafeString(fb.value[8 : len(fb.value)-1])
|
||||
if uint64(len(value)) != size {
|
||||
@@ -301,7 +314,7 @@ func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp inse
|
||||
logger.Errorf("%s: field name size should not exceed %d bytes; got %d bytes: %q; skipping this field", streamName, maxFieldNameLen, len(name), name)
|
||||
continue
|
||||
}
|
||||
if !isValidFieldName(name) {
|
||||
if !isValidJournaldFieldName(name) {
|
||||
logger.Errorf("%s: invalid field name %q; it must consist of `A-Z0-9_` chars and must start from non-digit char; skipping this field", streamName, name)
|
||||
continue
|
||||
}
|
||||
@@ -337,19 +350,13 @@ func journaldPriorityToLevel(priority string) string {
|
||||
// See https://wiki.archlinux.org/title/Systemd/Journal#Priority_level
|
||||
// and https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level
|
||||
switch priority {
|
||||
case "0":
|
||||
return "emerg"
|
||||
case "1":
|
||||
return "alert"
|
||||
case "2":
|
||||
case "0", "1", "2":
|
||||
return "critical"
|
||||
case "3":
|
||||
return "error"
|
||||
case "4":
|
||||
return "warning"
|
||||
case "5":
|
||||
return "notice"
|
||||
case "6":
|
||||
case "5", "6":
|
||||
return "info"
|
||||
case "7":
|
||||
return "debug"
|
||||
@@ -357,21 +364,3 @@ func journaldPriorityToLevel(priority string) string {
|
||||
return priority
|
||||
}
|
||||
}
|
||||
|
||||
func isValidFieldName(s string) bool {
|
||||
if len(s) == 0 {
|
||||
return false
|
||||
}
|
||||
c := s[0]
|
||||
if !(c >= 'A' && c <= 'Z' || c == '_') {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := 1; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -8,11 +8,11 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func TestIsValidFieldName(t *testing.T) {
|
||||
func TestIsValidJournaldFieldName(t *testing.T) {
|
||||
f := func(name string, resultExpected bool) {
|
||||
t.Helper()
|
||||
|
||||
result := isValidFieldName(name)
|
||||
result := isValidJournaldFieldName(name)
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected result for isValidJournaldFieldName(%q); got %v; want %v", name, result, resultExpected)
|
||||
}
|
||||
@@ -102,24 +102,6 @@ func TestPushJournald_Success(t *testing.T) {
|
||||
"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
|
||||
)
|
||||
|
||||
// Parse binary data with trailing newline
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x14\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n\n_PID=2763\n\n",
|
||||
[]int64{1729698775704404000},
|
||||
`{"_CMDLINE":"python3","_msg":"foo\nbar\n\n\nasda\nasda\n","_PID":"2763"}`,
|
||||
)
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00\n_PID=2763\n\n",
|
||||
[]int64{1729698775704404000},
|
||||
`{"_CMDLINE":"python3","_PID":"2763"}`,
|
||||
)
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n\n_PID=2763\n\n",
|
||||
[]int64{1729698775704404000},
|
||||
`{"_CMDLINE":"python3","_msg":"123456789\n","_PID":"2763"}`,
|
||||
)
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890\n_PID=2763\n\n",
|
||||
[]int64{1729698775704404000},
|
||||
`{"_CMDLINE":"python3","_msg":"1234567890","_PID":"2763"}`,
|
||||
)
|
||||
|
||||
// Empty field name must be ignored
|
||||
f("__REALTIME_TIMESTAMP=91723819283\na=b\n=Test message", nil, "")
|
||||
f("__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n__REALTIME_TIMESTAMP=91723819283\n=Test message\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
|
||||
@@ -156,10 +138,7 @@ func TestPushJournald_Failure(t *testing.T) {
|
||||
}
|
||||
|
||||
// too short binary encoded message
|
||||
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasd")
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00_PID=2763\n\n")
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890_PID=2763\n\n")
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n_PID=2763\n\n")
|
||||
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasda")
|
||||
|
||||
// too long binary encoded message
|
||||
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasdakljlsfd")
|
||||
|
||||
@@ -4,55 +4,12 @@ import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func BenchmarkIsValidFieldName(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(benchmarkFields)))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for _, field := range benchmarkFields {
|
||||
if !isValidFieldName(field) {
|
||||
panic(fmt.Errorf("cannot validate field %q", field))
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
var benchmarkFields = strings.Split(
|
||||
"E,_BOOT_ID,_UID,_GID,_MACHINE_ID,_HOSTNAME,_RUNTIME_SCOPE,_TRANSPORT,_CAP_EFFECTIVE,_SYSTEMD_CGROUP,_SYSTEMD_UNIT,"+
|
||||
"_SYSTEMD_SLICE,CODE_FILE,CODE_LINE,CODE_FUNC,SYSLOG_IDENTIFIER,_COMM,_EXE,_CMDLINE,MESSAGE,_PID,_SOURCE_REALTIME_TIMESTAMP,_REALTIME_TIMESTAMP",
|
||||
",")
|
||||
|
||||
func BenchmarkPushJournaldPerformance(b *testing.B) {
|
||||
cp := &insertutil.CommonParams{
|
||||
TimeFields: []string{"__REALTIME_TIMESTAMP"},
|
||||
MsgFields: []string{"MESSAGE"},
|
||||
}
|
||||
const dataChunkSize = 1024 * 1024
|
||||
|
||||
data := generateJournaldData(dataChunkSize)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(data)))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
r := &bytes.Reader{}
|
||||
blp := &insertutil.BenchmarkLogMessageProcessor{}
|
||||
for pb.Next() {
|
||||
r.Reset(data)
|
||||
if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func generateJournaldData(size int) []byte {
|
||||
var buf []byte
|
||||
timestamp := time.Now().UnixMicro()
|
||||
@@ -80,3 +37,26 @@ func generateJournaldData(size int) []byte {
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
func BenchmarkPushJournaldPerformance(b *testing.B) {
|
||||
cp := &insertutil.CommonParams{
|
||||
TimeFields: []string{"__REALTIME_TIMESTAMP"},
|
||||
MsgFields: []string{"MESSAGE"},
|
||||
}
|
||||
const dataChunkSize = 1024 * 1024
|
||||
|
||||
data := generateJournaldData(dataChunkSize)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(data)))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
r := &bytes.Reader{}
|
||||
blp := &insertutil.BenchmarkLogMessageProcessor{}
|
||||
for pb.Next() {
|
||||
r.Reset(data)
|
||||
if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -119,5 +119,5 @@ var (
|
||||
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
|
||||
|
||||
requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
|
||||
requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
|
||||
)
|
||||
|
||||
@@ -58,7 +58,7 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
|
||||
|
||||
var (
|
||||
requestsJSONTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
requestJSONDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
)
|
||||
|
||||
func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
|
||||
|
||||
@@ -62,7 +62,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
|
||||
var (
|
||||
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
)
|
||||
|
||||
func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
|
||||
|
||||
@@ -70,7 +70,7 @@ var (
|
||||
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
|
||||
requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
)
|
||||
|
||||
func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) error {
|
||||
|
||||
@@ -101,7 +101,7 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
||||
currentYear := 2023
|
||||
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
|
||||
resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
|
||||
{"priority":"165","facility_keyword":"local4","level":"notice","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"165","facility_keyword":"local4","level":"info","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"123","facility_keyword":"solaris-cron","level":"error","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
|
||||
f(data, currentYear, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
@@ -742,23 +742,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
|
||||
|
||||
#### rate_prometheus
|
||||
|
||||
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
|
||||
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
|
||||
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
|
||||
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
|
||||
|
||||
See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
|
||||
|
||||
See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rate_over_sum
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
208
app/vlselect/vmui/assets/index-DhqzKCNf.js
Normal file
208
app/vlselect/vmui/assets/index-DhqzKCNf.js
Normal file
File diff suppressed because one or more lines are too long
67
app/vlselect/vmui/assets/vendor-D8IJGiEn.js
Normal file
67
app/vlselect/vmui/assets/vendor-D8IJGiEn.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -35,10 +35,10 @@
|
||||
<meta property="og:title" content="UI for VictoriaLogs">
|
||||
<meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
|
||||
<meta property="og:description" content="Explore your log data with VictoriaLogs UI">
|
||||
<script type="module" crossorigin src="./assets/index-721xTF8u.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-V4vnRsM-.js">
|
||||
<script type="module" crossorigin src="./assets/index-DhqzKCNf.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-C36SC0pJ.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
||||
@@ -89,18 +89,15 @@ func (t *Type) ValidateExpr(expr string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SupportedType is true if given datasource type is supported
|
||||
func SupportedType(dsType string) bool {
|
||||
return dsType == "graphite" || dsType == "prometheus" || dsType == "vlogs"
|
||||
}
|
||||
|
||||
// UnmarshalYAML implements the yaml.Unmarshaler interface.
|
||||
func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
|
||||
var s string
|
||||
if err := unmarshal(&s); err != nil {
|
||||
return err
|
||||
}
|
||||
if !SupportedType(s) {
|
||||
switch s {
|
||||
case "graphite", "prometheus", "vlogs":
|
||||
default:
|
||||
return fmt.Errorf("unknown datasource type=%q, want prometheus, graphite or vlogs", s)
|
||||
}
|
||||
t.Name = s
|
||||
|
||||
@@ -148,13 +148,9 @@ func main() {
|
||||
if err != nil {
|
||||
logger.Fatalf("failed to init datasource: %s", err)
|
||||
}
|
||||
totalRows, droppedRows, err := replay(groupsCfg, q, rw)
|
||||
if err != nil {
|
||||
if err := replay(groupsCfg, q, rw); err != nil {
|
||||
logger.Fatalf("replay failed: %s", err)
|
||||
}
|
||||
if droppedRows > 0 {
|
||||
logger.Fatalf("failed to push all generated samples to remote write url, dropped %d samples out of %d", droppedRows, totalRows)
|
||||
}
|
||||
logger.Infof("replay succeed!")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -194,10 +194,9 @@ func mergeLabels(target string, metaLabels *promutil.Labels, cfg *Config) *promu
|
||||
alertsPath = address[n:]
|
||||
address = address[:n]
|
||||
}
|
||||
m.Add("__address__", address)
|
||||
m.Add("__scheme__", scheme)
|
||||
m.Add("__alerts_path__", alertsPath)
|
||||
m.AddFrom(metaLabels)
|
||||
// force labels
|
||||
m.Set("__address__", address)
|
||||
m.Set("__scheme__", scheme)
|
||||
m.Set("__alerts_path__", alertsPath)
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package notifier
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -54,11 +53,8 @@ func (cw *configWatcher) notifiers() []Notifier {
|
||||
for _, n := range ns {
|
||||
notifiers = append(notifiers, n.Notifier)
|
||||
}
|
||||
|
||||
}
|
||||
// deterministically sort the output
|
||||
sort.Slice(notifiers, func(i, j int) bool {
|
||||
return notifiers[i].Addr() < notifiers[j].Addr()
|
||||
})
|
||||
return notifiers
|
||||
}
|
||||
|
||||
@@ -89,12 +85,12 @@ func (cw *configWatcher) reload(path string) error {
|
||||
}
|
||||
|
||||
func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn getLabels) error {
|
||||
targetMetadata, errors := getTargetMetadata(labelsFn, cw.cfg)
|
||||
targets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
|
||||
for _, err := range errors {
|
||||
return fmt.Errorf("failed to init notifier for %q: %w", typeK, err)
|
||||
}
|
||||
|
||||
cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
|
||||
cw.setTargets(typeK, targets)
|
||||
|
||||
cw.wg.Add(1)
|
||||
go func() {
|
||||
@@ -109,22 +105,22 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
targetMetadata, errors := getTargetMetadata(labelsFn, cw.cfg)
|
||||
updateTargets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
|
||||
for _, err := range errors {
|
||||
logger.Errorf("failed to init notifier for %q: %w", typeK, err)
|
||||
}
|
||||
cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
|
||||
cw.setTargets(typeK, updateTargets)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func getTargetMetadata(labelsFn getLabels, cfg *Config) (map[string]*promutil.Labels, []error) {
|
||||
func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator) ([]Target, []error) {
|
||||
metaLabels, err := labelsFn()
|
||||
if err != nil {
|
||||
return nil, []error{fmt.Errorf("failed to get labels: %w", err)}
|
||||
}
|
||||
targetMetadata := make(map[string]*promutil.Labels, len(metaLabels))
|
||||
var targets []Target
|
||||
var errors []error
|
||||
duplicates := make(map[string]struct{})
|
||||
for _, labels := range metaLabels {
|
||||
@@ -147,9 +143,18 @@ func getTargetMetadata(labelsFn getLabels, cfg *Config) (map[string]*promutil.La
|
||||
continue
|
||||
}
|
||||
duplicates[u] = struct{}{}
|
||||
targetMetadata[u] = processedLabels
|
||||
|
||||
am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
|
||||
if err != nil {
|
||||
errors = append(errors, err)
|
||||
continue
|
||||
}
|
||||
targets = append(targets, Target{
|
||||
Notifier: am,
|
||||
Labels: processedLabels,
|
||||
})
|
||||
}
|
||||
return targetMetadata, errors
|
||||
return targets, errors
|
||||
}
|
||||
|
||||
type getLabels func() ([]*promutil.Labels, error)
|
||||
@@ -236,40 +241,21 @@ func (cw *configWatcher) mustStop() {
|
||||
|
||||
func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
|
||||
cw.targetsMu.Lock()
|
||||
newT := make(map[string]Target)
|
||||
for _, t := range targets {
|
||||
newT[t.Addr()] = t
|
||||
}
|
||||
oldT := cw.targets[key]
|
||||
|
||||
for _, ot := range oldT {
|
||||
if _, ok := newT[ot.Addr()]; !ok {
|
||||
ot.Notifier.Close()
|
||||
}
|
||||
}
|
||||
cw.targets[key] = targets
|
||||
cw.targetsMu.Unlock()
|
||||
}
|
||||
|
||||
func (cw *configWatcher) updateTargets(key TargetType, targetMetadata map[string]*promutil.Labels, cfg *Config, genFn AlertURLGenerator) {
|
||||
cw.targetsMu.Lock()
|
||||
defer cw.targetsMu.Unlock()
|
||||
oldTargets := cw.targets[key]
|
||||
var updatedTargets []Target
|
||||
for _, ot := range oldTargets {
|
||||
if _, ok := targetMetadata[ot.Addr()]; !ok {
|
||||
// if target not exists in currentTargets, close it
|
||||
ot.Notifier.Close()
|
||||
} else {
|
||||
updatedTargets = append(updatedTargets, ot)
|
||||
delete(targetMetadata, ot.Addr())
|
||||
}
|
||||
}
|
||||
// create new resources for the new targets
|
||||
for addr, labels := range targetMetadata {
|
||||
am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
|
||||
if err != nil {
|
||||
logger.Errorf("failed to init %s notifier with addr %q: %w", key, addr, err)
|
||||
continue
|
||||
}
|
||||
updatedTargets = append(updatedTargets, Target{
|
||||
Notifier: am,
|
||||
Labels: labels,
|
||||
})
|
||||
}
|
||||
|
||||
cw.targets[key] = updatedTargets
|
||||
}
|
||||
|
||||
// mergeHTTPClientConfigs merges fields between child and parent params
|
||||
// by populating child from parent params if they're missing.
|
||||
func mergeHTTPClientConfigs(parent, child promauth.HTTPClientConfig) promauth.HTTPClientConfig {
|
||||
|
||||
@@ -8,10 +8,8 @@ import (
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
|
||||
)
|
||||
|
||||
func TestConfigWatcherReload(t *testing.T) {
|
||||
@@ -61,11 +59,6 @@ static_configs:
|
||||
}
|
||||
|
||||
func TestConfigWatcherStart(t *testing.T) {
|
||||
oldSDCheckInterval := consul.SDCheckInterval
|
||||
defer func() { consul.SDCheckInterval = oldSDCheckInterval }()
|
||||
consulCheckInterval := 100 * time.Millisecond
|
||||
consul.SDCheckInterval = &consulCheckInterval
|
||||
|
||||
consulSDServer := newFakeConsulServer()
|
||||
defer consulSDServer.Close()
|
||||
|
||||
@@ -104,11 +97,6 @@ consul_sd_configs:
|
||||
if n2.Addr() != expAddr2 {
|
||||
t.Fatalf("exp address %q; got %q", expAddr2, n2.Addr())
|
||||
}
|
||||
|
||||
f := func() bool { return len(cw.notifiers()) == 1 }
|
||||
if !waitFor(f, time.Second) {
|
||||
t.Fatalf("expected to get 1 notifiers; got %d", len(cw.notifiers()))
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigWatcherReloadConcurrent supposed to test concurrent
|
||||
@@ -205,7 +193,6 @@ const (
|
||||
)
|
||||
|
||||
func newFakeConsulServer() *httptest.Server {
|
||||
requestCount := 0
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/agent/self", func(rw http.ResponseWriter, _ *http.Request) {
|
||||
rw.Write([]byte(`{"Config": {"Datacenter": "dc1"}}`))
|
||||
@@ -220,9 +207,8 @@ func newFakeConsulServer() *httptest.Server {
|
||||
}`))
|
||||
})
|
||||
mux.HandleFunc("/v1/health/service/alertmanager", func(rw http.ResponseWriter, _ *http.Request) {
|
||||
if requestCount == 0 {
|
||||
rw.Header().Set("X-Consul-Index", "1")
|
||||
rw.Write([]byte(`
|
||||
rw.Header().Set("X-Consul-Index", "1")
|
||||
rw.Write([]byte(`
|
||||
[
|
||||
{
|
||||
"Node": {
|
||||
@@ -311,56 +297,6 @@ func newFakeConsulServer() *httptest.Server {
|
||||
}
|
||||
}
|
||||
]`))
|
||||
} else {
|
||||
rw.Header().Set("X-Consul-Index", "2")
|
||||
rw.Write([]byte(`
|
||||
[
|
||||
{
|
||||
"Node": {
|
||||
"ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72",
|
||||
"Node": "machine",
|
||||
"Address": "127.0.0.1",
|
||||
"Datacenter": "dc1",
|
||||
"TaggedAddresses": {
|
||||
"lan": "127.0.0.1",
|
||||
"lan_ipv4": "127.0.0.1",
|
||||
"wan": "127.0.0.1",
|
||||
"wan_ipv4": "127.0.0.1"
|
||||
},
|
||||
"Meta": {
|
||||
"consul-network-segment": ""
|
||||
},
|
||||
"CreateIndex": 13,
|
||||
"ModifyIndex": 14
|
||||
},
|
||||
"Service": {
|
||||
"ID": "am3",
|
||||
"Service": "alertmanager",
|
||||
"Tags": [
|
||||
"alertmanager",
|
||||
"__scheme__=http"
|
||||
],
|
||||
"Address": "",
|
||||
"Meta": null,
|
||||
"Port": 9097,
|
||||
"Weights": {
|
||||
"Passing": 1,
|
||||
"Warning": 1
|
||||
},
|
||||
"EnableTagOverride": false,
|
||||
"Proxy": {
|
||||
"Mode": "",
|
||||
"MeshGateway": {},
|
||||
"Expose": {}
|
||||
},
|
||||
"Connect": {},
|
||||
"CreateIndex": 16,
|
||||
"ModifyIndex": 16
|
||||
}
|
||||
}
|
||||
]`))
|
||||
}
|
||||
requestCount++
|
||||
})
|
||||
|
||||
return httptest.NewServer(mux)
|
||||
@@ -421,13 +357,3 @@ func TestParseLabels_Success(t *testing.T) {
|
||||
PathPrefix: "test",
|
||||
}, "https://alertmanager:9093/api/v1/alerts")
|
||||
}
|
||||
|
||||
func waitFor(f func() bool, timeout time.Duration) bool {
|
||||
for start := time.Now(); time.Since(start) < timeout; {
|
||||
if f() == true {
|
||||
return true
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -216,7 +216,7 @@ var (
|
||||
)
|
||||
|
||||
// GetDroppedRows returns value of droppedRows metric
|
||||
func GetDroppedRows() int { return int(droppedRows.Get()) }
|
||||
func GetDroppedRows() int64 { return int64(droppedRows.Get()) }
|
||||
|
||||
// flush is a blocking function that marshals WriteRequest and sends
|
||||
// it to remote-write endpoint. Flush performs limited amount of retries
|
||||
|
||||
@@ -20,8 +20,9 @@ var (
|
||||
"The time filter in RFC3339 format to finish the replay by. E.g. '2020-01-01T20:07:00Z'. "+
|
||||
"By default, is set to the current time.")
|
||||
replayRulesDelay = flag.Duration("replay.rulesDelay", time.Second,
|
||||
"Delay before evaluating the next rule within the group. Is important for chained rules. "+
|
||||
"Keep it equal or bigger than -remoteWrite.flushInterval. When set to >0, replay ignores group's concurrency setting.")
|
||||
"Delay between rules evaluation within the group. Could be important if there are chained rules inside the group "+
|
||||
"and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule."+
|
||||
"Keep it equal or bigger than -remoteWrite.flushInterval.")
|
||||
replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
|
||||
"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
|
||||
replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
|
||||
@@ -30,13 +31,13 @@ var (
|
||||
"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
|
||||
)
|
||||
|
||||
func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {
|
||||
func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) error {
|
||||
if *replayMaxDatapoints < 1 {
|
||||
return 0, 0, fmt.Errorf("replay.maxDatapointsPerQuery can't be lower than 1")
|
||||
return fmt.Errorf("replay.maxDatapointsPerQuery can't be lower than 1")
|
||||
}
|
||||
tFrom, err := time.Parse(time.RFC3339, *replayFrom)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to parse replay.timeFrom=%q: %w", *replayFrom, err)
|
||||
return fmt.Errorf("failed to parse replay.timeFrom=%q: %w", *replayFrom, err)
|
||||
}
|
||||
|
||||
// use tFrom location for default value, otherwise filters could have different locations
|
||||
@@ -44,12 +45,12 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
|
||||
if *replayTo != "" {
|
||||
tTo, err = time.Parse(time.RFC3339, *replayTo)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to parse replay.timeTo=%q: %w", *replayTo, err)
|
||||
return fmt.Errorf("failed to parse replay.timeTo=%q: %w", *replayTo, err)
|
||||
}
|
||||
}
|
||||
|
||||
if !tTo.After(tFrom) {
|
||||
return 0, 0, fmt.Errorf("replay.timeTo=%v must be bigger than replay.timeFrom=%v", tTo, tFrom)
|
||||
return fmt.Errorf("replay.timeTo=%v must be bigger than replay.timeFrom=%v", tTo, tFrom)
|
||||
}
|
||||
labels := make(map[string]string)
|
||||
for _, s := range *externalLabels {
|
||||
@@ -58,7 +59,7 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
|
||||
}
|
||||
n := strings.IndexByte(s, '=')
|
||||
if n < 0 {
|
||||
return 0, 0, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
|
||||
return fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
|
||||
}
|
||||
labels[s[:n]] = s[n+1:]
|
||||
}
|
||||
@@ -69,14 +70,18 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
|
||||
"\nmax data points per request: %d\n",
|
||||
tFrom, tTo, *replayMaxDatapoints)
|
||||
|
||||
var total int
|
||||
for _, cfg := range groupsCfg {
|
||||
ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
|
||||
totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar)
|
||||
total += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar)
|
||||
}
|
||||
logger.Infof("replay evaluation finished, generated %d samples", totalRows)
|
||||
logger.Infof("replay evaluation finished, generated %d samples", total)
|
||||
if err := rw.Close(); err != nil {
|
||||
return 0, 0, err
|
||||
return err
|
||||
}
|
||||
droppedRows = remotewrite.GetDroppedRows()
|
||||
return totalRows, droppedRows, nil
|
||||
droppedRows := remotewrite.GetDroppedRows()
|
||||
if droppedRows > 0 {
|
||||
return fmt.Errorf("failed to push all generated samples to remote write url, dropped %d samples out of %d", droppedRows, total)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,45 +8,38 @@ import (
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
)
|
||||
|
||||
type fakeReplayQuerier struct {
|
||||
datasource.FakeQuerier
|
||||
registry map[string]map[string][]datasource.Metric
|
||||
registry map[string]map[string]struct{}
|
||||
}
|
||||
|
||||
func (fr *fakeReplayQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
|
||||
return fr
|
||||
}
|
||||
|
||||
type fakeRWClient struct{}
|
||||
|
||||
func (fc *fakeRWClient) Push(_ prompbmarshal.TimeSeries) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fc *fakeRWClient) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to time.Time) (res datasource.Result, err error) {
|
||||
key := fmt.Sprintf("%s+%s", from.Format("15:04:05"), to.Format("15:04:05"))
|
||||
dps, ok := fr.registry[q]
|
||||
if !ok {
|
||||
return res, fmt.Errorf("unexpected query received: %q", q)
|
||||
}
|
||||
metrics, ok := dps[key]
|
||||
_, ok = dps[key]
|
||||
if !ok {
|
||||
return res, fmt.Errorf("unexpected time range received: %q", key)
|
||||
}
|
||||
res.Data = metrics
|
||||
delete(dps, key)
|
||||
if len(fr.registry[q]) < 1 {
|
||||
delete(fr.registry, q)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func TestReplay(t *testing.T) {
|
||||
f := func(from, to string, maxDP int, ruleDelay time.Duration, cfg []config.Group, qb *fakeReplayQuerier, expectTotalRows int) {
|
||||
f := func(from, to string, maxDP int, cfg []config.Group, qb *fakeReplayQuerier) {
|
||||
t.Helper()
|
||||
|
||||
fromOrig, toOrig, maxDatapointsOrig := *replayFrom, *replayTo, *replayMaxDatapoints
|
||||
@@ -58,172 +51,90 @@ func TestReplay(t *testing.T) {
|
||||
}()
|
||||
|
||||
*replayRuleRetryAttempts = 1
|
||||
*replayRulesDelay = ruleDelay
|
||||
rwb := &fakeRWClient{}
|
||||
*replayRulesDelay = time.Millisecond
|
||||
rwb := &remotewrite.DebugClient{}
|
||||
*replayFrom = from
|
||||
*replayTo = to
|
||||
*replayMaxDatapoints = maxDP
|
||||
totalRows, _, err := replay(cfg, qb, rwb)
|
||||
if err != nil {
|
||||
if err := replay(cfg, qb, rwb); err != nil {
|
||||
t.Fatalf("replay failed: %s", err)
|
||||
}
|
||||
if totalRows != expectTotalRows {
|
||||
t.Fatalf("unexpected total rows count: got %d, want %d", totalRows, expectTotalRows)
|
||||
if len(qb.registry) > 0 {
|
||||
t.Fatalf("not all requests were sent: %#v", qb.registry)
|
||||
}
|
||||
}
|
||||
|
||||
// one rule + one response
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, []config.Group{
|
||||
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string][]datasource.Metric{
|
||||
"sum(up)": {"12:00:00+12:02:00": {
|
||||
{
|
||||
Timestamps: []int64{1},
|
||||
Values: []float64{1},
|
||||
},
|
||||
}},
|
||||
registry: map[string]map[string]struct{}{
|
||||
"sum(up)": {"12:00:00+12:02:00": {}},
|
||||
},
|
||||
}, 1)
|
||||
})
|
||||
|
||||
// one rule + multiple responses
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
|
||||
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string][]datasource.Metric{
|
||||
registry: map[string]map[string]struct{}{
|
||||
"sum(up)": {
|
||||
"12:00:00+12:01:00": {
|
||||
{
|
||||
Timestamps: []int64{1},
|
||||
Values: []float64{1},
|
||||
},
|
||||
},
|
||||
"12:00:00+12:01:00": {},
|
||||
"12:01:00+12:02:00": {},
|
||||
"12:02:00+12:02:30": {
|
||||
{
|
||||
Timestamps: []int64{1},
|
||||
Values: []float64{1},
|
||||
},
|
||||
},
|
||||
"12:02:00+12:02:30": {},
|
||||
},
|
||||
},
|
||||
}, 2)
|
||||
})
|
||||
|
||||
// datapoints per step
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, []config.Group{
|
||||
{Interval: promutil.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string][]datasource.Metric{
|
||||
registry: map[string]map[string]struct{}{
|
||||
"sum(up)": {
|
||||
"12:00:00+13:00:00": {
|
||||
{
|
||||
Timestamps: []int64{1, 2},
|
||||
Values: []float64{1, 2},
|
||||
},
|
||||
},
|
||||
"13:00:00+14:00:00": {
|
||||
{
|
||||
Timestamps: []int64{1},
|
||||
Values: []float64{1},
|
||||
},
|
||||
},
|
||||
"12:00:00+13:00:00": {},
|
||||
"13:00:00+14:00:00": {},
|
||||
"14:00:00+15:00:00": {},
|
||||
"15:00:00+15:02:30": {},
|
||||
},
|
||||
},
|
||||
}, 3)
|
||||
})
|
||||
|
||||
// multiple recording rules + multiple responses
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
|
||||
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
|
||||
{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string][]datasource.Metric{
|
||||
registry: map[string]map[string]struct{}{
|
||||
"sum(up)": {
|
||||
"12:00:00+12:01:00": {
|
||||
{
|
||||
Timestamps: []int64{1, 2},
|
||||
Values: []float64{1, 2},
|
||||
},
|
||||
},
|
||||
"12:00:00+12:01:00": {},
|
||||
"12:01:00+12:02:00": {},
|
||||
"12:02:00+12:02:30": {},
|
||||
},
|
||||
"max(up)": {
|
||||
"12:00:00+12:01:00": {},
|
||||
"12:01:00+12:02:00": {
|
||||
{
|
||||
Timestamps: []int64{1, 2},
|
||||
Values: []float64{1, 2},
|
||||
},
|
||||
},
|
||||
"12:01:00+12:02:00": {},
|
||||
"12:02:00+12:02:30": {},
|
||||
},
|
||||
},
|
||||
}, 4)
|
||||
})
|
||||
|
||||
// multiple alerting rules + multiple responses
|
||||
// alerting rule generates two series `ALERTS` and `ALERTS_FOR_STATE` when triggered
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
|
||||
{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
|
||||
{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string][]datasource.Metric{
|
||||
registry: map[string]map[string]struct{}{
|
||||
"sum(up) > 1": {
|
||||
"12:00:00+12:01:00": {
|
||||
{
|
||||
Timestamps: []int64{1, 2},
|
||||
Values: []float64{1, 2},
|
||||
},
|
||||
},
|
||||
"12:00:00+12:01:00": {},
|
||||
"12:01:00+12:02:00": {},
|
||||
"12:02:00+12:02:30": {},
|
||||
},
|
||||
"max(up) < 1": {
|
||||
"12:00:00+12:01:00": {
|
||||
{
|
||||
Timestamps: []int64{1},
|
||||
Values: []float64{1},
|
||||
},
|
||||
},
|
||||
"12:01:00+12:02:00": {},
|
||||
"12:02:00+12:02:30": {},
|
||||
},
|
||||
},
|
||||
}, 6)
|
||||
|
||||
// multiple recording rules in one group+ multiple responses + concurrency
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 0, []config.Group{
|
||||
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up) > 1"}, {Record: "bar", Expr: "max(up) < 1"}}, Concurrency: 2}}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string][]datasource.Metric{
|
||||
"sum(up) > 1": {
|
||||
"12:00:00+12:01:00": {
|
||||
{
|
||||
Timestamps: []int64{1},
|
||||
Values: []float64{1},
|
||||
},
|
||||
},
|
||||
"12:01:00+12:02:00": {
|
||||
{
|
||||
Timestamps: []int64{1},
|
||||
Values: []float64{1},
|
||||
},
|
||||
},
|
||||
"12:02:00+12:02:30": {
|
||||
{
|
||||
Timestamps: []int64{1},
|
||||
Values: []float64{1},
|
||||
},
|
||||
},
|
||||
},
|
||||
"max(up) < 1": {
|
||||
"12:00:00+12:01:00": {},
|
||||
"12:01:00+12:02:00": {{
|
||||
Timestamps: []int64{1},
|
||||
Values: []float64{1},
|
||||
}},
|
||||
"12:01:00+12:02:00": {},
|
||||
"12:02:00+12:02:30": {},
|
||||
},
|
||||
},
|
||||
}, 4)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -445,17 +445,11 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
|
||||
|
||||
g.infof("re-started")
|
||||
case <-t.C:
|
||||
// calculate the real wall clock offset by stripping the monotonic clock first,
|
||||
// then evalTS can be corrected when wall clock is adjusted.
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8790#issuecomment-2986541829
|
||||
offset := time.Now().Round(0).Sub(evalTS.Round(0))
|
||||
missed := (offset / g.Interval) - 1
|
||||
missed := (time.Since(evalTS) / g.Interval) - 1
|
||||
if missed < 0 {
|
||||
// missed can become < 0 due to irregular delays during evaluation
|
||||
// which can result in time.Since(evalTS) < g.Interval;
|
||||
// or the system wall clock was changed backward
|
||||
// which can result in time.Since(evalTS) < g.Interval
|
||||
missed = 0
|
||||
evalTS = time.Now()
|
||||
}
|
||||
if missed > 0 {
|
||||
g.metrics.iterationMissed.Inc()
|
||||
@@ -520,84 +514,36 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
|
||||
iterations := int(end.Sub(start)/step) + 1
|
||||
fmt.Printf("\nGroup %q"+
|
||||
"\ninterval: \t%v"+
|
||||
"\nconcurrency: \t %d"+
|
||||
"\nrequests to make per rule: \t%d"+
|
||||
"\nrequests to make: \t%d"+
|
||||
"\nmax range per request: \t%v\n",
|
||||
g.Name, g.Interval, g.Concurrency, iterations, step)
|
||||
g.Name, g.Interval, iterations, step)
|
||||
if g.Limit > 0 {
|
||||
fmt.Printf("\nWarning: `limit: %d` param has no effect during replay.\n",
|
||||
fmt.Printf("\nPlease note, `limit: %d` param has no effect during replay.\n",
|
||||
g.Limit)
|
||||
}
|
||||
concurrency := g.Concurrency
|
||||
if g.Concurrency > 1 && replayDelay > 0 {
|
||||
fmt.Printf("\nWarning: group concurrency %d will be ignored since `-replay.rulesDelay` is %.3f seconds."+
|
||||
" Set -replay.rulesDelay=0 to enable concurrency for replay.\n", g.Concurrency, replayDelay.Seconds())
|
||||
concurrency = 1
|
||||
}
|
||||
|
||||
if concurrency == 1 {
|
||||
for _, rule := range g.Rules {
|
||||
var bar *pb.ProgressBar
|
||||
if !disableProgressBar {
|
||||
bar = pb.StartNew(iterations)
|
||||
}
|
||||
// pass ri as a copy, so it can be modified within the replayRuleRange
|
||||
total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts)
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
// sleep to let remote storage to flush data on-disk
|
||||
// so chained rules could be calculated correctly
|
||||
time.Sleep(replayDelay)
|
||||
for _, rule := range g.Rules {
|
||||
fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
|
||||
var bar *pb.ProgressBar
|
||||
if !disableProgressBar {
|
||||
bar = pb.StartNew(iterations)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
sem := make(chan struct{}, g.Concurrency)
|
||||
res := make(chan int, len(g.Rules)*iterations)
|
||||
wg := sync.WaitGroup{}
|
||||
var bar *pb.ProgressBar
|
||||
if !disableProgressBar {
|
||||
bar = pb.StartNew(iterations * len(g.Rules))
|
||||
}
|
||||
for _, r := range g.Rules {
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(r Rule, ri rangeIterator) {
|
||||
// pass ri as a copy, so it can be modified within the replayRuleRange
|
||||
res <- replayRuleRange(r, ri, bar, rw, replayRuleRetryAttempts)
|
||||
<-sem
|
||||
wg.Done()
|
||||
}(r, ri)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(res)
|
||||
close(sem)
|
||||
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
|
||||
total = 0
|
||||
for n := range res {
|
||||
total += n
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts int) int {
|
||||
fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
|
||||
total := 0
|
||||
for ri.next() {
|
||||
n, err := replayRule(r, ri.s, ri.e, rw, replayRuleRetryAttempts)
|
||||
if err != nil {
|
||||
logger.Fatalf("rule %q: %s", r, err)
|
||||
ri.reset()
|
||||
for ri.next() {
|
||||
n, err := replayRule(rule, ri.s, ri.e, rw, replayRuleRetryAttempts)
|
||||
if err != nil {
|
||||
logger.Fatalf("rule %q: %s", rule, err)
|
||||
}
|
||||
total += n
|
||||
if bar != nil {
|
||||
bar.Increment()
|
||||
}
|
||||
}
|
||||
if bar != nil {
|
||||
bar.Increment()
|
||||
bar.Finish()
|
||||
}
|
||||
total += n
|
||||
// sleep to let remote storage to flush data on-disk
|
||||
// so chained rules could be calculated correctly
|
||||
time.Sleep(replayDelay)
|
||||
}
|
||||
return total
|
||||
}
|
||||
@@ -624,10 +570,11 @@ type rangeIterator struct {
|
||||
s, e time.Time
|
||||
}
|
||||
|
||||
// next iterates with given step between start and end
|
||||
// by modifying iter, s and e.
|
||||
// Returns true until it reaches end.
|
||||
// next modifies ri and isn't thread-safe.
|
||||
func (ri *rangeIterator) reset() {
|
||||
ri.iter = 0
|
||||
ri.s, ri.e = time.Time{}, time.Time{}
|
||||
}
|
||||
|
||||
func (ri *rangeIterator) next() bool {
|
||||
ri.s = ri.start.Add(ri.step * time.Duration(ri.iter))
|
||||
if !ri.end.After(ri.s) {
|
||||
|
||||
@@ -99,15 +99,3 @@ textarea.curl-area {
|
||||
padding: 0;
|
||||
overflow: scroll;
|
||||
}
|
||||
|
||||
.w-10 {
|
||||
width: 10%;
|
||||
}
|
||||
|
||||
.w-20 {
|
||||
width: 20%;
|
||||
}
|
||||
|
||||
.w-60 {
|
||||
width: 60%;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
|
||||
@@ -28,7 +27,6 @@ var (
|
||||
// such as Grafana, and proxied via vmselect.
|
||||
{"api/v1/rules", "list all loaded groups and rules"},
|
||||
{"api/v1/alerts", "list all active alerts"},
|
||||
{"api/v1/notifiers", "list all notifiers"},
|
||||
{fmt.Sprintf("api/v1/alert?%s=<int>&%s=<int>", paramGroupID, paramAlertID), "get alert status by group and alert ID"},
|
||||
}
|
||||
systemLinks = [][2]string{
|
||||
@@ -44,10 +42,6 @@ var (
|
||||
{Name: "Notifiers", URL: "notifiers"},
|
||||
{Name: "Docs", URL: "https://docs.victoriametrics.com/victoriametrics/vmalert/"},
|
||||
}
|
||||
ruleTypeMap = map[string]string{
|
||||
"alert": ruleTypeAlerting,
|
||||
"record": ruleTypeRecording,
|
||||
}
|
||||
)
|
||||
|
||||
type requestHandler struct {
|
||||
@@ -95,13 +89,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
|
||||
WriteRuleDetails(w, r, rule)
|
||||
return true
|
||||
case "/vmalert/groups":
|
||||
rf, err := newRulesFilter(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
filter := r.URL.Query().Get("filter")
|
||||
rf := extractRulesFilter(r, filter)
|
||||
data := rh.groups(rf)
|
||||
WriteListGroups(w, r, data, rf.filter)
|
||||
WriteListGroups(w, r, data, filter)
|
||||
return true
|
||||
case "/vmalert/notifiers":
|
||||
WriteListTargets(w, r, notifier.GetTargets())
|
||||
@@ -111,35 +102,23 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
|
||||
// served without `vmalert` prefix:
|
||||
case "/rules":
|
||||
// Grafana makes an extra request to `/rules`
|
||||
// handler in addition to `/api/v1/rules` calls in alerts UI
|
||||
var data []*apiGroup
|
||||
rf, err := newRulesFilter(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
// handler in addition to `/api/v1/rules` calls in alerts UI,
|
||||
var data []apiGroup
|
||||
filter := r.URL.Query().Get("filter")
|
||||
rf := extractRulesFilter(r, filter)
|
||||
data = rh.groups(rf)
|
||||
WriteListGroups(w, r, data, rf.filter)
|
||||
WriteListGroups(w, r, data, filter)
|
||||
return true
|
||||
|
||||
case "/vmalert/api/v1/notifiers", "/api/v1/notifiers":
|
||||
data, err := rh.listNotifiers()
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write(data)
|
||||
return true
|
||||
case "/vmalert/api/v1/rules", "/api/v1/rules":
|
||||
// path used by Grafana for ng alerting
|
||||
var data []byte
|
||||
rf, err := newRulesFilter(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
var err error
|
||||
|
||||
filter := r.URL.Query().Get("filter")
|
||||
rf := extractRulesFilter(r, filter)
|
||||
data, err = rh.listGroups(rf)
|
||||
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
@@ -150,12 +129,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
|
||||
|
||||
case "/vmalert/api/v1/alerts", "/api/v1/alerts":
|
||||
// path used by Grafana for ng alerting
|
||||
rf, err := newRulesFilter(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
data, err := rh.listAlerts(rf)
|
||||
data, err := rh.listAlerts()
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
@@ -244,7 +218,7 @@ func (rh *requestHandler) getAlert(r *http.Request) (*apiAlert, error) {
|
||||
type listGroupsResponse struct {
|
||||
Status string `json:"status"`
|
||||
Data struct {
|
||||
Groups []*apiGroup `json:"groups"`
|
||||
Groups []apiGroup `json:"groups"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
@@ -255,102 +229,82 @@ type rulesFilter struct {
|
||||
ruleNames []string
|
||||
ruleType string
|
||||
excludeAlerts bool
|
||||
filter string
|
||||
dsType config.Type
|
||||
onlyUnhealthy bool
|
||||
onlyNoMatch bool
|
||||
}
|
||||
|
||||
func newRulesFilter(r *http.Request) (*rulesFilter, error) {
|
||||
rf := &rulesFilter{}
|
||||
query := r.URL.Query()
|
||||
func extractRulesFilter(r *http.Request, filter string) rulesFilter {
|
||||
rf := rulesFilter{}
|
||||
|
||||
ruleTypeParam := query.Get("type")
|
||||
if len(ruleTypeParam) > 0 {
|
||||
if ruleType, ok := ruleTypeMap[ruleTypeParam]; ok {
|
||||
rf.ruleType = ruleType
|
||||
} else {
|
||||
return nil, errResponse(fmt.Errorf(`invalid parameter "type": not supported value %q`, ruleTypeParam), http.StatusBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
dsType := query.Get("datasource_type")
|
||||
if len(dsType) > 0 {
|
||||
if config.SupportedType(dsType) {
|
||||
rf.dsType = config.NewRawType(dsType)
|
||||
} else {
|
||||
return nil, errResponse(fmt.Errorf(`invalid parameter "datasource_type": not supported value %q`, dsType), http.StatusBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
filter := strings.ToLower(query.Get("filter"))
|
||||
if len(filter) > 0 {
|
||||
if filter == "nomatch" || filter == "unhealthy" {
|
||||
rf.filter = filter
|
||||
} else {
|
||||
return nil, errResponse(fmt.Errorf(`invalid parameter "filter": not supported value %q`, filter), http.StatusBadRequest)
|
||||
}
|
||||
var ruleType string
|
||||
ruleTypeParam := r.URL.Query().Get("type")
|
||||
// for some reason, `type` in filter doesn't match `type` in response,
|
||||
// so we use this matching here
|
||||
if ruleTypeParam == "alert" {
|
||||
ruleType = ruleTypeAlerting
|
||||
} else if ruleTypeParam == "record" {
|
||||
ruleType = ruleTypeRecording
|
||||
}
|
||||
rf.ruleType = ruleType
|
||||
|
||||
rf.excludeAlerts = httputil.GetBool(r, "exclude_alerts")
|
||||
rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
|
||||
rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
|
||||
rf.files = append([]string{}, r.Form["file[]"]...)
|
||||
return rf, nil
|
||||
switch filter {
|
||||
case "unhealthy":
|
||||
rf.onlyUnhealthy = true
|
||||
case "noMatch":
|
||||
rf.onlyNoMatch = true
|
||||
}
|
||||
return rf
|
||||
}
|
||||
|
||||
func (rf *rulesFilter) matchesGroup(group *rule.Group) bool {
|
||||
if len(rf.groupNames) > 0 && !slices.Contains(rf.groupNames, group.Name) {
|
||||
return false
|
||||
}
|
||||
if len(rf.files) > 0 && !slices.Contains(rf.files, group.File) {
|
||||
return false
|
||||
}
|
||||
if len(rf.dsType.Name) > 0 && rf.dsType.String() != group.Type.String() {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (rh *requestHandler) groups(rf *rulesFilter) []*apiGroup {
|
||||
func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
|
||||
rh.m.groupsMu.RLock()
|
||||
defer rh.m.groupsMu.RUnlock()
|
||||
|
||||
groups := make([]*apiGroup, 0)
|
||||
groups := make([]apiGroup, 0)
|
||||
for _, group := range rh.m.groups {
|
||||
if !rf.matchesGroup(group) {
|
||||
if len(rf.groupNames) > 0 && !slices.Contains(rf.groupNames, group.Name) {
|
||||
continue
|
||||
}
|
||||
if len(rf.files) > 0 && !slices.Contains(rf.files, group.File) {
|
||||
continue
|
||||
}
|
||||
|
||||
g := groupToAPI(group)
|
||||
// the returned list should always be non-nil
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
|
||||
filteredRules := make([]apiRule, 0)
|
||||
for _, rule := range g.Rules {
|
||||
if rf.ruleType != "" && rf.ruleType != rule.Type {
|
||||
for _, r := range g.Rules {
|
||||
if rf.ruleType != "" && rf.ruleType != r.Type {
|
||||
continue
|
||||
}
|
||||
if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, rule.Name) {
|
||||
continue
|
||||
}
|
||||
if (rule.LastError == "" && rf.filter == "unhealthy") || (!isNoMatch(rule) && rf.filter == "nomatch") {
|
||||
if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, r.Name) {
|
||||
continue
|
||||
}
|
||||
if rf.excludeAlerts {
|
||||
rule.Alerts = nil
|
||||
r.Alerts = nil
|
||||
}
|
||||
if rule.LastError != "" {
|
||||
if (r.LastError == "" && rf.onlyUnhealthy) || (!isNoMatch(r) && rf.onlyNoMatch) {
|
||||
continue
|
||||
}
|
||||
if r.LastError != "" {
|
||||
g.Unhealthy++
|
||||
} else {
|
||||
g.Healthy++
|
||||
}
|
||||
if isNoMatch(rule) {
|
||||
if isNoMatch(r) {
|
||||
g.NoMatch++
|
||||
}
|
||||
filteredRules = append(filteredRules, rule)
|
||||
filteredRules = append(filteredRules, r)
|
||||
}
|
||||
g.Rules = filteredRules
|
||||
groups = append(groups, g)
|
||||
}
|
||||
// sort list of groups for deterministic output
|
||||
slices.SortFunc(groups, func(a, b *apiGroup) int {
|
||||
slices.SortFunc(groups, func(a, b apiGroup) int {
|
||||
if a.Name != b.Name {
|
||||
return strings.Compare(a.Name, b.Name)
|
||||
}
|
||||
@@ -359,7 +313,7 @@ func (rh *requestHandler) groups(rf *rulesFilter) []*apiGroup {
|
||||
return groups
|
||||
}
|
||||
|
||||
func (rh *requestHandler) listGroups(rf *rulesFilter) ([]byte, error) {
|
||||
func (rh *requestHandler) listGroups(rf rulesFilter) ([]byte, error) {
|
||||
lr := listGroupsResponse{Status: "success"}
|
||||
lr.Data.Groups = rh.groups(rf)
|
||||
b, err := json.Marshal(lr)
|
||||
@@ -406,17 +360,14 @@ func (rh *requestHandler) groupAlerts() []groupAlerts {
|
||||
return gAlerts
|
||||
}
|
||||
|
||||
func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
|
||||
func (rh *requestHandler) listAlerts() ([]byte, error) {
|
||||
rh.m.groupsMu.RLock()
|
||||
defer rh.m.groupsMu.RUnlock()
|
||||
|
||||
lr := listAlertsResponse{Status: "success"}
|
||||
lr.Data.Alerts = make([]*apiAlert, 0)
|
||||
for _, group := range rh.m.groups {
|
||||
if !rf.matchesGroup(group) {
|
||||
continue
|
||||
}
|
||||
for _, r := range group.Rules {
|
||||
for _, g := range rh.m.groups {
|
||||
for _, r := range g.Rules {
|
||||
a, ok := r.(*rule.AlertingRule)
|
||||
if !ok {
|
||||
continue
|
||||
@@ -440,42 +391,6 @@ func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
|
||||
return b, nil
|
||||
}
|
||||
|
||||
type listNotifiersResponse struct {
|
||||
Status string `json:"status"`
|
||||
Data struct {
|
||||
Notifiers []*apiNotifier `json:"notifiers"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (rh *requestHandler) listNotifiers() ([]byte, error) {
|
||||
targets := notifier.GetTargets()
|
||||
|
||||
lr := listNotifiersResponse{Status: "success"}
|
||||
lr.Data.Notifiers = make([]*apiNotifier, 0)
|
||||
for protoName, protoTargets := range targets {
|
||||
notifier := &apiNotifier{
|
||||
Kind: string(protoName),
|
||||
Targets: make([]*apiTarget, 0, len(protoTargets)),
|
||||
}
|
||||
for _, target := range protoTargets {
|
||||
notifier.Targets = append(notifier.Targets, &apiTarget{
|
||||
Address: target.Notifier.Addr(),
|
||||
Labels: target.Labels.ToMap(),
|
||||
})
|
||||
}
|
||||
lr.Data.Notifiers = append(lr.Data.Notifiers, notifier)
|
||||
}
|
||||
|
||||
b, err := json.Marshal(lr)
|
||||
if err != nil {
|
||||
return nil, &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf(`error encoding list of notifiers: %w`, err),
|
||||
StatusCode: http.StatusInternalServerError,
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func errResponse(err error, sc int) *httpserver.ErrorWithStatusCode {
|
||||
return &httpserver.ErrorWithStatusCode{
|
||||
Err: err,
|
||||
|
||||
@@ -93,18 +93,18 @@
|
||||
{%= tpl.Footer(r) %}
|
||||
{% endfunc %}
|
||||
|
||||
{% func ListGroups(r *http.Request, groups []*apiGroup, filter string) %}
|
||||
{% func ListGroups(r *http.Request, groups []apiGroup, filter string) %}
|
||||
{%code
|
||||
prefix := vmalertutil.Prefix(r.URL.Path)
|
||||
filters := map[string]string{
|
||||
"": "All",
|
||||
"unhealthy": "Unhealthy",
|
||||
"nomatch": "No Match",
|
||||
"noMatch": "No Match",
|
||||
}
|
||||
icons := map[string]string{
|
||||
"": "all",
|
||||
"unhealthy": "unhealthy",
|
||||
"nomatch": "nomatch",
|
||||
"noMatch": "nomatch",
|
||||
}
|
||||
currentText := filters[filter]
|
||||
currentIcon := icons[filter]
|
||||
@@ -161,9 +161,9 @@
|
||||
<table class="table table-striped table-hover table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" class="w-60">Rule</th>
|
||||
<th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series</th>
|
||||
<th scope="col" class="w-20" class="text-center" title="How many seconds ago rule was executed">Updated</th>
|
||||
<th scope="col" style="width: 60%">Rule</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many series were produced by the rule">Series</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@@ -594,9 +594,9 @@
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" title="The time when event was created">Updated at</th>
|
||||
<th scope="col" class="w-10 text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
|
||||
{% if seriesFetchedEnabled %}<th scope="col" class="w-10 text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>{% endif %}
|
||||
<th scope="col" class="w-10 text-center" title="How many seconds request took">Duration</th>
|
||||
<th scope="col" style="width: 10%" class="text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
|
||||
{% if seriesFetchedEnabled %}<th scope="col" style="width: 10%" class="text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>{% endif %}
|
||||
<th scope="col" style="width: 10%" class="text-center" title="How many seconds request took">Duration</th>
|
||||
<th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>
|
||||
<th scope="col" class="text-center" title="cURL command with request example">cURL</th>
|
||||
</tr>
|
||||
|
||||
@@ -316,7 +316,7 @@ func Welcome(r *http.Request) string {
|
||||
}
|
||||
|
||||
//line app/vmalert/web.qtpl:96
|
||||
func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*apiGroup, filter string) {
|
||||
func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []apiGroup, filter string) {
|
||||
//line app/vmalert/web.qtpl:96
|
||||
qw422016.N().S(`
|
||||
`)
|
||||
@@ -325,12 +325,12 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*apiG
|
||||
filters := map[string]string{
|
||||
"": "All",
|
||||
"unhealthy": "Unhealthy",
|
||||
"nomatch": "No Match",
|
||||
"noMatch": "No Match",
|
||||
}
|
||||
icons := map[string]string{
|
||||
"": "all",
|
||||
"unhealthy": "unhealthy",
|
||||
"nomatch": "nomatch",
|
||||
"noMatch": "nomatch",
|
||||
}
|
||||
currentText := filters[filter]
|
||||
currentIcon := icons[filter]
|
||||
@@ -523,9 +523,9 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*apiG
|
||||
<table class="table table-striped table-hover table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" class="w-60">Rule</th>
|
||||
<th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series</th>
|
||||
<th scope="col" class="w-20" class="text-center" title="How many seconds ago rule was executed">Updated</th>
|
||||
<th scope="col" style="width: 60%">Rule</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many series were produced by the rule">Series</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@@ -722,7 +722,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*apiG
|
||||
}
|
||||
|
||||
//line app/vmalert/web.qtpl:222
|
||||
func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []*apiGroup, filter string) {
|
||||
func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []apiGroup, filter string) {
|
||||
//line app/vmalert/web.qtpl:222
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmalert/web.qtpl:222
|
||||
@@ -733,7 +733,7 @@ func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []*apiG
|
||||
}
|
||||
|
||||
//line app/vmalert/web.qtpl:222
|
||||
func ListGroups(r *http.Request, groups []*apiGroup, filter string) string {
|
||||
func ListGroups(r *http.Request, groups []apiGroup, filter string) string {
|
||||
//line app/vmalert/web.qtpl:222
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmalert/web.qtpl:222
|
||||
@@ -1697,17 +1697,17 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule apiRule)
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" title="The time when event was created">Updated at</th>
|
||||
<th scope="col" class="w-10 text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
|
||||
<th scope="col" style="width: 10%" class="text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
|
||||
`)
|
||||
//line app/vmalert/web.qtpl:598
|
||||
if seriesFetchedEnabled {
|
||||
//line app/vmalert/web.qtpl:598
|
||||
qw422016.N().S(`<th scope="col" class="w-10 text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>`)
|
||||
qw422016.N().S(`<th scope="col" style="width: 10%" class="text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>`)
|
||||
//line app/vmalert/web.qtpl:598
|
||||
}
|
||||
//line app/vmalert/web.qtpl:598
|
||||
qw422016.N().S(`
|
||||
<th scope="col" class="w-10 text-center" title="How many seconds request took">Duration</th>
|
||||
<th scope="col" style="width: 10%" class="text-center" title="How many seconds request took">Duration</th>
|
||||
<th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>
|
||||
<th scope="col" class="text-center" title="cURL command with request example">cURL</th>
|
||||
</tr>
|
||||
|
||||
@@ -19,34 +19,25 @@ import (
|
||||
func TestHandler(t *testing.T) {
|
||||
fq := &datasource.FakeQuerier{}
|
||||
fq.Add(datasource.Metric{
|
||||
Values: []float64{1},
|
||||
Timestamps: []int64{0},
|
||||
Values: []float64{1}, Timestamps: []int64{0},
|
||||
})
|
||||
m := &manager{groups: map[uint64]*rule.Group{}}
|
||||
var ar *rule.AlertingRule
|
||||
var rr *rule.RecordingRule
|
||||
for _, dsType := range []string{"prometheus", "", "graphite"} {
|
||||
g := rule.NewGroup(config.Group{
|
||||
Name: "group",
|
||||
File: "rules.yaml",
|
||||
Type: config.NewRawType(dsType),
|
||||
Concurrency: 1,
|
||||
Rules: []config.Rule{
|
||||
{
|
||||
ID: 0,
|
||||
Alert: "alert",
|
||||
},
|
||||
{
|
||||
ID: 1,
|
||||
Record: "record",
|
||||
},
|
||||
},
|
||||
}, fq, 1*time.Minute, nil)
|
||||
ar = g.Rules[0].(*rule.AlertingRule)
|
||||
rr = g.Rules[1].(*rule.RecordingRule)
|
||||
g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
|
||||
m.groups[g.CreateID()] = g
|
||||
}
|
||||
g := rule.NewGroup(config.Group{
|
||||
Name: "group",
|
||||
File: "rules.yaml",
|
||||
Concurrency: 1,
|
||||
Rules: []config.Rule{
|
||||
{ID: 0, Alert: "alert"},
|
||||
{ID: 1, Record: "record"},
|
||||
},
|
||||
}, fq, 1*time.Minute, nil)
|
||||
ar := g.Rules[0].(*rule.AlertingRule)
|
||||
rr := g.Rules[1].(*rule.RecordingRule)
|
||||
|
||||
g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
|
||||
|
||||
m := &manager{groups: map[uint64]*rule.Group{
|
||||
g.CreateID(): g,
|
||||
}}
|
||||
rh := &requestHandler{m: m}
|
||||
|
||||
getResp := func(t *testing.T, url string, to any, code int) {
|
||||
@@ -63,7 +54,7 @@ func TestHandler(t *testing.T) {
|
||||
t.Fatalf("err closing body %s", err)
|
||||
}
|
||||
}()
|
||||
if to != nil && code < 300 {
|
||||
if to != nil {
|
||||
if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
|
||||
t.Fatalf("unexpected err %s", err)
|
||||
}
|
||||
@@ -104,23 +95,14 @@ func TestHandler(t *testing.T) {
|
||||
t.Run("/api/v1/alerts", func(t *testing.T) {
|
||||
lr := listAlertsResponse{}
|
||||
getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
|
||||
if length := len(lr.Data.Alerts); length != 3 {
|
||||
t.Fatalf("expected 3 alert got %d", length)
|
||||
if length := len(lr.Data.Alerts); length != 1 {
|
||||
t.Fatalf("expected 1 alert got %d", length)
|
||||
}
|
||||
|
||||
lr = listAlertsResponse{}
|
||||
getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
|
||||
if length := len(lr.Data.Alerts); length != 3 {
|
||||
t.Fatalf("expected 3 alert got %d", length)
|
||||
}
|
||||
|
||||
lr = listAlertsResponse{}
|
||||
getResp(t, ts.URL+"/api/v1/alerts?datasource_type=test", &lr, 400)
|
||||
|
||||
lr = listAlertsResponse{}
|
||||
getResp(t, ts.URL+"/api/v1/alerts?datasource_type=prometheus", &lr, 200)
|
||||
if length := len(lr.Data.Alerts); length != 2 {
|
||||
t.Fatalf("expected 2 alert got %d", length)
|
||||
if length := len(lr.Data.Alerts); length != 1 {
|
||||
t.Fatalf("expected 1 alert got %d", length)
|
||||
}
|
||||
})
|
||||
t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
|
||||
@@ -156,14 +138,14 @@ func TestHandler(t *testing.T) {
|
||||
t.Run("/api/v1/rules", func(t *testing.T) {
|
||||
lr := listGroupsResponse{}
|
||||
getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
|
||||
if length := len(lr.Data.Groups); length != 3 {
|
||||
t.Fatalf("expected 3 group got %d", length)
|
||||
if length := len(lr.Data.Groups); length != 1 {
|
||||
t.Fatalf("expected 1 group got %d", length)
|
||||
}
|
||||
|
||||
lr = listGroupsResponse{}
|
||||
getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
|
||||
if length := len(lr.Data.Groups); length != 3 {
|
||||
t.Fatalf("expected 3 group got %d", length)
|
||||
if length := len(lr.Data.Groups); length != 1 {
|
||||
t.Fatalf("expected 1 group got %d", length)
|
||||
}
|
||||
})
|
||||
t.Run("/api/v1/rule?ruleID&groupID", func(t *testing.T) {
|
||||
@@ -190,10 +172,10 @@ func TestHandler(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("/api/v1/rules&filters", func(t *testing.T) {
|
||||
check := func(url string, statusCode, expGroups, expRules int) {
|
||||
check := func(url string, expGroups, expRules int) {
|
||||
t.Helper()
|
||||
lr := listGroupsResponse{}
|
||||
getResp(t, ts.URL+url, &lr, statusCode)
|
||||
getResp(t, ts.URL+url, &lr, 200)
|
||||
if length := len(lr.Data.Groups); length != expGroups {
|
||||
t.Fatalf("expected %d groups got %d", expGroups, length)
|
||||
}
|
||||
@@ -209,31 +191,25 @@ func TestHandler(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
check("/api/v1/rules?type=alert", 200, 3, 3)
|
||||
check("/api/v1/rules?type=record", 200, 3, 3)
|
||||
check("/api/v1/rules?type=records", 400, 0, 0)
|
||||
check("/api/v1/rules?type=alert", 1, 1)
|
||||
check("/api/v1/rules?type=record", 1, 1)
|
||||
|
||||
check("/vmalert/api/v1/rules?type=alert", 200, 3, 3)
|
||||
check("/vmalert/api/v1/rules?type=record", 200, 3, 3)
|
||||
check("/vmalert/api/v1/rules?type=recording", 400, 0, 0)
|
||||
|
||||
check("/vmalert/api/v1/rules?datasource_type=prometheus", 200, 2, 4)
|
||||
check("/vmalert/api/v1/rules?datasource_type=graphite", 200, 1, 2)
|
||||
check("/vmalert/api/v1/rules?datasource_type=graphiti", 400, 0, 0)
|
||||
check("/vmalert/api/v1/rules?type=alert", 1, 1)
|
||||
check("/vmalert/api/v1/rules?type=record", 1, 1)
|
||||
|
||||
// no filtering expected due to bad params
|
||||
check("/api/v1/rules?type=badParam", 400, 0, 0)
|
||||
check("/api/v1/rules?foo=bar", 200, 3, 6)
|
||||
check("/api/v1/rules?type=badParam", 1, 2)
|
||||
check("/api/v1/rules?foo=bar", 1, 2)
|
||||
|
||||
check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 200, 0, 0)
|
||||
check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 200, 3, 6)
|
||||
check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 0, 0)
|
||||
check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 1, 2)
|
||||
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=foo", 200, 0, 0)
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 200, 3, 6)
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=foo", 0, 0)
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 1, 2)
|
||||
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 200, 3, 0)
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 200, 3, 3)
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 200, 3, 6)
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 1, 0)
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 1, 1)
|
||||
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 1, 2)
|
||||
})
|
||||
t.Run("/api/v1/rules&exclude_alerts=true", func(t *testing.T) {
|
||||
// check if response returns active alerts by default
|
||||
@@ -283,7 +259,7 @@ func TestEmptyResponse(t *testing.T) {
|
||||
t.Fatalf("err closing body %s", err)
|
||||
}
|
||||
}()
|
||||
if to != nil && code < 300 {
|
||||
if to != nil {
|
||||
if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
|
||||
t.Fatalf("unexpected err %s", err)
|
||||
}
|
||||
|
||||
@@ -20,16 +20,6 @@ const (
|
||||
paramRuleID = "rule_id"
|
||||
)
|
||||
|
||||
type apiNotifier struct {
|
||||
Kind string `json:"kind"`
|
||||
Targets []*apiTarget `json:"targets"`
|
||||
}
|
||||
|
||||
type apiTarget struct {
|
||||
Address string `json:"address"`
|
||||
Labels map[string]string `json:"labels"`
|
||||
}
|
||||
|
||||
// apiAlert represents a notifier.AlertingRule state
|
||||
// for WEB view
|
||||
// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
|
||||
@@ -118,7 +108,7 @@ type apiGroup struct {
|
||||
|
||||
// groupAlerts represents a group of alerts for WEB view
|
||||
type groupAlerts struct {
|
||||
Group *apiGroup
|
||||
Group apiGroup
|
||||
Alerts []*apiAlert
|
||||
}
|
||||
|
||||
@@ -337,7 +327,7 @@ func newAlertAPI(ar *rule.AlertingRule, a *notifier.Alert) *apiAlert {
|
||||
return aa
|
||||
}
|
||||
|
||||
func groupToAPI(g *rule.Group) *apiGroup {
|
||||
func groupToAPI(g *rule.Group) apiGroup {
|
||||
g = g.DeepCopy()
|
||||
ag := apiGroup{
|
||||
// encode as string to avoid rounding
|
||||
@@ -363,7 +353,7 @@ func groupToAPI(g *rule.Group) *apiGroup {
|
||||
for _, r := range g.Rules {
|
||||
ag.Rules = append(ag.Rules, ruleToAPI(r))
|
||||
}
|
||||
return &ag
|
||||
return ag
|
||||
}
|
||||
|
||||
func urlValuesToStrings(values url.Values) []string {
|
||||
|
||||
@@ -70,7 +70,7 @@ var (
|
||||
Usage: "VictoriaMetrics address to perform import requests. \n" +
|
||||
"Should be the same as --httpListenAddr value for single-node version or vminsert component. \n" +
|
||||
"When importing into the clustered version do not forget to set additionally --vm-account-id flag. \n" +
|
||||
"Please note, that vmctl performs initial readiness check for the given address by checking /health endpoint.",
|
||||
"Please note, that `vmctl` performs initial readiness check for the given address by checking `/health` endpoint.",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmUser,
|
||||
@@ -514,27 +514,27 @@ var (
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeSrcBearerToken,
|
||||
Usage: "Optional bearer auth token to use for the corresponding --vm-native-src-addr",
|
||||
Usage: "Optional bearer auth token to use for the corresponding `--vm-native-src-addr`",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeSrcCertFile,
|
||||
Usage: "Optional path to client-side TLS certificate file to use when connecting to --vm-native-src-addr",
|
||||
Usage: "Optional path to client-side TLS certificate file to use when connecting to `--vm-native-src-addr`",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeSrcKeyFile,
|
||||
Usage: "Optional path to client-side TLS key to use when connecting to --vm-native-src-addr",
|
||||
Usage: "Optional path to client-side TLS key to use when connecting to `--vm-native-src-addr`",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeSrcCAFile,
|
||||
Usage: "Optional path to TLS CA file to use for verifying connections to --vm-native-src-addr. By default, system CA is used",
|
||||
Usage: "Optional path to TLS CA file to use for verifying connections to `--vm-native-src-addr`. By default, system CA is used",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeSrcServerName,
|
||||
Usage: "Optional TLS server name to use for connections to --vm-native-src-addr. By default, the server name from --vm-native-src-addr is used",
|
||||
Usage: "Optional TLS server name to use for connections to `--vm-native-src-addr`. By default, the server name from `--vm-native-src-addr` is used",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: vmNativeSrcInsecureSkipVerify,
|
||||
Usage: "Whether to skip TLS certificate verification when connecting to --vm-native-src-addr",
|
||||
Usage: "Whether to skip TLS certificate verification when connecting to `--vm-native-src-addr`",
|
||||
Value: false,
|
||||
},
|
||||
|
||||
@@ -563,27 +563,27 @@ var (
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeDstBearerToken,
|
||||
Usage: "Optional bearer auth token to use for the corresponding --vm-native-dst-addr",
|
||||
Usage: "Optional bearer auth token to use for the corresponding `--vm-native-dst-addr`",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeDstCertFile,
|
||||
Usage: "Optional path to client-side TLS certificate file to use when connecting to --vm-native-dst-addr",
|
||||
Usage: "Optional path to client-side TLS certificate file to use when connecting to `--vm-native-dst-addr`",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeDstKeyFile,
|
||||
Usage: "Optional path to client-side TLS key to use when connecting to --vm-native-dst-addr",
|
||||
Usage: "Optional path to client-side TLS key to use when connecting to `--vm-native-dst-addr`",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeDstCAFile,
|
||||
Usage: "Optional path to TLS CA file to use for verifying connections to --vm-native-dst-addr. By default, system CA is used",
|
||||
Usage: "Optional path to TLS CA file to use for verifying connections to `--vm-native-dst-addr`. By default, system CA is used",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: vmNativeDstServerName,
|
||||
Usage: "Optional TLS server name to use for connections to --vm-native-dst-addr. By default, the server name from --vm-native-dst-addr is used",
|
||||
Usage: "Optional TLS server name to use for connections to `--vm-native-dst-addr`. By default, the server name from `--vm-native-dst-addr` is used",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: vmNativeDstInsecureSkipVerify,
|
||||
Usage: "Whether to skip TLS certificate verification when connecting to --vm-native-dst-addr",
|
||||
Usage: "Whether to skip TLS certificate verification when connecting to `--vm-native-dst-addr`",
|
||||
Value: false,
|
||||
},
|
||||
|
||||
@@ -597,7 +597,7 @@ var (
|
||||
Name: vmRateLimit,
|
||||
Usage: "Optional data transfer rate limit in bytes per second.\n" +
|
||||
"By default, the rate limit is disabled. It can be useful for limiting load on source or destination databases. \n" +
|
||||
"Rate limit is applied per worker, see --vm-concurrency.",
|
||||
"Rate limit is applied per worker, see `--vm-concurrency`.",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: vmInterCluster,
|
||||
|
||||
@@ -140,15 +140,6 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU
|
||||
|
||||
written, err := io.Copy(w, reader)
|
||||
if err != nil {
|
||||
// io.Copy could fail if ImportPipe will fail before and close the pr
|
||||
// so we check if that's the case and to not ignore importErr if it exists.
|
||||
select {
|
||||
case importErr := <-importCh:
|
||||
if importErr != nil {
|
||||
return fmt.Errorf("failed to import %s: %w", p.dst.Addr, importErr)
|
||||
}
|
||||
default:
|
||||
}
|
||||
return fmt.Errorf("failed to write into %q: %s", p.dst.Addr, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -562,15 +562,6 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"status":"success","data":{"alerts":[]}}`)
|
||||
return true
|
||||
case "/api/v1/notifiers", "/notifiers":
|
||||
notifiersRequests.Inc()
|
||||
if len(*vmalertProxyURL) > 0 {
|
||||
proxyVMAlertRequests(w, r)
|
||||
return true
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"status":"success","data":{"notifiers":[]}}`)
|
||||
return true
|
||||
case "/api/v1/metadata":
|
||||
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
|
||||
metadataRequests.Inc()
|
||||
@@ -700,10 +691,9 @@ var (
|
||||
expandWithExprsRequests = metrics.NewCounter(`vm_http_requests_total{path="/expand-with-exprs"}`)
|
||||
prettifyQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/prettify-query"}`)
|
||||
|
||||
vmalertRequests = metrics.NewCounter(`vm_http_requests_total{path="/vmalert"}`)
|
||||
rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
|
||||
alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
|
||||
notifiersRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/notifiers"}`)
|
||||
vmalertRequests = metrics.NewCounter(`vm_http_requests_total{path="/vmalert"}`)
|
||||
rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
|
||||
alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
|
||||
|
||||
metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
|
||||
buildInfoRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/buildinfo"}`)
|
||||
|
||||
@@ -377,13 +377,11 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
|
||||
preFunc := func(_ []float64, _ []int64) {}
|
||||
funcName = strings.ToLower(funcName)
|
||||
|
||||
// window > lookbackDelta could result in negative delta.
|
||||
// See issue: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8342
|
||||
stalenessInterval := lookbackDelta
|
||||
if stalenessInterval != 0 {
|
||||
// If stalenessInterval was set, it should additionally account for [window] range to cover following cases:
|
||||
// * window > stalenessInterval, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8342
|
||||
// * window captures prevValue in doInternal while removeCounterResets does not,
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8935#issuecomment-3000735468
|
||||
stalenessInterval += window
|
||||
if stalenessInterval != 0 && stalenessInterval < window {
|
||||
stalenessInterval = window
|
||||
}
|
||||
|
||||
if rollupFuncsRemoveCounterResets[funcName] {
|
||||
|
||||
@@ -742,23 +742,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
|
||||
|
||||
#### rate_prometheus
|
||||
|
||||
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
|
||||
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
|
||||
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
|
||||
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
|
||||
|
||||
See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
|
||||
|
||||
See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rate_over_sum
|
||||
|
||||
File diff suppressed because one or more lines are too long
209
app/vmselect/vmui/assets/index-D-ssBbZq.js
Normal file
209
app/vmselect/vmui/assets/index-D-ssBbZq.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -36,10 +36,10 @@
|
||||
<meta property="og:title" content="UI for VictoriaMetrics">
|
||||
<meta property="og:url" content="https://victoriametrics.com/">
|
||||
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
|
||||
<script type="module" crossorigin src="./assets/index-BiQY-19a.js"></script>
|
||||
<script type="module" crossorigin src="./assets/index-D-ssBbZq.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-ojCMu5lE.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
||||
@@ -63,8 +63,6 @@ var (
|
||||
|
||||
cacheSizeStorageTSID = flagutil.NewBytes("storage.cacheSizeStorageTSID", 0, "Overrides max size for storage/tsid cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
cacheSizeStorageMetricName = flagutil.NewBytes("storage.cacheSizeStorageMetricName", 0, "Overrides max size for storage/metricName cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
cacheSizeIndexDBIndexBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBIndexBlocks", 0, "Overrides max size for indexdb/indexBlocks cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
cacheSizeIndexDBDataBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBDataBlocks", 0, "Overrides max size for indexdb/dataBlocks cache. "+
|
||||
@@ -113,7 +111,6 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
|
||||
storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
|
||||
storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
|
||||
storage.SetMetricNamesStatsCacheSize(cacheSizeMetricNamesStats.IntN())
|
||||
storage.SetMetricNameCacheSize(cacheSizeStorageMetricName.IntN())
|
||||
mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
|
||||
mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
|
||||
mergeset.SetDataBlocksSparseCacheSize(cacheSizeIndexDBDataBlocksSparse.IntN())
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
manifest.json provides metadata used when your web app is installed on a
|
||||
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
|
||||
-->
|
||||
<link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
|
||||
<link rel="manifest" href="/manifest.json"/>
|
||||
<!--
|
||||
Notice the use of in the tags above.
|
||||
It will be replaced with the URL of the `public` folder during the build.
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<link rel="icon" href="/favicon.victorialogs.svg" />
|
||||
<link rel="apple-touch-icon" href="/favicon.victorialogs.svg" />
|
||||
<link rel="mask-icon" href="/favicon.victorialogs.svg" color="#000000">
|
||||
<link rel="icon" href="/favicon.svg" />
|
||||
<link rel="apple-touch-icon" href="/favicon.svg" />
|
||||
<link rel="mask-icon" href="/favicon.svg" color="#000000">
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
|
||||
<meta name="theme-color" content="#000000"/>
|
||||
@@ -13,7 +13,7 @@
|
||||
manifest.json provides metadata used when your web app is installed on a
|
||||
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
|
||||
-->
|
||||
<link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
|
||||
<link rel="manifest" href="/manifest.json"/>
|
||||
<!--
|
||||
Notice the use of in the tags above.
|
||||
It will be replaced with the URL of the `public` folder during the build.
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
manifest.json provides metadata used when your web app is installed on a
|
||||
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
|
||||
-->
|
||||
<link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
|
||||
<link rel="manifest" href="/manifest.json"/>
|
||||
<!--
|
||||
Notice the use of in the tags above.
|
||||
It will be replaced with the URL of the `public` folder during the build.
|
||||
|
||||
34
app/vmui/packages/vmui/package-lock.json
generated
34
app/vmui/packages/vmui/package-lock.json
generated
@@ -10,8 +10,6 @@
|
||||
"dependencies": {
|
||||
"@types/lodash.debounce": "^4.0.9",
|
||||
"@types/lodash.get": "^4.4.9",
|
||||
"@types/lodash.orderBy": "^4.6.9",
|
||||
"@types/lodash.throttle": "^4.1.9",
|
||||
"@types/qs": "^6.9.18",
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-input-mask": "^3.0.6",
|
||||
@@ -20,8 +18,6 @@
|
||||
"dayjs": "^1.11.13",
|
||||
"lodash.debounce": "^4.0.8",
|
||||
"lodash.get": "^4.4.2",
|
||||
"lodash.orderBy": "^4.6.0",
|
||||
"lodash.throttle": "^4.1.1",
|
||||
"marked": "^15.0.8",
|
||||
"marked-emoji": "^2.0.0",
|
||||
"preact": "^10.26.5",
|
||||
@@ -2195,24 +2191,6 @@
|
||||
"@types/lodash": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/lodash.orderBy": {
|
||||
"version": "4.6.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/lodash.orderby/-/lodash.orderby-4.6.9.tgz",
|
||||
"integrity": "sha512-T9o2wkIJOmxXwVTPTmwJ59W6eTi2FseiLR369fxszG649Po/xe9vqFNhf/MtnvT5jrbDiyWKxPFPZbpSVK0SVQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/lodash": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/lodash.throttle": {
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/lodash.throttle/-/lodash.throttle-4.1.9.tgz",
|
||||
"integrity": "sha512-PCPVfpfueguWZQB7pJQK890F2scYKoDUL3iM522AptHWn7d5NQmeS/LTEHIcLr5PaTzl3dK2Z0xSUHHTHwaL5g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/lodash": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "22.14.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz",
|
||||
@@ -5764,18 +5742,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.orderBy": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.orderby/-/lodash.orderby-4.6.0.tgz",
|
||||
"integrity": "sha512-T0rZxKmghOOf5YPnn8EY5iLYeWCpZq8G41FfqoVHH5QDTAFaghJRmAdLiadEDq+ztgM2q5PjA+Z1fOwGrLgmtg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.throttle": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz",
|
||||
"integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/loose-envify": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
|
||||
|
||||
@@ -7,8 +7,6 @@
|
||||
"dependencies": {
|
||||
"@types/lodash.debounce": "^4.0.9",
|
||||
"@types/lodash.get": "^4.4.9",
|
||||
"@types/lodash.orderBy": "^4.6.9",
|
||||
"@types/lodash.throttle": "^4.1.9",
|
||||
"@types/qs": "^6.9.18",
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-input-mask": "^3.0.6",
|
||||
@@ -17,8 +15,6 @@
|
||||
"dayjs": "^1.11.13",
|
||||
"lodash.debounce": "^4.0.8",
|
||||
"lodash.get": "^4.4.2",
|
||||
"lodash.orderBy": "^4.6.0",
|
||||
"lodash.throttle": "^4.1.1",
|
||||
"marked": "^15.0.8",
|
||||
"marked-emoji": "^2.0.0",
|
||||
"preact": "^10.26.5",
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
<svg width="48" height="48" fill="#e94600" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0"/>
|
||||
<path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z"/>
|
||||
<path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB |
@@ -742,23 +742,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
|
||||
|
||||
#### rate_prometheus
|
||||
|
||||
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
|
||||
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
|
||||
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
|
||||
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
|
||||
|
||||
See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
|
||||
|
||||
See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rate_over_sum
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { FC } from "preact/compat";
|
||||
import React, { FC } from "preact/compat";
|
||||
import classNames from "classnames";
|
||||
import { MouseEvent as ReactMouseEvent, ReactNode } from "react";
|
||||
import "./style.scss";
|
||||
@@ -14,7 +14,6 @@ interface ButtonProps {
|
||||
disabled?: boolean
|
||||
children?: ReactNode
|
||||
className?: string
|
||||
"data-id"?: string
|
||||
onClick?: (e: ReactMouseEvent<HTMLButtonElement>) => void
|
||||
onMouseDown?: (e: ReactMouseEvent<HTMLButtonElement>) => void
|
||||
}
|
||||
@@ -32,7 +31,6 @@ const Button: FC<ButtonProps> = ({
|
||||
disabled,
|
||||
onClick,
|
||||
onMouseDown,
|
||||
"data-id": dataId
|
||||
}) => {
|
||||
|
||||
const classesButton = classNames({
|
||||
@@ -52,7 +50,6 @@ const Button: FC<ButtonProps> = ({
|
||||
aria-label={ariaLabel}
|
||||
onClick={onClick}
|
||||
onMouseDown={onMouseDown}
|
||||
data-id={dataId}
|
||||
>
|
||||
{startIcon}{children}{endIcon}
|
||||
</button>
|
||||
|
||||
@@ -656,33 +656,3 @@ export const ScrollToTopIcon = () => (
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
|
||||
export const SortIcon = () => (
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path d="M4 3 L4 15 L1.5 15 L5.5 21 L9.5 15 L7 15 L7 3 Z"/>
|
||||
<path d="M13 21 L13 9 L10.5 9 L14.5 3 L18.5 9 L16 9 L16 21 Z"/>
|
||||
</svg>
|
||||
);
|
||||
|
||||
export const SortArrowDownIcon = () => (
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path d="M10.5 3 L10.5 15 L8 15 L12 21 L16 15 L13.5 15 L13.5 3 Z"/>
|
||||
</svg>
|
||||
);
|
||||
|
||||
export const SortArrowUpIcon = () => (
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path d="M10.5 21 L10.5 9 L8 9 L12 3 L16 9 L13.5 9 L13.5 21 Z"/>
|
||||
</svg>
|
||||
);
|
||||
|
||||
|
||||
|
||||
@@ -18,8 +18,8 @@ const title = "Table settings";
|
||||
interface TableSettingsProps {
|
||||
columns: string[];
|
||||
selectedColumns?: string[];
|
||||
tableCompact?: boolean;
|
||||
toggleTableCompact?: () => void;
|
||||
tableCompact: boolean;
|
||||
toggleTableCompact: () => void;
|
||||
onChangeColumns: (arr: string[]) => void
|
||||
}
|
||||
|
||||
@@ -195,20 +195,18 @@ const TableSettings: FC<TableSettingsProps> = ({
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{toggleTableCompact && tableCompact !== undefined && (
|
||||
<div className="vm-table-settings-modal-section">
|
||||
<div className="vm-table-settings-modal-section__title">
|
||||
<div className="vm-table-settings-modal-section">
|
||||
<div className="vm-table-settings-modal-section__title">
|
||||
Table view
|
||||
</div>
|
||||
<div className="vm-table-settings-modal-columns-list__item">
|
||||
<Switch
|
||||
label={"Compact view"}
|
||||
value={tableCompact}
|
||||
onChange={toggleTableCompact}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className="vm-table-settings-modal-columns-list__item">
|
||||
<Switch
|
||||
label={"Compact view"}
|
||||
value={tableCompact}
|
||||
onChange={toggleTableCompact}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>)}
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
&-header {
|
||||
background-color: $color-background-block;
|
||||
z-index: 3;
|
||||
z-index: 1;
|
||||
margin: -$padding-medium 0-$padding-medium 0;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
|
||||
@@ -1,67 +1,21 @@
|
||||
import React, { FC, useMemo, useCallback, createPortal } from "preact/compat";
|
||||
import React, { FC } from "preact/compat";
|
||||
import DownloadLogsButton from "../../../DownloadLogsButton/DownloadLogsButton";
|
||||
import { createPortal } from "preact/compat";
|
||||
import JsonViewComponent from "../../../../../components/Views/JsonView/JsonView";
|
||||
import { ViewProps } from "../../types";
|
||||
import EmptyLogs from "../components/EmptyLogs/EmptyLogs";
|
||||
import JsonViewSettings from "./JsonViewSettings/JsonViewSettings";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import orderBy from "lodash.orderBy";
|
||||
import "./style.scss";
|
||||
import { Logs } from "../../../../../api/types";
|
||||
import { SortDirection } from "./types";
|
||||
import { useCallback } from "react";
|
||||
|
||||
const MemoizedJsonView = React.memo(JsonViewComponent);
|
||||
|
||||
const jsonQuerySortParam = "json_sort";
|
||||
const fieldSortQueryParamName = "json_field_sort";
|
||||
|
||||
const JsonView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
const getLogs = useCallback(() => data, [data]);
|
||||
|
||||
const [searchParams] = useSearchParams();
|
||||
const sortParam = searchParams.get(jsonQuerySortParam);
|
||||
const fieldSortParam = searchParams.get(fieldSortQueryParamName) as SortDirection;
|
||||
|
||||
const [sortField, sortDirection] = useMemo(() => {
|
||||
const [sortField, sortDirection] = sortParam?.split(":").map(decodeURIComponent) || [];
|
||||
return [sortField, sortDirection as "asc" | "desc" | undefined];
|
||||
}, [sortParam]);
|
||||
|
||||
const fields = useMemo(() => {
|
||||
const keys = new Set(data.flatMap(Object.keys));
|
||||
return Array.from(keys);
|
||||
}, [data]);
|
||||
|
||||
const orderedFieldsData = useMemo(() => {
|
||||
if (!fieldSortParam) return data;
|
||||
const orderedFields = fields.toSorted((a, b) => fieldSortParam === "asc" ? a.localeCompare(b): b.localeCompare(a));
|
||||
return data.map((item) => {
|
||||
return orderedFields.reduce((acc, field) => {
|
||||
if (item[field]) acc[field] = item[field];
|
||||
return acc;
|
||||
}, {} as Logs);
|
||||
});
|
||||
}, [fields, fieldSortParam, data]);
|
||||
|
||||
const sortedData = useMemo(() => {
|
||||
if (!sortField || !sortDirection) return orderedFieldsData;
|
||||
return orderBy(orderedFieldsData, [sortField], [sortDirection]);
|
||||
}, [orderedFieldsData, sortField, sortDirection]);
|
||||
|
||||
const renderSettings = () => {
|
||||
if (!settingsRef.current) return null;
|
||||
|
||||
return createPortal(
|
||||
data.length > 0 && (
|
||||
<div className="vm-json-view__settings-container">
|
||||
<DownloadLogsButton getLogs={getLogs} />
|
||||
<JsonViewSettings
|
||||
fields={fields}
|
||||
sortQueryParamName={jsonQuerySortParam}
|
||||
fieldSortQueryParamName={fieldSortQueryParamName}
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
data.length > 0 && <DownloadLogsButton getLogs={getLogs} />,
|
||||
settingsRef.current
|
||||
);
|
||||
};
|
||||
@@ -71,11 +25,9 @@ const JsonView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
return (
|
||||
<>
|
||||
{renderSettings()}
|
||||
<MemoizedJsonView
|
||||
data={sortedData}
|
||||
/>
|
||||
<MemoizedJsonView data={data} />
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default JsonView;
|
||||
export default JsonView;
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
import { FC, useMemo, useRef } from "preact/compat";
|
||||
import Button from "../../../../../../components/Main/Button/Button";
|
||||
import { SettingsIcon, SortArrowDownIcon, SortArrowUpIcon, SortIcon } from "../../../../../../components/Main/Icons";
|
||||
import Tooltip from "../../../../../../components/Main/Tooltip/Tooltip";
|
||||
import Select from "../../../../../../components/Main/Select/Select";
|
||||
import useBoolean from "../../../../../../hooks/useBoolean";
|
||||
import { useState, useEffect, useCallback } from "react";
|
||||
import Modal from "../../../../../../components/Main/Modal/Modal";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import "./style.scss";
|
||||
import { SortDirection } from "../types";
|
||||
|
||||
const title = "JSON settings";
|
||||
const directionList = ["asc", "desc"];
|
||||
|
||||
interface JsonSettingsProps {
|
||||
fields: string[];
|
||||
sortQueryParamName: string;
|
||||
fieldSortQueryParamName: string;
|
||||
}
|
||||
|
||||
const JsonViewSettings: FC<JsonSettingsProps> = ({
|
||||
fields,
|
||||
sortQueryParamName,
|
||||
fieldSortQueryParamName
|
||||
}) => {
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const buttonRef = useRef<HTMLDivElement>(null);
|
||||
const [fieldSortDirection, setFieldSortDirection] = useState<SortDirection>(null);
|
||||
|
||||
const {
|
||||
value: openSettings,
|
||||
toggle: toggleOpenSettings,
|
||||
setFalse: handleClose,
|
||||
} = useBoolean(false);
|
||||
|
||||
const [sortField, setSortField] = useState<string | null>(null);
|
||||
const [sortDirection, setSortDirection] = useState<SortDirection>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const sortParam = searchParams.get(sortQueryParamName);
|
||||
const isSortDirection = (value: string) : value is Exclude<SortDirection, null> => directionList.includes(value);
|
||||
if (sortParam) {
|
||||
const [field, direction] = sortParam.split(":").map(decodeURIComponent);
|
||||
if (field && (isSortDirection(direction))) {
|
||||
setSortField(field);
|
||||
setSortDirection(direction);
|
||||
}
|
||||
}
|
||||
|
||||
const fieldSortParam = searchParams.get(fieldSortQueryParamName);
|
||||
if (fieldSortParam === "asc" || fieldSortParam === "desc") {
|
||||
setFieldSortDirection(fieldSortParam);
|
||||
}
|
||||
}, [searchParams, sortQueryParamName, fieldSortQueryParamName, setSortField, setSortDirection, setFieldSortDirection]);
|
||||
|
||||
const updateSortParams = useCallback((field: string | null, direction: SortDirection) => {
|
||||
const updatedParams = new URLSearchParams(searchParams.toString());
|
||||
|
||||
if (!field || !direction) {
|
||||
updatedParams.delete(sortQueryParamName);
|
||||
} else {
|
||||
updatedParams.set(sortQueryParamName, `${field}:${direction || ""}`);
|
||||
}
|
||||
|
||||
setSearchParams(updatedParams);
|
||||
}, [searchParams, sortQueryParamName]);
|
||||
|
||||
const handleSort = (field: string) => {
|
||||
const newDirection: SortDirection = sortDirection || "asc";
|
||||
setSortField(field);
|
||||
setSortDirection(newDirection);
|
||||
updateSortParams(field, newDirection);
|
||||
};
|
||||
|
||||
const resetSort = () => {
|
||||
setSortField(null);
|
||||
setSortDirection(null);
|
||||
updateSortParams(null, null);
|
||||
};
|
||||
|
||||
const changeFieldSortDirection = useCallback(() => {
|
||||
let newFieldSortDirection: SortDirection = null;
|
||||
if (fieldSortDirection === null) {
|
||||
newFieldSortDirection = "asc";
|
||||
}else if (fieldSortDirection === "asc") {
|
||||
newFieldSortDirection = "desc";
|
||||
}
|
||||
setFieldSortDirection(newFieldSortDirection);
|
||||
const updatedParams = new URLSearchParams(searchParams.toString());
|
||||
|
||||
if (!newFieldSortDirection) {
|
||||
updatedParams.delete(fieldSortQueryParamName);
|
||||
} else {
|
||||
updatedParams.set(fieldSortQueryParamName, encodeURIComponent(newFieldSortDirection));
|
||||
}
|
||||
|
||||
setSearchParams(updatedParams);
|
||||
},[fieldSortDirection, searchParams, fieldSortQueryParamName]);
|
||||
|
||||
const handleChangeSortDirection = (direction: string) => {
|
||||
const field = sortField || fields[0];
|
||||
setSortField(field);
|
||||
setSortDirection(direction as SortDirection);
|
||||
updateSortParams(field, direction as SortDirection);
|
||||
};
|
||||
|
||||
const fieldSortMeta = useMemo(() => ({
|
||||
default: {
|
||||
title: "Set field sort order. Click to sort in ascending order",
|
||||
icon: <SortIcon />
|
||||
},
|
||||
asc: {
|
||||
title: "Fields sorted ascending. Click to sort in descending order",
|
||||
icon: <SortArrowDownIcon />
|
||||
},
|
||||
desc: {
|
||||
title: "Fields sorted descending. Click to reset sort",
|
||||
icon: <SortArrowUpIcon />
|
||||
},
|
||||
}), []);
|
||||
|
||||
const fieldSortButton = useMemo(() => {
|
||||
const { title, icon } = fieldSortMeta[fieldSortDirection ?? "default"];
|
||||
return <Tooltip title={title}>
|
||||
<Button
|
||||
variant="text"
|
||||
startIcon={icon}
|
||||
onClick={changeFieldSortDirection}
|
||||
ariaLabel={title}
|
||||
/>
|
||||
</Tooltip>;
|
||||
}, [fieldSortDirection, toggleOpenSettings, changeFieldSortDirection, fieldSortMeta]);
|
||||
|
||||
|
||||
return (
|
||||
<div className="vm-json-settings">
|
||||
{fieldSortButton}
|
||||
<Tooltip title={title}>
|
||||
<div ref={buttonRef}>
|
||||
<Button
|
||||
variant="text"
|
||||
startIcon={<SettingsIcon/>}
|
||||
onClick={toggleOpenSettings}
|
||||
ariaLabel={title}
|
||||
/>
|
||||
</div>
|
||||
</Tooltip>
|
||||
{openSettings && (
|
||||
<Modal
|
||||
title={title}
|
||||
className="vm-json-settings-modal"
|
||||
onClose={handleClose}
|
||||
>
|
||||
<div className="vm-json-settings-modal-section">
|
||||
<div className="vm-json-settings-modal-section__sort-settings-container">
|
||||
<Select
|
||||
value={sortField || ""}
|
||||
onChange={handleSort}
|
||||
list={fields}
|
||||
label="Select field"
|
||||
/>
|
||||
<Select
|
||||
value={sortDirection || ""}
|
||||
onChange={handleChangeSortDirection}
|
||||
list={directionList}
|
||||
label="Sort direction"
|
||||
/>
|
||||
{(sortField || sortDirection) && (
|
||||
<Button
|
||||
variant="outlined"
|
||||
color="error"
|
||||
onClick={resetSort}
|
||||
>
|
||||
Reset sort
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</Modal>)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default JsonViewSettings;
|
||||
@@ -1,34 +0,0 @@
|
||||
@use "src/styles/variables" as *;
|
||||
|
||||
.vm-json-settings {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
|
||||
&-modal {
|
||||
.vm-modal-content-body {
|
||||
min-width: clamp(300px, 600px, 90vw);
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
&-section {
|
||||
padding-block: $padding-global;
|
||||
border-top: $border-divider;
|
||||
|
||||
&:first-child {
|
||||
padding-top: 0;
|
||||
border-top: none;
|
||||
}
|
||||
|
||||
&__sort-settings-container {
|
||||
display: grid;
|
||||
padding: $padding-medium;
|
||||
grid-template-columns: 1fr 1fr 80px;
|
||||
gap: $padding-medium;
|
||||
|
||||
@media (max-width: 500px) {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
@use "src/styles/variables" as *;
|
||||
|
||||
.vm-json-view {
|
||||
&__settings-container {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
export type SortDirection = "asc" | "desc" | null;
|
||||
@@ -7,7 +7,7 @@ import { useLiveTailingLogs } from "./useLiveTailingLogs";
|
||||
import { LOGS_DISPLAY_FIELDS, LOGS_URL_PARAMS } from "../../../../../constants/logs";
|
||||
import { useMemo } from "react";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import throttle from "lodash.throttle";
|
||||
import throttle from "lodash/throttle";
|
||||
import GroupLogsItem from "../../../GroupLogs/GroupLogsItem";
|
||||
import LiveTailingSettings from "./LiveTailingSettings";
|
||||
import Alert from "../../../../../components/Main/Alert/Alert";
|
||||
|
||||
@@ -3,6 +3,7 @@ import DownloadLogsButton from "../../../DownloadLogsButton/DownloadLogsButton";
|
||||
import { createPortal } from "preact/compat";
|
||||
import "./style.scss";
|
||||
import { ViewProps } from "../../types";
|
||||
import useBoolean from "../../../../../hooks/useBoolean";
|
||||
import useStateSearchParams from "../../../../../hooks/useStateSearchParams";
|
||||
import TableLogs from "../../TableLogs";
|
||||
import SelectLimit from "../../../../../components/Main/Pagination/SelectLimit/SelectLimit";
|
||||
@@ -17,6 +18,7 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
const { setSearchParamsFromKeys } = useSearchParamsFromObject();
|
||||
const [displayColumns, setDisplayColumns] = useState<string[]>([]);
|
||||
const [rowsPerPage, setRowsPerPage] = useStateSearchParams(100, "rows_per_page");
|
||||
const { value: tableCompact, toggle: toggleTableCompact } = useBoolean(false);
|
||||
|
||||
const columns = useMemo(() => {
|
||||
const keys = new Set<string>();
|
||||
@@ -50,6 +52,8 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
columns={columns}
|
||||
selectedColumns={displayColumns}
|
||||
onChangeColumns={setDisplayColumns}
|
||||
tableCompact={tableCompact}
|
||||
toggleTableCompact={toggleTableCompact}
|
||||
/>
|
||||
</div>
|
||||
</div>,
|
||||
@@ -65,7 +69,7 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
<MemoizedTableView
|
||||
logs={data}
|
||||
displayColumns={displayColumns}
|
||||
tableCompact={false}
|
||||
tableCompact={tableCompact}
|
||||
columns={columns}
|
||||
rowsPerPage={Number(rowsPerPage)}
|
||||
/>
|
||||
|
||||
@@ -18,22 +18,22 @@ import (
|
||||
pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
)
|
||||
|
||||
// PrometheusQuerier contains methods available to Prometheus-like HTTP API for Querying
|
||||
type PrometheusQuerier interface {
|
||||
PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
|
||||
PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
|
||||
PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
|
||||
PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse
|
||||
PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
|
||||
// APIQuerier contains methods available to Prometheus-like HTTP API for Querying
|
||||
type APIQuerier interface {
|
||||
APIV1Export(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse
|
||||
APIV1Query(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse
|
||||
APIV1QueryRange(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse
|
||||
APIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *APIV1SeriesResponse
|
||||
APIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
|
||||
}
|
||||
|
||||
// Writer contains methods for writing new data
|
||||
type Writer interface {
|
||||
// Prometheus APIs
|
||||
PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts)
|
||||
PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
|
||||
PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts)
|
||||
PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts)
|
||||
// APIWriter contains methods for writing new data
|
||||
type APIWriter interface {
|
||||
// Prometheus-like APIs
|
||||
APIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts)
|
||||
APIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
|
||||
APIV1ImportCSV(t *testing.T, records []string, opts QueryOpts)
|
||||
APIV1ImportNative(t *testing.T, data []byte, opts QueryOpts)
|
||||
|
||||
// Graphit APIs
|
||||
GraphiteWrite(t *testing.T, records []string, opts QueryOpts)
|
||||
@@ -54,11 +54,11 @@ type StorageMerger interface {
|
||||
ForceMerge(t *testing.T)
|
||||
}
|
||||
|
||||
// PrometheusWriteQuerier encompasses the methods for writing, flushing and
|
||||
// WriteQuerier encompasses the methods for writing, flushing and
|
||||
// querying the data.
|
||||
type PrometheusWriteQuerier interface {
|
||||
Writer
|
||||
PrometheusQuerier
|
||||
type WriteQuerier interface {
|
||||
APIWriter
|
||||
APIQuerier
|
||||
StorageFlusher
|
||||
StorageMerger
|
||||
}
|
||||
@@ -138,9 +138,9 @@ func (qos *QueryOptsLogs) asURLValues() url.Values {
|
||||
return uv
|
||||
}
|
||||
|
||||
// PrometheusAPIV1QueryResponse is an inmemory representation of the
|
||||
// APIV1QueryResponse is an inmemory representation of the
|
||||
// /prometheus/api/v1/query or /prometheus/api/v1/query_range response.
|
||||
type PrometheusAPIV1QueryResponse struct {
|
||||
type APIV1QueryResponse struct {
|
||||
Status string
|
||||
Data *QueryData
|
||||
ErrorType string
|
||||
@@ -148,12 +148,12 @@ type PrometheusAPIV1QueryResponse struct {
|
||||
IsPartial bool
|
||||
}
|
||||
|
||||
// NewPrometheusAPIV1QueryResponse is a test helper function that creates a new
|
||||
// instance of PrometheusAPIV1QueryResponse by unmarshalling a json string.
|
||||
func NewPrometheusAPIV1QueryResponse(t *testing.T, s string) *PrometheusAPIV1QueryResponse {
|
||||
// NewAPIV1QueryResponse is a test helper function that creates a new
|
||||
// instance of APIV1QueryResponse by unmarshalling a json string.
|
||||
func NewAPIV1QueryResponse(t *testing.T, s string) *APIV1QueryResponse {
|
||||
t.Helper()
|
||||
|
||||
res := &PrometheusAPIV1QueryResponse{}
|
||||
res := &APIV1QueryResponse{}
|
||||
if err := json.Unmarshal([]byte(s), res); err != nil {
|
||||
t.Fatalf("could not unmarshal query response data=\n%s\n: %v", string(s), err)
|
||||
}
|
||||
@@ -161,7 +161,7 @@ func NewPrometheusAPIV1QueryResponse(t *testing.T, s string) *PrometheusAPIV1Que
|
||||
}
|
||||
|
||||
// Sort performs data.Result sort by metric labels
|
||||
func (pqr *PrometheusAPIV1QueryResponse) Sort() {
|
||||
func (pqr *APIV1QueryResponse) Sort() {
|
||||
if pqr.Data == nil {
|
||||
return
|
||||
}
|
||||
@@ -257,9 +257,9 @@ func (s *Sample) UnmarshalJSON(b []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// PrometheusAPIV1SeriesResponse is an inmemory representation of the
|
||||
// APIV1SeriesResponse is an inmemory representation of the
|
||||
// /prometheus/api/v1/series response.
|
||||
type PrometheusAPIV1SeriesResponse struct {
|
||||
type APIV1SeriesResponse struct {
|
||||
Status string
|
||||
IsPartial bool
|
||||
Data []map[string]string
|
||||
@@ -268,12 +268,12 @@ type PrometheusAPIV1SeriesResponse struct {
|
||||
Error string
|
||||
}
|
||||
|
||||
// NewPrometheusAPIV1SeriesResponse is a test helper function that creates a new
|
||||
// instance of PrometheusAPIV1SeriesResponse by unmarshalling a json string.
|
||||
func NewPrometheusAPIV1SeriesResponse(t *testing.T, s string) *PrometheusAPIV1SeriesResponse {
|
||||
// NewAPIV1SeriesResponse is a test helper function that creates a new
|
||||
// instance of APIV1SeriesResponse by unmarshalling a json string.
|
||||
func NewAPIV1SeriesResponse(t *testing.T, s string) *APIV1SeriesResponse {
|
||||
t.Helper()
|
||||
|
||||
res := &PrometheusAPIV1SeriesResponse{}
|
||||
res := &APIV1SeriesResponse{}
|
||||
if err := json.Unmarshal([]byte(s), res); err != nil {
|
||||
t.Fatalf("could not unmarshal series response data:\n%s\n err: %v", string(s), err)
|
||||
}
|
||||
@@ -281,7 +281,7 @@ func NewPrometheusAPIV1SeriesResponse(t *testing.T, s string) *PrometheusAPIV1Se
|
||||
}
|
||||
|
||||
// Sort sorts the response data.
|
||||
func (r *PrometheusAPIV1SeriesResponse) Sort() *PrometheusAPIV1SeriesResponse {
|
||||
func (r *APIV1SeriesResponse) Sort() *APIV1SeriesResponse {
|
||||
str := func(m map[string]string) string {
|
||||
s := []string{}
|
||||
for k, v := range m {
|
||||
|
||||
@@ -146,7 +146,7 @@ func (tc *TestCase) MustStartVmagent(instance string, flags []string, promScrape
|
||||
// Vmcluster represents a typical cluster setup: several vmstorage replicas, one
|
||||
// vminsert, and one vmselect.
|
||||
//
|
||||
// Both Vmsingle and Vmcluster implement the PrometheusWriteQuerier used in
|
||||
// Both Vmsingle and Vmcluster implement the WriteQuerier used in
|
||||
// business logic tests to abstract out the infrasture.
|
||||
//
|
||||
// This type is not suitable for infrastructure tests where custom cluster
|
||||
@@ -309,8 +309,8 @@ func (tc *TestCase) StopApp(instance string) {
|
||||
}
|
||||
}
|
||||
|
||||
// StopPrometheusWriteQuerier stop all apps that are a part of the pwq.
|
||||
func (tc *TestCase) StopPrometheusWriteQuerier(pwq PrometheusWriteQuerier) {
|
||||
// StopWriteQuerier stop all apps that are a part of the pwq.
|
||||
func (tc *TestCase) StopWriteQuerier(pwq WriteQuerier) {
|
||||
tc.t.Helper()
|
||||
switch t := pwq.(type) {
|
||||
case *Vmsingle:
|
||||
@@ -433,24 +433,3 @@ func (tc *TestCase) MustStartVlsingle(instance string, flags []string) *Vlsingle
|
||||
tc.addApp(instance, app)
|
||||
return app
|
||||
}
|
||||
|
||||
// MustStartDefaultVlagent is a test helper function that starts an instance of
|
||||
// vlagent with defaults suitable for most tests.
|
||||
func (tc *TestCase) MustStartDefaultVlagent(remoteWriteURLs []string) *Vlagent {
|
||||
tc.t.Helper()
|
||||
|
||||
return tc.MustStartVlagent("vlagent", remoteWriteURLs, nil)
|
||||
}
|
||||
|
||||
// MustStartVlagent is a test helper function that starts an instance of
|
||||
// vlagent and fails the test if the app fails to start.
|
||||
func (tc *TestCase) MustStartVlagent(instance string, remoteWriteURLs []string, flags []string) *Vlagent {
|
||||
tc.t.Helper()
|
||||
|
||||
app, err := StartVlagent(instance, remoteWriteURLs, flags, tc.cli)
|
||||
if err != nil {
|
||||
tc.t.Fatalf("Could not start %s: %v", instance, err)
|
||||
}
|
||||
tc.addApp(instance, app)
|
||||
return app
|
||||
}
|
||||
|
||||
@@ -11,10 +11,10 @@ import (
|
||||
)
|
||||
|
||||
type testBackupRestoreOpts struct {
|
||||
startSUT func() at.PrometheusWriteQuerier
|
||||
startSUT func() at.WriteQuerier
|
||||
stopSUT func()
|
||||
storageDataPaths []string
|
||||
snapshotCreateURLs func(at.PrometheusWriteQuerier) []string
|
||||
snapshotCreateURLs func(at.WriteQuerier) []string
|
||||
}
|
||||
|
||||
func TestSingleBackupRestore(t *testing.T) {
|
||||
@@ -24,7 +24,7 @@ func TestSingleBackupRestore(t *testing.T) {
|
||||
storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
|
||||
|
||||
opts := testBackupRestoreOpts{
|
||||
startSUT: func() at.PrometheusWriteQuerier {
|
||||
startSUT: func() at.WriteQuerier {
|
||||
return tc.MustStartVmsingle("vmsingle", []string{
|
||||
"-storageDataPath=" + storageDataPath,
|
||||
"-retentionPeriod=100y",
|
||||
@@ -37,7 +37,7 @@ func TestSingleBackupRestore(t *testing.T) {
|
||||
storageDataPaths: []string{
|
||||
storageDataPath,
|
||||
},
|
||||
snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
|
||||
snapshotCreateURLs: func(sut at.WriteQuerier) []string {
|
||||
return []string{
|
||||
sut.(*at.Vmsingle).SnapshotCreateURL(),
|
||||
}
|
||||
@@ -55,7 +55,7 @@ func TestClusterBackupRestore(t *testing.T) {
|
||||
storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
|
||||
|
||||
opts := testBackupRestoreOpts{
|
||||
startSUT: func() at.PrometheusWriteQuerier {
|
||||
startSUT: func() at.WriteQuerier {
|
||||
return tc.MustStartCluster(&at.ClusterOptions{
|
||||
Vmstorage1Instance: "vmstorage1",
|
||||
Vmstorage1Flags: []string{
|
||||
@@ -85,7 +85,7 @@ func TestClusterBackupRestore(t *testing.T) {
|
||||
storage1DataPath,
|
||||
storage2DataPath,
|
||||
},
|
||||
snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
|
||||
snapshotCreateURLs: func(sut at.WriteQuerier) []string {
|
||||
c := sut.(*at.Vmcluster)
|
||||
return []string{
|
||||
c.Vmstorages[0].SnapshotCreateURL(),
|
||||
@@ -127,18 +127,18 @@ func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
|
||||
|
||||
// assertSeries retrieves set of all metric names from the storage and
|
||||
// compares it with the expected set.
|
||||
assertSeries := func(app at.PrometheusQuerier, query string, start, end int64, want []map[string]string) {
|
||||
assertSeries := func(app at.APIQuerier, query string, start, end int64, want []map[string]string) {
|
||||
t.Helper()
|
||||
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
|
||||
return app.APIV1Series(t, query, at.QueryOpts{
|
||||
Start: fmt.Sprintf("%d", start),
|
||||
End: fmt.Sprintf("%d", end),
|
||||
}).Sort()
|
||||
},
|
||||
Want: &at.PrometheusAPIV1SeriesResponse{
|
||||
Want: &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
Data: want,
|
||||
},
|
||||
@@ -148,18 +148,18 @@ func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
|
||||
|
||||
// assertSeries retrieves all data from the storage and compares it with the
|
||||
// expected result.
|
||||
assertQueryResults := func(app at.PrometheusQuerier, query string, start, end int64, want []*at.QueryResult) {
|
||||
assertQueryResults := func(app at.APIQuerier, query string, start, end int64, want []*at.QueryResult) {
|
||||
t.Helper()
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
|
||||
return app.APIV1QueryRange(t, query, at.QueryOpts{
|
||||
Start: fmt.Sprintf("%d", start),
|
||||
End: fmt.Sprintf("%d", end),
|
||||
Step: "60s",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -171,7 +171,7 @@ func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
|
||||
})
|
||||
}
|
||||
|
||||
createBackup := func(sut at.PrometheusWriteQuerier, name string) {
|
||||
createBackup := func(sut at.WriteQuerier, name string) {
|
||||
for i, storageDataPath := range opts.storageDataPaths {
|
||||
replica := fmt.Sprintf("replica-%d", i)
|
||||
instance := fmt.Sprintf("vmbackup-%s-%s", name, replica)
|
||||
@@ -216,13 +216,13 @@ func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
|
||||
|
||||
sut := opts.startSUT()
|
||||
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, batch1Data, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, batch1Data, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
|
||||
assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
|
||||
createBackup(sut, "batch1")
|
||||
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, batch2Data, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, batch2Data, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
assertSeries(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12Series)
|
||||
assertQueryResults(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12QueryResults)
|
||||
|
||||
@@ -5,12 +5,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
||||
pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
)
|
||||
|
||||
func TestSingleDeduplication_dedulicationIsOff(t *testing.T) {
|
||||
@@ -80,7 +81,7 @@ func TestClusterDeduplication_deduplicationIsOn(t *testing.T) {
|
||||
}
|
||||
|
||||
// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication
|
||||
func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplicationIsOn bool) {
|
||||
func testDeduplication(tc *at.TestCase, sut at.WriteQuerier, deduplicationIsOn bool) {
|
||||
t := tc.T()
|
||||
|
||||
firstDayOfThisMonth := func() time.Time {
|
||||
@@ -134,11 +135,11 @@ func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplica
|
||||
},
|
||||
}
|
||||
|
||||
sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
|
||||
sut.APIV1Write(t, data, apptest.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
sut.ForceMerge(t)
|
||||
|
||||
wantDuplicates := &at.PrometheusAPIV1QueryResponse{
|
||||
wantDuplicates := &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -166,7 +167,7 @@ func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplica
|
||||
},
|
||||
},
|
||||
}
|
||||
wantDeduped := &at.PrometheusAPIV1QueryResponse{
|
||||
wantDeduped := &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -207,7 +208,7 @@ func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplica
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected response",
|
||||
Got: func() any {
|
||||
got := sut.PrometheusAPIV1Export(t, `{__name__=~"metric.*"}`, apptest.QueryOpts{
|
||||
got := sut.APIV1Export(t, `{__name__=~"metric.*"}`, apptest.QueryOpts{
|
||||
ReduceMemUsage: "1",
|
||||
Start: fmt.Sprintf("%d", start.UnixMilli()),
|
||||
End: fmt.Sprintf("%d", end.UnixMilli()),
|
||||
|
||||
@@ -33,9 +33,9 @@ func TestClusterExportImportNative(t *testing.T) {
|
||||
|
||||
// testExportImportNative test export and import in VictoriaMetrics’ native format.
|
||||
// see: https://docs.victoriametrics.com/#how-to-import-data-in-native-format
|
||||
func testExportImportNative(t *testing.T, sut at.PrometheusWriteQuerier) {
|
||||
func testExportImportNative(t *testing.T, sut at.WriteQuerier) {
|
||||
// create test data
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`native_export_import 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
}, at.QueryOpts{
|
||||
ExtraLabels: []string{"el1=elv1", "el2=elv2"},
|
||||
@@ -43,27 +43,27 @@ func testExportImportNative(t *testing.T, sut at.PrometheusWriteQuerier) {
|
||||
sut.ForceFlush(t)
|
||||
|
||||
// export test data via native export API
|
||||
exportResult := sut.PrometheusAPIV1ExportNative(t, "native_export_import", at.QueryOpts{
|
||||
exportResult := sut.APIV1ExportNative(t, "native_export_import", at.QueryOpts{
|
||||
Start: "2024-02-05T08:50:00.700Z",
|
||||
End: "2024-02-05T09:00:00.700Z",
|
||||
})
|
||||
|
||||
// re-import test data via native import API
|
||||
sut.PrometheusAPIV1ImportNative(t, exportResult, at.QueryOpts{})
|
||||
sut.APIV1ImportNative(t, exportResult, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
// check query result
|
||||
got := sut.PrometheusAPIV1QueryRange(t, "native_export_import", at.QueryOpts{
|
||||
got := sut.APIV1QueryRange(t, "native_export_import", at.QueryOpts{
|
||||
Start: "2024-02-05T08:57:36.700Z",
|
||||
End: "2024-02-05T08:57:36.700Z",
|
||||
Step: "60s",
|
||||
})
|
||||
|
||||
cmpOptions := []cmp.Option{
|
||||
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.EquateNaNs(),
|
||||
}
|
||||
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "native_export_import", "el1": "elv1", "el2":"elv2"}, "values": []}]}}`)
|
||||
want := at.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "native_export_import", "el1": "elv1", "el2":"elv2"}, "values": []}]}}`)
|
||||
want.Data.Result[0].Samples = []*at.Sample{
|
||||
at.NewSample(t, "2024-02-05T08:57:36.700Z", 10),
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
|
||||
wantMetrics []map[string]string
|
||||
wantSamples []*at.Sample
|
||||
}
|
||||
f := func(sut at.PrometheusQuerier, opts *opts) {
|
||||
f := func(sut at.APIQuerier, opts *opts) {
|
||||
t.Helper()
|
||||
wantResult := []*at.QueryResult{}
|
||||
for idx, wm := range opts.wantMetrics {
|
||||
@@ -35,16 +35,16 @@ func TestSingleIngestionProtocols(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /export query response",
|
||||
Got: func() any {
|
||||
got := sut.PrometheusAPIV1Export(t, opts.query, at.QueryOpts{
|
||||
got := sut.APIV1Export(t, opts.query, at.QueryOpts{
|
||||
Start: "2024-02-05T08:50:00.700Z",
|
||||
End: "2024-02-05T09:00:00.700Z",
|
||||
})
|
||||
got.Sort()
|
||||
return got
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
|
||||
Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -109,7 +109,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
|
||||
})
|
||||
|
||||
// CSV import
|
||||
sut.PrometheusAPIV1ImportCSV(t, []string{
|
||||
sut.APIV1ImportCSV(t, []string{
|
||||
`GOOG,1.23,4.56,NYSE,1707123457`,
|
||||
`MSFT,23,56,NASDAQ,1707123457`,
|
||||
}, at.QueryOpts{
|
||||
@@ -158,7 +158,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
|
||||
})
|
||||
|
||||
// prometheus text exposition format
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`importprometheus_series 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
`importprometheus_series2{label="foo",label1="value1"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
|
||||
}, at.QueryOpts{
|
||||
@@ -227,7 +227,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
sut.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
|
||||
sut.APIV1Write(t, pbData, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
f(sut, &opts{
|
||||
query: `{__name__=~"prometheusrw.+"}`,
|
||||
@@ -282,22 +282,22 @@ func TestClusterIngestionProtocols(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /export query response",
|
||||
Got: func() any {
|
||||
got := vmselect.PrometheusAPIV1Export(t, opts.query, at.QueryOpts{
|
||||
got := vmselect.APIV1Export(t, opts.query, at.QueryOpts{
|
||||
Start: "2024-02-05T08:50:00.700Z",
|
||||
End: "2024-02-05T09:00:00.700Z",
|
||||
})
|
||||
got.Sort()
|
||||
return got
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
|
||||
Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// prometheus text exposition format
|
||||
vminsert.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
vminsert.APIV1ImportPrometheus(t, []string{
|
||||
`importprometheus_series 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
`importprometheus_series2{label="foo",label1="value1"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
|
||||
}, at.QueryOpts{
|
||||
@@ -358,7 +358,7 @@ func TestClusterIngestionProtocols(t *testing.T) {
|
||||
})
|
||||
|
||||
// CSV import
|
||||
vminsert.PrometheusAPIV1ImportCSV(t, []string{
|
||||
vminsert.APIV1ImportCSV(t, []string{
|
||||
`GOOG,1.23,4.56,NYSE,1707123457`, // 2024-02-05T08:57:37.000Z
|
||||
`MSFT,23,56,NASDAQ,1707123457`, // 2024-02-05T08:57:37.000Z
|
||||
}, at.QueryOpts{
|
||||
@@ -474,7 +474,7 @@ func TestClusterIngestionProtocols(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
vminsert.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
|
||||
vminsert.APIV1Write(t, pbData, at.QueryOpts{})
|
||||
vmstorage.ForceFlush(t)
|
||||
f(&opts{
|
||||
query: `{__name__=~"prometheusrw.+"}`,
|
||||
|
||||
@@ -55,10 +55,10 @@ func TestClusterKeyConceptsQueryData(t *testing.T) {
|
||||
}
|
||||
|
||||
// testKeyConceptsQueryData verifies cases from https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data
|
||||
func testKeyConceptsQueryData(t *testing.T, sut at.PrometheusWriteQuerier) {
|
||||
func testKeyConceptsQueryData(t *testing.T, sut at.WriteQuerier) {
|
||||
|
||||
// Insert example data from documentation.
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, docData, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, docData, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
testInstantQuery(t, sut)
|
||||
@@ -69,14 +69,14 @@ func testKeyConceptsQueryData(t *testing.T, sut at.PrometheusWriteQuerier) {
|
||||
// testInstantQuery verifies the statements made in the `Instant query` section
|
||||
// of the VictoriaMetrics documentation. See:
|
||||
// https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query
|
||||
func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
|
||||
func testInstantQuery(t *testing.T, q at.APIQuerier) {
|
||||
// Get the value of the foo_bar time series at 2022-05-10T08:03:00Z with the
|
||||
// step of 5m and timeout 5s. There is no sample at exactly this timestamp.
|
||||
// Therefore, VictoriaMetrics will search for the nearest sample within the
|
||||
// [time-5m..time] interval.
|
||||
got := q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:03:00.000Z", Step: "5m"})
|
||||
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
|
||||
opt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
got := q.APIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:03:00.000Z", Step: "5m"})
|
||||
want := at.NewAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
|
||||
opt := cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
if diff := cmp.Diff(want, got, opt); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
|
||||
// Therefore, VictoriaMetrics will search for the nearest sample within the
|
||||
// [time-1m..time] interval. Since the nearest sample is 2m away and the
|
||||
// step is 1m, then the VictoriaMetrics must return empty response.
|
||||
got = q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:18:00.000Z", Step: "1m"})
|
||||
got = q.APIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:18:00.000Z", Step: "1m"})
|
||||
if len(got.Data.Result) > 0 {
|
||||
t.Errorf("unexpected response: got non-empty result, want empty result:\n%v", got)
|
||||
}
|
||||
@@ -95,14 +95,14 @@ func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
|
||||
// testRangeQuery verifies the statements made in the `Range query` section of
|
||||
// the VictoriaMetrics documentation. See:
|
||||
// https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query
|
||||
func testRangeQuery(t *testing.T, q at.PrometheusQuerier) {
|
||||
func testRangeQuery(t *testing.T, q at.APIQuerier) {
|
||||
f := func(start, end, step string, wantSamples []*at.Sample) {
|
||||
t.Helper()
|
||||
|
||||
got := q.PrometheusAPIV1QueryRange(t, "foo_bar", at.QueryOpts{Start: start, End: end, Step: step})
|
||||
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
|
||||
got := q.APIV1QueryRange(t, "foo_bar", at.QueryOpts{Start: start, End: end, Step: step})
|
||||
want := at.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
|
||||
want.Data.Result[0].Samples = wantSamples
|
||||
opt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
opt := cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
if diff := cmp.Diff(want, got, opt); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
@@ -168,11 +168,11 @@ func testRangeQuery(t *testing.T, q at.PrometheusQuerier) {
|
||||
// will not produce ephemeral points.
|
||||
//
|
||||
// See: https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query
|
||||
func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.PrometheusQuerier) {
|
||||
func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.APIQuerier) {
|
||||
f := func(timestamp string, want *at.Sample) {
|
||||
t.Helper()
|
||||
|
||||
gotInstant := q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: timestamp, Step: "1m"})
|
||||
gotInstant := q.APIV1Query(t, "foo_bar", at.QueryOpts{Time: timestamp, Step: "1m"})
|
||||
if want == nil {
|
||||
if got, want := len(gotInstant.Data.Result), 0; got != want {
|
||||
t.Errorf("unexpected instant result size: got %d, want %d", got, want)
|
||||
@@ -185,7 +185,7 @@ func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.Prometheu
|
||||
}
|
||||
}
|
||||
|
||||
rangeRes := q.PrometheusAPIV1QueryRange(t, "foo_bar", at.QueryOpts{
|
||||
rangeRes := q.APIV1QueryRange(t, "foo_bar", at.QueryOpts{
|
||||
Start: "2022-05-10T07:59:00.000Z",
|
||||
End: "2022-05-10T08:17:00.000Z",
|
||||
Step: "1m",
|
||||
@@ -231,7 +231,7 @@ func TestClusterMillisecondPrecisionInInstantQueries(t *testing.T) {
|
||||
testMillisecondPrecisionInInstantQueries(tc, sut)
|
||||
}
|
||||
|
||||
func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
type opts struct {
|
||||
@@ -242,7 +242,7 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
|
||||
wantSample *at.Sample
|
||||
wantSamples []*at.Sample
|
||||
}
|
||||
f := func(sut at.PrometheusQuerier, opts *opts) {
|
||||
f := func(sut at.APIQuerier, opts *opts) {
|
||||
t.Helper()
|
||||
wantResult := []*at.QueryResult{}
|
||||
if opts.wantMetric != nil && (opts.wantSample != nil || len(opts.wantSamples) > 0) {
|
||||
@@ -255,19 +255,19 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1Query(t, opts.query, at.QueryOpts{
|
||||
return sut.APIV1Query(t, opts.query, at.QueryOpts{
|
||||
Time: opts.qtime,
|
||||
Step: opts.step,
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
|
||||
Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`series1{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
`series1{label="foo"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
|
||||
}, at.QueryOpts{})
|
||||
@@ -326,7 +326,7 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
|
||||
|
||||
// Insert samples with different dates. The difference in ms between the two
|
||||
// timestamps is 4236579304.
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`series2{label="foo"} 10 1638564958042`, // 2021-12-03T20:55:58.042Z
|
||||
`series2{label="foo"} 20 1642801537346`, // 2022-01-21T21:45:37.346Z
|
||||
}, at.QueryOpts{})
|
||||
|
||||
@@ -16,7 +16,7 @@ func TestSingleMaxIngestionRateIncrementsMetric(t *testing.T) {
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
sut := tc.MustStartVmsingle("vmsingle", []string{"-maxIngestionRate=1"})
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
|
||||
if got := sut.GetMetric(t, "vm_max_ingestion_rate_limit_reached_total"); got <= 0 {
|
||||
t.Fatalf("Unexpected vm_max_ingestion_rate_limit_reached_total: got %f, want >0", got)
|
||||
}
|
||||
@@ -26,7 +26,7 @@ func TestSingleMaxIngestionRateDoesNotIncrementMetric(t *testing.T) {
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
sut := tc.MustStartVmsingle("vmsingle", []string{"-maxIngestionRate=15"})
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
|
||||
if got, want := sut.GetMetric(t, "vm_max_ingestion_rate_limit_reached_total"), 0.0; got != want {
|
||||
t.Fatalf("Unexpected vm_max_ingestion_rate_limit_reached_total: got %f, want >0", got)
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
|
||||
}
|
||||
tsdbMetricNameEntryCmpOpts := cmpopts.IgnoreFields(apptest.TSDBStatusResponseMetricNameEntry{}, "LastRequestTimestamp")
|
||||
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, dataSet, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, dataSet, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
// verify ingest request correctly registered
|
||||
@@ -55,7 +55,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
|
||||
}
|
||||
|
||||
// verify query request correctly registered
|
||||
sut.PrometheusAPIV1Query(t, `{__name__!=""}`, at.QueryOpts{Time: ingestDateTime})
|
||||
sut.APIV1Query(t, `{__name__!=""}`, at.QueryOpts{Time: ingestDateTime})
|
||||
expected = apptest.MetricNamesStatsResponse{
|
||||
Records: []at.MetricNamesStatsRecord{
|
||||
{MetricName: largeMetricName, QueryRequestsCount: 1},
|
||||
@@ -97,7 +97,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
|
||||
}
|
||||
|
||||
// perform query request for single metric and check counter increase
|
||||
sut.PrometheusAPIV1Query(t, `metric_name_2`, at.QueryOpts{Time: ingestDateTime})
|
||||
sut.APIV1Query(t, `metric_name_2`, at.QueryOpts{Time: ingestDateTime})
|
||||
expected = apptest.MetricNamesStatsResponse{
|
||||
Records: []at.MetricNamesStatsRecord{
|
||||
{MetricName: largeMetricName, QueryRequestsCount: 1},
|
||||
@@ -187,7 +187,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
|
||||
// ingest per tenant data and verify it with search
|
||||
tenantIDs := []string{"1:1", "1:15", "15:15"}
|
||||
for _, tenantID := range tenantIDs {
|
||||
vminsert.PrometheusAPIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{Tenant: tenantID})
|
||||
vminsert.APIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{Tenant: tenantID})
|
||||
vmstorage1.ForceFlush(t)
|
||||
vmstorage2.ForceFlush(t)
|
||||
|
||||
@@ -206,7 +206,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
|
||||
}
|
||||
|
||||
// verify query request registered correctly
|
||||
vmselect.PrometheusAPIV1Query(t, `{__name__!=""}`, apptest.QueryOpts{
|
||||
vmselect.APIV1Query(t, `{__name__!=""}`, apptest.QueryOpts{
|
||||
Tenant: tenantID, Time: ingestDateTime,
|
||||
})
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ func TestClusterInstantQuery(t *testing.T) {
|
||||
testQueryRangeWithAtModifier(t, sut)
|
||||
}
|
||||
|
||||
func testInstantQueryWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQuerier) {
|
||||
func testInstantQueryWithUTFNames(t *testing.T, sut apptest.WriteQuerier) {
|
||||
data := []pb.TimeSeries{
|
||||
{
|
||||
Labels: []pb.Label{
|
||||
@@ -59,18 +59,18 @@ func testInstantQueryWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQueri
|
||||
},
|
||||
}
|
||||
|
||||
sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
|
||||
sut.APIV1Write(t, data, apptest.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
var got, want *apptest.PrometheusAPIV1QueryResponse
|
||||
var got, want *apptest.APIV1QueryResponse
|
||||
cmpOptions := []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.EquateNaNs(),
|
||||
}
|
||||
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "3fooµ¥", "3👋tfにちは": "漢©®€£"}}]}}`)
|
||||
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "3fooµ¥", "3👋tfにちは": "漢©®€£"}}]}}`)
|
||||
fn := func(query string) {
|
||||
got = sut.PrometheusAPIV1Query(t, query, apptest.QueryOpts{
|
||||
got = sut.APIV1Query(t, query, apptest.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: "2024-01-01T00:01:00.000Z",
|
||||
})
|
||||
@@ -112,24 +112,24 @@ var staleNaNsData = func() []pb.TimeSeries {
|
||||
}
|
||||
}()
|
||||
|
||||
func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.PrometheusWriteQuerier) {
|
||||
func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.WriteQuerier) {
|
||||
|
||||
sut.PrometheusAPIV1Write(t, staleNaNsData, apptest.QueryOpts{})
|
||||
sut.APIV1Write(t, staleNaNsData, apptest.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
var got, want *apptest.PrometheusAPIV1QueryResponse
|
||||
var got, want *apptest.APIV1QueryResponse
|
||||
cmpOptions := []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.EquateNaNs(),
|
||||
}
|
||||
|
||||
// Verify that instant query returns the first point.
|
||||
|
||||
got = sut.PrometheusAPIV1Query(t, "metric", apptest.QueryOpts{
|
||||
got = sut.APIV1Query(t, "metric", apptest.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: "2024-01-01T00:01:00.000Z",
|
||||
})
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}}]}}`)
|
||||
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}}]}}`)
|
||||
want.Data.Result[0].Sample = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
|
||||
if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
@@ -137,11 +137,11 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
|
||||
|
||||
// Verify that instant query does not return stale NaN.
|
||||
|
||||
got = sut.PrometheusAPIV1Query(t, "metric", apptest.QueryOpts{
|
||||
got = sut.APIV1Query(t, "metric", apptest.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: "2024-01-01T00:02:00.000Z",
|
||||
})
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": []}}`)
|
||||
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": []}}`)
|
||||
// Empty response, stale NaN is not included into response
|
||||
if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
@@ -151,11 +151,11 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
|
||||
// while it must not.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5806
|
||||
|
||||
got = sut.PrometheusAPIV1Query(t, "metric[2m]", apptest.QueryOpts{
|
||||
got = sut.APIV1Query(t, "metric[2m]", apptest.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: "2024-01-01T00:02:00.000Z",
|
||||
})
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
|
||||
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
|
||||
s := make([]*apptest.Sample, 2)
|
||||
s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
|
||||
s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
|
||||
@@ -166,11 +166,11 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
|
||||
|
||||
// Verify that exported data contains stale NaN.
|
||||
|
||||
got = sut.PrometheusAPIV1Export(t, `{__name__="metric"}`, apptest.QueryOpts{
|
||||
got = sut.APIV1Export(t, `{__name__="metric"}`, apptest.QueryOpts{
|
||||
Start: "2024-01-01T00:01:00.000Z",
|
||||
End: "2024-01-01T00:02:00.000Z",
|
||||
})
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
|
||||
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
|
||||
s = make([]*apptest.Sample, 2)
|
||||
s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
|
||||
s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
|
||||
@@ -184,7 +184,7 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
|
||||
// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8444
|
||||
// However, conversion of math.NaN to int64 could behave differently depending on platform and Go version.
|
||||
// Hence, this test could succeed for some platforms even if fix is rolled back.
|
||||
func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQuerier) {
|
||||
func testQueryRangeWithAtModifier(t *testing.T, sut apptest.WriteQuerier) {
|
||||
data := []pb.TimeSeries{
|
||||
{
|
||||
Labels: []pb.Label{
|
||||
@@ -204,10 +204,10 @@ func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQueri
|
||||
},
|
||||
}
|
||||
|
||||
sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
|
||||
sut.APIV1Write(t, data, apptest.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
resp := sut.PrometheusAPIV1QueryRange(t, `vector(1) @ up`, apptest.QueryOpts{
|
||||
resp := sut.APIV1QueryRange(t, `vector(1) @ up`, apptest.QueryOpts{
|
||||
Start: "2025-01-01T00:00:00Z",
|
||||
End: "2025-01-01T00:02:00Z",
|
||||
Step: "10s",
|
||||
@@ -217,7 +217,7 @@ func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQueri
|
||||
t.Fatalf("unexpected status: %q", resp.Status)
|
||||
}
|
||||
|
||||
resp = sut.PrometheusAPIV1QueryRange(t, `vector(1) @ metricNaN`, apptest.QueryOpts{
|
||||
resp = sut.APIV1QueryRange(t, `vector(1) @ metricNaN`, apptest.QueryOpts{
|
||||
Start: "2025-01-01T00:00:00Z",
|
||||
End: "2025-01-01T00:02:00Z",
|
||||
Step: "10s",
|
||||
|
||||
@@ -37,7 +37,7 @@ func TestClusterMultilevelSelect(t *testing.T) {
|
||||
|
||||
const numMetrics = 1000
|
||||
records := make([]string, numMetrics)
|
||||
want := &apptest.PrometheusAPIV1SeriesResponse{
|
||||
want := &apptest.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
IsPartial: false,
|
||||
Data: make([]map[string]string, numMetrics),
|
||||
@@ -49,7 +49,7 @@ func TestClusterMultilevelSelect(t *testing.T) {
|
||||
}
|
||||
want.Sort()
|
||||
qopts := apptest.QueryOpts{Tenant: "0"}
|
||||
vminsert.PrometheusAPIV1ImportPrometheus(t, records, qopts)
|
||||
vminsert.APIV1ImportPrometheus(t, records, qopts)
|
||||
vmstorage.ForceFlush(t)
|
||||
|
||||
// Retrieve all time series and verify that both vmselect (L1) and
|
||||
@@ -60,7 +60,7 @@ func TestClusterMultilevelSelect(t *testing.T) {
|
||||
tc.Assert(&apptest.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, qopts)
|
||||
res := app.APIV1Series(t, `{__name__=~".*"}`, qopts)
|
||||
res.Sort()
|
||||
return res
|
||||
},
|
||||
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
os.RemoveAll(t.Name())
|
||||
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
cmpSROpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Status", "IsPartial")
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
cmpSROpt := cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Status", "IsPartial")
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
@@ -37,12 +37,12 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
}
|
||||
|
||||
// test for empty tenants request
|
||||
got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
|
||||
got := vmselect.APIV1Query(t, "foo_bar", apptest.QueryOpts{
|
||||
Tenant: "multitenant",
|
||||
Step: "5m",
|
||||
Time: "2022-05-10T08:03:00.000Z",
|
||||
})
|
||||
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
@@ -51,12 +51,12 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
tenantIDs := []string{"1:1", "1:15"}
|
||||
instantCT := "2022-05-10T08:05:00.000Z"
|
||||
for _, tenantID := range tenantIDs {
|
||||
vminsert.PrometheusAPIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
|
||||
vminsert.APIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
|
||||
vmstorage.ForceFlush(t)
|
||||
got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
|
||||
got := vmselect.APIV1Query(t, "foo_bar", apptest.QueryOpts{
|
||||
Tenant: tenantID, Time: instantCT,
|
||||
})
|
||||
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}]}}`)
|
||||
want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}]}}`)
|
||||
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
@@ -64,7 +64,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
// verify all tenants searchable with multitenant APIs
|
||||
|
||||
// /api/v1/query
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t,
|
||||
want = apptest.NewAPIV1QueryResponse(t,
|
||||
`{"data":
|
||||
{"result":[
|
||||
{"metric":{"__name__":"foo_bar","vm_account_id":"1","vm_project_id": "1"},"value":[1652169900,"3"]},
|
||||
@@ -73,7 +73,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
}
|
||||
}`,
|
||||
)
|
||||
got = vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
|
||||
got = vmselect.APIV1Query(t, "foo_bar", apptest.QueryOpts{
|
||||
Tenant: "multitenant",
|
||||
Time: instantCT,
|
||||
})
|
||||
@@ -83,14 +83,14 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
|
||||
// /api/v1/query_range aggregated by tenant labels
|
||||
query := "sum(foo_bar) by(vm_account_id,vm_project_id)"
|
||||
got = vmselect.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
|
||||
got = vmselect.APIV1QueryRange(t, query, apptest.QueryOpts{
|
||||
Tenant: "multitenant",
|
||||
Start: "2022-05-10T07:59:00.000Z",
|
||||
End: "2022-05-10T08:05:00.000Z",
|
||||
Step: "1m",
|
||||
})
|
||||
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t,
|
||||
want = apptest.NewAPIV1QueryResponse(t,
|
||||
`{"data":
|
||||
{"result": [
|
||||
{"metric": {"vm_account_id": "1","vm_project_id":"1"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]},
|
||||
@@ -104,7 +104,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
|
||||
// verify /api/v1/series response
|
||||
|
||||
wantSR := apptest.NewPrometheusAPIV1SeriesResponse(t,
|
||||
wantSR := apptest.NewAPIV1SeriesResponse(t,
|
||||
`{"data": [
|
||||
{"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
|
||||
{"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}
|
||||
@@ -112,7 +112,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
}`)
|
||||
wantSR.Sort()
|
||||
|
||||
gotSR := vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
|
||||
gotSR := vmselect.APIV1Series(t, "foo_bar", apptest.QueryOpts{
|
||||
Tenant: "multitenant",
|
||||
Start: "2022-05-10T08:03:00.000Z",
|
||||
})
|
||||
@@ -129,11 +129,11 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
|
||||
}
|
||||
|
||||
vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
|
||||
vminsert.APIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
|
||||
vmstorage.ForceFlush(t)
|
||||
|
||||
// /api/v1/query with query filters
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t,
|
||||
want = apptest.NewAPIV1QueryResponse(t,
|
||||
`{"data":
|
||||
{"result":[
|
||||
{"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"value":[1652169900,"1"]},
|
||||
@@ -142,7 +142,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
}
|
||||
}`,
|
||||
)
|
||||
got = vmselect.PrometheusAPIV1Query(t, `foo_bar{vm_account_id="5"}`, apptest.QueryOpts{
|
||||
got = vmselect.APIV1Query(t, `foo_bar{vm_account_id="5"}`, apptest.QueryOpts{
|
||||
Time: instantCT,
|
||||
Tenant: "multitenant",
|
||||
})
|
||||
@@ -152,14 +152,14 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
|
||||
// /api/v1/series with extra_filters
|
||||
|
||||
wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
|
||||
wantSR = apptest.NewAPIV1SeriesResponse(t,
|
||||
`{"data": [
|
||||
{"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"15"},
|
||||
{"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}
|
||||
]
|
||||
}`)
|
||||
wantSR.Sort()
|
||||
gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
|
||||
gotSR = vmselect.APIV1Series(t, "foo_bar", apptest.QueryOpts{
|
||||
Start: "2022-05-10T08:00:00.000Z",
|
||||
End: "2022-05-10T08:30:00.000Z",
|
||||
ExtraFilters: []string{`{vm_project_id="15"}`},
|
||||
@@ -175,7 +175,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
vmselect.DeleteSeries(t, "foo_bar", apptest.QueryOpts{
|
||||
Tenant: "5:15",
|
||||
})
|
||||
wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
|
||||
wantSR = apptest.NewAPIV1SeriesResponse(t,
|
||||
`{"data": [
|
||||
{"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
|
||||
{"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
|
||||
@@ -185,7 +185,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
}`)
|
||||
wantSR.Sort()
|
||||
|
||||
gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
|
||||
gotSR = vmselect.APIV1Series(t, "foo_bar", apptest.QueryOpts{
|
||||
Tenant: "multitenant",
|
||||
Start: "2022-05-10T08:03:00.000Z",
|
||||
})
|
||||
@@ -199,7 +199,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
Tenant: "multitenant",
|
||||
})
|
||||
|
||||
wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
|
||||
wantSR = apptest.NewAPIV1SeriesResponse(t,
|
||||
`{"data": [
|
||||
{"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
|
||||
{"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
|
||||
@@ -207,7 +207,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
|
||||
}`)
|
||||
wantSR.Sort()
|
||||
|
||||
gotSR = vmselect.PrometheusAPIV1Series(t, `foo_bar`, apptest.QueryOpts{
|
||||
gotSR = vmselect.APIV1Series(t, `foo_bar`, apptest.QueryOpts{
|
||||
Tenant: "multitenant",
|
||||
Start: "2022-05-10T08:03:00.000Z",
|
||||
})
|
||||
|
||||
@@ -11,7 +11,7 @@ func TestSingleSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
tc := at.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.PrometheusWriteQuerier {
|
||||
testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.WriteQuerier {
|
||||
return tc.MustStartVmsingle("vmsingle-"+name, []string{
|
||||
"-storageDataPath=" + tc.Dir() + "/vmsingle",
|
||||
"-retentionPeriod=100y",
|
||||
@@ -25,7 +25,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
tc := at.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.PrometheusWriteQuerier {
|
||||
testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.WriteQuerier {
|
||||
// Using static ports for vmstorage because random ports may cause
|
||||
// changes in how data is sharded.
|
||||
vmstorage1 := tc.MustStartVmstorage("vmstorage1-"+name, []string{
|
||||
@@ -59,7 +59,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
type startSUTFunc func(name string, disablePerDayIndex bool) at.PrometheusWriteQuerier
|
||||
type startSUTFunc func(name string, disablePerDayIndex bool) at.WriteQuerier
|
||||
|
||||
// testDisablePerDayIndex_Search shows what search results to expect when data
|
||||
// is first inserted with per-day index enabled and then with per-day index
|
||||
@@ -78,17 +78,17 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
|
||||
wantSeries []map[string]string
|
||||
wantQueryResults []*at.QueryResult
|
||||
}
|
||||
assertSearchResults := func(sut at.PrometheusQuerier, opts *opts) {
|
||||
assertSearchResults := func(sut at.APIQuerier, opts *opts) {
|
||||
t.Helper()
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
return sut.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
Start: opts.start,
|
||||
End: opts.end,
|
||||
}).Sort()
|
||||
},
|
||||
Want: &at.PrometheusAPIV1SeriesResponse{
|
||||
Want: &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
Data: opts.wantSeries,
|
||||
},
|
||||
@@ -96,13 +96,13 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1QueryRange(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
return sut.APIV1QueryRange(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
Start: opts.start,
|
||||
End: opts.end,
|
||||
Step: "1d",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -116,7 +116,7 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
|
||||
// is searchable.
|
||||
sut := start("with-per-day-index", false)
|
||||
sample1 := []string{"metric1 111 1704067200000"} // 2024-01-01T00:00:00Z
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, sample1, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, sample1, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
assertSearchResults(sut, &opts{
|
||||
start: "2024-01-01T00:00:00Z",
|
||||
@@ -132,10 +132,10 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
|
||||
|
||||
// Restart vmsingle with disabled per-day index, insert sample2, and confirm
|
||||
// that both sample1 and sample2 is searchable.
|
||||
tc.StopPrometheusWriteQuerier(sut)
|
||||
tc.StopWriteQuerier(sut)
|
||||
sut = start("without-per-day-index", true)
|
||||
sample2 := []string{"metric2 222 1704067200000"} // 2024-01-01T00:00:00Z
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, sample2, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, sample2, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
assertSearchResults(sut, &opts{
|
||||
start: "2024-01-01T00:00:00Z",
|
||||
@@ -165,9 +165,9 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
|
||||
// - sample2 is not searchable when the time range is <= 40 days
|
||||
// - sample2 becomes searchable when the time range is > 40 days
|
||||
sample3 := []string{"metric1 333 1705708800000"} // 2024-01-20T00:00:00Z
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, sample3, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, sample3, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
tc.StopPrometheusWriteQuerier(sut)
|
||||
tc.StopWriteQuerier(sut)
|
||||
sut = start("with-per-day-index2", false)
|
||||
|
||||
// Time range is 1 day (Jan 1st) <= 40 days
|
||||
@@ -295,14 +295,14 @@ func testClusterActiveTimeseriesMetric(t *testing.T, disablePerDayIndex bool) {
|
||||
})
|
||||
}
|
||||
|
||||
func testActiveTimeseriesMetric(tc *at.TestCase, sut at.PrometheusWriteQuerier, getActiveTimeseries func() int) {
|
||||
func testActiveTimeseriesMetric(tc *at.TestCase, sut at.WriteQuerier, getActiveTimeseries func() int) {
|
||||
t := tc.T()
|
||||
const numSamples = 1000
|
||||
samples := make([]string, numSamples)
|
||||
for i := range numSamples {
|
||||
samples[i] = fmt.Sprintf("metric_%03d %d", i, i)
|
||||
}
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, samples, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, samples, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: `unexpected vm_cache_entries{type="storage/hour_metric_ids"} metric value`,
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
func TestClusterMaxUniqueTimeseries(t *testing.T) {
|
||||
os.RemoveAll(t.Name())
|
||||
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
@@ -54,14 +54,14 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
|
||||
// write data to two tenants
|
||||
tenantIDs := []string{"0:0", "1:15"}
|
||||
for _, tenantID := range tenantIDs {
|
||||
vminsert.PrometheusAPIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
|
||||
vminsert.APIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
|
||||
vmstorage.ForceFlush(t)
|
||||
}
|
||||
|
||||
instantCT := "2022-05-10T08:05:00.000Z"
|
||||
|
||||
// success - `/api/v1/query`
|
||||
want := apptest.NewPrometheusAPIV1QueryResponse(t,
|
||||
want := apptest.NewAPIV1QueryResponse(t,
|
||||
`{"data":
|
||||
{"result":[
|
||||
{"metric":{"__name__":"foo_bar1","instance":"a"},"value":[1652169900,"1"]}
|
||||
@@ -69,7 +69,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
|
||||
}
|
||||
}`,
|
||||
)
|
||||
queryRes := vmselectSmallLimit.PrometheusAPIV1Query(t, "foo_bar1", apptest.QueryOpts{
|
||||
queryRes := vmselectSmallLimit.APIV1Query(t, "foo_bar1", apptest.QueryOpts{
|
||||
Time: instantCT,
|
||||
})
|
||||
if diff := cmp.Diff(want, queryRes, cmpOpt); diff != "" {
|
||||
@@ -78,7 +78,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
|
||||
|
||||
// success - multitenant `/api/v1/query`
|
||||
// query is split into two queries for each tenant, so the final result can exceed the limit.
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t,
|
||||
want = apptest.NewAPIV1QueryResponse(t,
|
||||
`{"data":
|
||||
{"result":[
|
||||
{"metric":{"__name__":"foo_bar1","instance":"a","vm_account_id":"0","vm_project_id":"0"},"value":[1652169900,"1"]},
|
||||
@@ -87,7 +87,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
|
||||
}
|
||||
}`,
|
||||
)
|
||||
queryRes = vmselectSmallLimit.PrometheusAPIV1Query(t, "foo_bar1", apptest.QueryOpts{
|
||||
queryRes = vmselectSmallLimit.APIV1Query(t, "foo_bar1", apptest.QueryOpts{
|
||||
Time: instantCT,
|
||||
Tenant: "multitenant",
|
||||
})
|
||||
@@ -96,7 +96,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
|
||||
}
|
||||
|
||||
// fail - `/api/v1/query`, exceed vmselect `maxUniqueTimeseries`
|
||||
queryRes = vmselectSmallLimit.PrometheusAPIV1Query(t, "foo_bar2", apptest.QueryOpts{
|
||||
queryRes = vmselectSmallLimit.APIV1Query(t, "foo_bar2", apptest.QueryOpts{
|
||||
Time: instantCT,
|
||||
})
|
||||
if queryRes.ErrorType != "422" {
|
||||
@@ -104,7 +104,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
|
||||
}
|
||||
|
||||
// fail - `/api/v1/query`, exceed vmstorage `maxUniqueTimeseries`
|
||||
queryRes = vmselectNoLimit.PrometheusAPIV1Query(t, "foo_bar3", apptest.QueryOpts{
|
||||
queryRes = vmselectNoLimit.APIV1Query(t, "foo_bar3", apptest.QueryOpts{
|
||||
Time: instantCT,
|
||||
})
|
||||
if queryRes.ErrorType != "422" {
|
||||
@@ -112,7 +112,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
|
||||
}
|
||||
|
||||
// fail - `/api/v1/query`, vmselect `maxUniqueTimeseries` cannot exceed vmstorage `maxUniqueTimeseries`
|
||||
queryRes = vmselectBigLimit.PrometheusAPIV1Query(t, "foo_bar3", apptest.QueryOpts{
|
||||
queryRes = vmselectBigLimit.APIV1Query(t, "foo_bar3", apptest.QueryOpts{
|
||||
Time: instantCT,
|
||||
})
|
||||
if queryRes.ErrorType != "422" {
|
||||
@@ -123,7 +123,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
|
||||
func TestClusterMaxSeries(t *testing.T) {
|
||||
os.RemoveAll(t.Name())
|
||||
|
||||
cmpSROpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Status", "IsPartial")
|
||||
cmpSROpt := cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Status", "IsPartial")
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
@@ -153,18 +153,18 @@ func TestClusterMaxSeries(t *testing.T) {
|
||||
}
|
||||
|
||||
// write data
|
||||
vminsert.PrometheusAPIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{})
|
||||
vminsert.APIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{})
|
||||
vmstorage.ForceFlush(t)
|
||||
|
||||
// success - `/api/v1/series`, vmselect `maxLabelsAPISeries` can exceed vmstorage `maxLabelsAPISeries``
|
||||
wantSR := apptest.NewPrometheusAPIV1SeriesResponse(t,
|
||||
wantSR := apptest.NewAPIV1SeriesResponse(t,
|
||||
`{"data": [
|
||||
{"__name__":"foo_bar3","instance":"a"},
|
||||
{"__name__":"foo_bar3","instance":"b"},
|
||||
{"__name__":"foo_bar3","instance":"c"}
|
||||
]
|
||||
}`)
|
||||
seriesRes := vmselectBigLimit.PrometheusAPIV1Series(t, "foo_bar3", apptest.QueryOpts{
|
||||
seriesRes := vmselectBigLimit.APIV1Series(t, "foo_bar3", apptest.QueryOpts{
|
||||
Start: "2022-05-10T08:03:00.000Z",
|
||||
})
|
||||
if diff := cmp.Diff(wantSR.Sort(), seriesRes.Sort(), cmpSROpt); diff != "" {
|
||||
@@ -172,7 +172,7 @@ func TestClusterMaxSeries(t *testing.T) {
|
||||
}
|
||||
|
||||
// fail - `/api/v1/series`, exceed vmselect `maxSeries`
|
||||
seriesRes1 := vmselectSmallLimit.PrometheusAPIV1Series(t, "foo_bar3", apptest.QueryOpts{
|
||||
seriesRes1 := vmselectSmallLimit.APIV1Series(t, "foo_bar3", apptest.QueryOpts{
|
||||
Start: "2022-05-10T08:03:00.000Z",
|
||||
})
|
||||
if seriesRes1.ErrorType != "422" {
|
||||
|
||||
@@ -57,7 +57,7 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
|
||||
wantMetrics []map[string]string
|
||||
wantSamples []*at.Sample
|
||||
}
|
||||
f := func(sut at.PrometheusQuerier, opts *opts) {
|
||||
f := func(sut at.APIQuerier, opts *opts) {
|
||||
t.Helper()
|
||||
wantResult := []*at.QueryResult{}
|
||||
for idx, wm := range opts.wantMetrics {
|
||||
@@ -70,19 +70,19 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1Query(t, opts.query, at.QueryOpts{
|
||||
return sut.APIV1Query(t, opts.query, at.QueryOpts{
|
||||
Time: opts.qtime,
|
||||
Step: opts.step,
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
|
||||
Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`importprometheus_series{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
`must_drop_series{label="foo"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
|
||||
}, at.QueryOpts{})
|
||||
@@ -180,7 +180,7 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
sut.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
|
||||
sut.APIV1Write(t, pbData, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
f(sut, &opts{
|
||||
query: `{label="foo2"}[120ms]`,
|
||||
|
||||
@@ -7,10 +7,11 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
)
|
||||
|
||||
type clusterWithReplication struct {
|
||||
@@ -94,7 +95,7 @@ func TestClusterReplication_DataIsWrittenSeveralTimes(t *testing.T) {
|
||||
for i := range numRecs {
|
||||
recs[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
|
||||
}
|
||||
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
tc.ForceFlush(c.vmstorages...)
|
||||
|
||||
// Verify that each storage node has metrics and that total metric count across
|
||||
@@ -151,7 +152,7 @@ func TestClusterReplication_Deduplication(t *testing.T) {
|
||||
ts = ts.Add(1 * time.Minute)
|
||||
}
|
||||
}
|
||||
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
tc.ForceFlush(c.vmstorages...)
|
||||
|
||||
// Check /api/v1/series response.
|
||||
@@ -164,12 +165,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
Start: "2024-01-01T00:00:00Z",
|
||||
End: "2024-01-31T00:00:00Z",
|
||||
}).Sort()
|
||||
},
|
||||
Want: &at.PrometheusAPIV1SeriesResponse{
|
||||
Want: &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
IsPartial: false,
|
||||
Data: []map[string]string{
|
||||
@@ -194,12 +195,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Query(t, "metric_1", at.QueryOpts{
|
||||
return app.APIV1Query(t, "metric_1", at.QueryOpts{
|
||||
Time: "2024-01-01T00:05:00Z",
|
||||
Step: "5m",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "vector",
|
||||
@@ -236,12 +237,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Query(t, "metric_1[5m]", at.QueryOpts{
|
||||
return app.APIV1Query(t, "metric_1[5m]", at.QueryOpts{
|
||||
Time: "2024-01-01T00:05:00Z",
|
||||
Step: "5m",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -273,13 +274,13 @@ func TestClusterReplication_Deduplication(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1QueryRange(t, "metric_1", at.QueryOpts{
|
||||
return app.APIV1QueryRange(t, "metric_1", at.QueryOpts{
|
||||
Start: "2024-01-01T00:00:00Z",
|
||||
End: "2024-01-01T00:10:00Z",
|
||||
Step: "5m",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -309,12 +310,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/export response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
|
||||
return app.APIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
|
||||
Start: "2024-01-01T00:00:00Z",
|
||||
End: "2024-01-01T00:03:00Z",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -360,7 +361,7 @@ func TestClusterReplication_PartialResponse(t *testing.T) {
|
||||
for i := range numRecs {
|
||||
recs[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
|
||||
}
|
||||
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
tc.ForceFlush(c.vmstorages...)
|
||||
|
||||
// Verify partial vs full response.
|
||||
@@ -370,14 +371,14 @@ func TestClusterReplication_PartialResponse(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
|
||||
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
|
||||
},
|
||||
Want: &at.PrometheusAPIV1SeriesResponse{
|
||||
Want: &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
IsPartial: wantPartial,
|
||||
},
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
|
||||
cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Data"),
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -437,7 +438,7 @@ func TestClusterReplication_SkipSlowReplicas(t *testing.T) {
|
||||
|
||||
const numRecs = 1000
|
||||
recs := make([]string, numRecs)
|
||||
wantSeries := &at.PrometheusAPIV1SeriesResponse{
|
||||
wantSeries := &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
Data: make([]map[string]string, numRecs),
|
||||
}
|
||||
@@ -447,7 +448,7 @@ func TestClusterReplication_SkipSlowReplicas(t *testing.T) {
|
||||
wantSeries.Data[i] = map[string]string{"__name__": name}
|
||||
}
|
||||
wantSeries.Sort()
|
||||
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
tc.ForceFlush(c.vmstorages...)
|
||||
|
||||
// Verify skipping slow replicas by counting the number of skipSlowReplicas
|
||||
@@ -458,12 +459,12 @@ func TestClusterReplication_SkipSlowReplicas(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
|
||||
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
|
||||
},
|
||||
Want: wantSeries,
|
||||
})
|
||||
|
||||
res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
|
||||
res := app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
|
||||
got := res.Trace.Contains("cancel request because -search.skipSlowReplicas is set and every group returned the needed number of responses according to replicationFactor")
|
||||
if got != want {
|
||||
t.Errorf("unexpected number of skipSlowReplicas messages in request trace: got %d, want %d (full trace:\n%v)", got, want, res.Trace)
|
||||
@@ -654,7 +655,7 @@ func TestClusterGroupReplication(t *testing.T) {
|
||||
numRecs = numMetrics * numSamples
|
||||
)
|
||||
var recs []string
|
||||
wantSeries := &at.PrometheusAPIV1SeriesResponse{
|
||||
wantSeries := &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
Data: make([]map[string]string, numMetrics),
|
||||
}
|
||||
@@ -668,7 +669,7 @@ func TestClusterGroupReplication(t *testing.T) {
|
||||
}
|
||||
}
|
||||
wantSeries.Sort()
|
||||
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
|
||||
c.forceFlush(tc)
|
||||
|
||||
opts := &testGroupReplicationOpts{
|
||||
@@ -694,7 +695,7 @@ type testGroupReplicationOpts struct {
|
||||
numGroups int
|
||||
numNodes int
|
||||
numRecs int
|
||||
wantSeries *at.PrometheusAPIV1SeriesResponse
|
||||
wantSeries *at.APIV1SeriesResponse
|
||||
}
|
||||
|
||||
// testGroupDataIsWrittenSeveralTimes checks that multiple
|
||||
@@ -747,7 +748,7 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
Start: "2024-01-01T00:00:00Z",
|
||||
End: "2024-01-31T00:00:00Z",
|
||||
}).Sort()
|
||||
@@ -768,12 +769,12 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Query(t, "metric_1", at.QueryOpts{
|
||||
return app.APIV1Query(t, "metric_1", at.QueryOpts{
|
||||
Time: "2024-01-01T00:05:00Z",
|
||||
Step: "5m",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "vector",
|
||||
@@ -810,12 +811,12 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Query(t, "metric_1[5m]", at.QueryOpts{
|
||||
return app.APIV1Query(t, "metric_1[5m]", at.QueryOpts{
|
||||
Time: "2024-01-01T00:05:00Z",
|
||||
Step: "5m",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -847,13 +848,13 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1QueryRange(t, "metric_1", at.QueryOpts{
|
||||
return app.APIV1QueryRange(t, "metric_1", at.QueryOpts{
|
||||
Start: "2024-01-01T00:00:00Z",
|
||||
End: "2024-01-01T00:10:00Z",
|
||||
Step: "5m",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -883,12 +884,12 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/export response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
|
||||
return app.APIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
|
||||
Start: "2024-01-01T00:00:00Z",
|
||||
End: "2024-01-01T00:03:00Z",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -929,7 +930,7 @@ func testGroupSkipSlowReplicas(tc *at.TestCase, opts *testGroupReplicationOpts)
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
Start: "2024-01-01T00:00:00Z",
|
||||
End: "2024-01-31T00:00:00Z",
|
||||
}).Sort()
|
||||
@@ -937,7 +938,7 @@ func testGroupSkipSlowReplicas(tc *at.TestCase, opts *testGroupReplicationOpts)
|
||||
Want: opts.wantSeries,
|
||||
})
|
||||
|
||||
res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
|
||||
res := app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
|
||||
got := res.Trace.Contains("cancel request because -search.skipSlowReplicas is set and every group returned the needed number of responses according to replicationFactor")
|
||||
if got < wantMin || got > wantMax {
|
||||
t.Errorf("unexpected number of skipSlowReplicas messages in request trace: got %d, %d <= want <= %d (full trace:\n%v)", got, wantMin, wantMax, res.Trace)
|
||||
@@ -973,7 +974,7 @@ func testGroupSkipSlowReplicas(tc *at.TestCase, opts *testGroupReplicationOpts)
|
||||
|
||||
// The data is replicated across N groups of M nodes. Replication factor is
|
||||
// globalRF. There is no replication across the nodes within each group or
|
||||
//it is unknown it there is one.
|
||||
// it is unknown it there is one.
|
||||
//
|
||||
// Max number of nodes to skip is M*(globalRF-1). This corresponds to the
|
||||
// case when N-globalRF+1 groups have received the response from all of
|
||||
@@ -1020,17 +1021,17 @@ func testGroupPartialResponse(tc *at.TestCase, opts *testGroupReplicationOpts) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
Start: "2024-01-01T00:00:00Z",
|
||||
End: "2024-01-31T00:00:00Z",
|
||||
}).Sort()
|
||||
},
|
||||
Want: &at.PrometheusAPIV1SeriesResponse{
|
||||
Want: &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
IsPartial: wantPartial,
|
||||
},
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
|
||||
cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Data"),
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1128,10 +1129,10 @@ func TestClusterReplication_PartialResponseMultitenant(t *testing.T) {
|
||||
recs[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
|
||||
}
|
||||
|
||||
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{
|
||||
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{
|
||||
Tenant: "0",
|
||||
})
|
||||
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{
|
||||
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{
|
||||
Tenant: "1",
|
||||
})
|
||||
tc.ForceFlush(c.vmstorages...)
|
||||
@@ -1144,14 +1145,14 @@ func TestClusterReplication_PartialResponseMultitenant(t *testing.T) {
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
Got: func() any {
|
||||
qo := at.QueryOpts{Tenant: "multitenant", Trace: "1"}
|
||||
return app.PrometheusAPIV1Query(t, `{__name__=~"metric_.*"}`, qo)
|
||||
return app.APIV1Query(t, `{__name__=~"metric_.*"}`, qo)
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
IsPartial: wantPartial,
|
||||
},
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Data"),
|
||||
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Data"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
func TestClusterRollupResultCache(t *testing.T) {
|
||||
os.RemoveAll(t.Name())
|
||||
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
@@ -34,10 +34,10 @@ func TestClusterRollupResultCache(t *testing.T) {
|
||||
`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
|
||||
}
|
||||
|
||||
vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
|
||||
vminsert.APIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
|
||||
vmstorage.ForceFlush(t)
|
||||
|
||||
want := apptest.NewPrometheusAPIV1QueryResponse(t,
|
||||
want := apptest.NewAPIV1QueryResponse(t,
|
||||
`{"data":
|
||||
{"result":[
|
||||
{"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"values":[[1652169720,"1"],[1652169780,"1"]]},
|
||||
@@ -47,7 +47,7 @@ func TestClusterRollupResultCache(t *testing.T) {
|
||||
}`,
|
||||
)
|
||||
|
||||
got := vmselect.PrometheusAPIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
|
||||
got := vmselect.APIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
|
||||
Tenant: "multitenant",
|
||||
Start: "2022-05-10T07:59:00.000Z",
|
||||
End: "2022-05-10T08:05:00.000Z",
|
||||
@@ -58,13 +58,13 @@ func TestClusterRollupResultCache(t *testing.T) {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
|
||||
want = apptest.NewPrometheusAPIV1QueryResponse(t,
|
||||
want = apptest.NewAPIV1QueryResponse(t,
|
||||
`{"data":
|
||||
{"result":[]}
|
||||
}`,
|
||||
)
|
||||
|
||||
got = vmselect.PrometheusAPIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
|
||||
got = vmselect.APIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
|
||||
Tenant: "multitenant",
|
||||
Start: "2022-05-10T07:59:00.000Z",
|
||||
End: "2022-05-10T08:05:00.000Z",
|
||||
|
||||
@@ -38,7 +38,7 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
|
||||
|
||||
const numMetrics = 1000
|
||||
records := make([]string, numMetrics)
|
||||
want := &apptest.PrometheusAPIV1SeriesResponse{
|
||||
want := &apptest.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
IsPartial: false,
|
||||
Data: make([]map[string]string, numMetrics),
|
||||
@@ -49,7 +49,7 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
|
||||
want.Data[i] = map[string]string{"__name__": name}
|
||||
}
|
||||
want.Sort()
|
||||
vminsert.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{})
|
||||
vminsert.APIV1ImportPrometheus(t, records, apptest.QueryOpts{})
|
||||
vmstorage1.ForceFlush(t)
|
||||
vmstorage2.ForceFlush(t)
|
||||
|
||||
@@ -74,7 +74,7 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
|
||||
tc.Assert(&apptest.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
res := vmselect.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{})
|
||||
res := vmselect.APIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{})
|
||||
res.Sort()
|
||||
return res
|
||||
},
|
||||
|
||||
@@ -28,7 +28,7 @@ func TestSingleSnapshots_CreateListDelete(t *testing.T) {
|
||||
for i := range numSamples {
|
||||
samples[i] = fmt.Sprintf("metric_%03d %d", i, i)
|
||||
}
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, samples, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, samples, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
// Create several snapshots using VictoriaMetrics and Prometheus endpoints.
|
||||
@@ -113,7 +113,7 @@ func TestClusterSnapshots_CreateListDelete(t *testing.T) {
|
||||
for i := range numSamples {
|
||||
samples[i] = fmt.Sprintf("metric_%03d %d", i, i)
|
||||
}
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, samples, at.QueryOpts{})
|
||||
sut.APIV1ImportPrometheus(t, samples, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
// Create several snapshots for both vmstorage replicas using
|
||||
|
||||
@@ -4,9 +4,10 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
)
|
||||
|
||||
// TestSingleSpecialQueryRegression is used to test queries that have experienced issues for specific data sets.
|
||||
@@ -35,13 +36,12 @@ func TestClusterSpecialQueryRegression(t *testing.T) {
|
||||
testSpecialQueryRegression(tc, sut)
|
||||
}
|
||||
|
||||
func testSpecialQueryRegression(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testSpecialQueryRegression(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
// prometheus
|
||||
testCaseSensitiveRegex(tc, sut)
|
||||
testDuplicateLabel(tc, sut)
|
||||
testTooBigLookbehindWindow(tc, sut)
|
||||
testMatchSeries(tc, sut)
|
||||
testNegativeIncrease(tc, sut)
|
||||
|
||||
// graphite
|
||||
testComparisonNotInfNotNan(tc, sut)
|
||||
@@ -51,12 +51,12 @@ func testSpecialQueryRegression(tc *at.TestCase, sut at.PrometheusWriteQuerier)
|
||||
testSubqueryAggregation(tc, sut)
|
||||
}
|
||||
|
||||
func testCaseSensitiveRegex(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testCaseSensitiveRegex(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// case-sensitive-regex
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/161
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`prometheus.sensitiveRegex{label="sensitiveRegex"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
`prometheus.sensitiveRegex{label="SensitiveRegex"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
}, at.QueryOpts{})
|
||||
@@ -65,12 +65,12 @@ func testCaseSensitiveRegex(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/export response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1Export(t, `{label=~'(?i)sensitiveregex'}`, at.QueryOpts{
|
||||
return sut.APIV1Export(t, `{label=~'(?i)sensitiveregex'}`, at.QueryOpts{
|
||||
Start: "2024-02-05T08:50:00.700Z",
|
||||
End: "2024-02-05T09:00:00.700Z",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -87,17 +87,17 @@ func testCaseSensitiveRegex(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
},
|
||||
},
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func testDuplicateLabel(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testDuplicateLabel(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// duplicate_label
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/172
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`prometheus.duplicate_label{label="duplicate", label="duplicate"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
}, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
@@ -105,12 +105,12 @@ func testDuplicateLabel(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/export response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1Export(t, `{__name__='prometheus.duplicate_label'}`, at.QueryOpts{
|
||||
return sut.APIV1Export(t, `{__name__='prometheus.duplicate_label'}`, at.QueryOpts{
|
||||
Start: "2024-02-05T08:50:00.700Z",
|
||||
End: "2024-02-05T09:00:00.700Z",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -123,17 +123,17 @@ func testDuplicateLabel(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
},
|
||||
},
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testTooBigLookbehindWindow(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// too big look-behind window
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`prometheus.too_big_lookbehind{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
}, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
@@ -141,12 +141,12 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1Query(t, `prometheus.too_big_lookbehind{label="foo"}[100y]`, at.QueryOpts{
|
||||
return sut.APIV1Query(t, `prometheus.too_big_lookbehind{label="foo"}[100y]`, at.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: "2024-02-05T08:57:36.700Z",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -164,7 +164,7 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
|
||||
|
||||
// too big look-behind window - query range
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`prometheus.too_big_lookbehind_range{label="foo"} 13 1707123496700`, // 2024-02-05T08:58:16.700Z
|
||||
`prometheus.too_big_lookbehind_range{label="foo"} 12 1707123466700`, // 2024-02-05T08:57:46.700Z
|
||||
`prometheus.too_big_lookbehind_range{label="foo"} 11 1707123436700`, // 2024-02-05T08:57:16.700Z
|
||||
@@ -175,13 +175,13 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1QueryRange(t, `prometheus.too_big_lookbehind_range{label="foo"}`, at.QueryOpts{
|
||||
return sut.APIV1QueryRange(t, `prometheus.too_big_lookbehind_range{label="foo"}`, at.QueryOpts{
|
||||
Start: "2024-02-05T08:56:46.700Z",
|
||||
End: "2024-02-05T08:58:16.700Z",
|
||||
Step: "30s",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -201,12 +201,12 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
|
||||
})
|
||||
}
|
||||
|
||||
func testMatchSeries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testMatchSeries(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// match_series
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/155
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
sut.APIV1ImportPrometheus(t, []string{
|
||||
`GenBearTemp{db="TenMinute",Park="1",TurbineType="V112"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
`GenBearTemp{db="TenMinute",Park="2",TurbineType="V112"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
`GenBearTemp{db="TenMinute",Park="3",TurbineType="V112"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
|
||||
@@ -217,12 +217,12 @@ func testMatchSeries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/series response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1Series(t, `{__name__="GenBearTemp"}`, at.QueryOpts{
|
||||
return sut.APIV1Series(t, `{__name__="GenBearTemp"}`, at.QueryOpts{
|
||||
Start: "2024-02-04T08:57:36.700Z",
|
||||
End: "2024-02-05T08:57:36.700Z",
|
||||
}).Sort()
|
||||
},
|
||||
Want: &at.PrometheusAPIV1SeriesResponse{
|
||||
Want: &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
IsPartial: false,
|
||||
Data: []map[string]string{
|
||||
@@ -235,54 +235,7 @@ func testMatchSeries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
})
|
||||
}
|
||||
|
||||
func testNegativeIncrease(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// negative increase when user overrides staleness interval
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8935#issuecomment-2978728661
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
`foo 108 1750109243514`, // 2025-06-16 21:27:23:514
|
||||
`foo 108 1750109258514`, // 2025-06-16 21:27:38:514
|
||||
// gap 75s
|
||||
`foo 1 1750109333514`, // 2025-06-16 21:28:53:514
|
||||
`foo 1 1750109348514`, // 2025-06-16 21:29:08:514
|
||||
}, at.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "regression for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8935#issuecomment-2978728661",
|
||||
DoNotRetry: true,
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1QueryRange(t, `increase(foo[1m])`, at.QueryOpts{
|
||||
Start: "2025-06-16T21:28:40.700Z",
|
||||
End: "2025-06-16T21:29:30.700Z",
|
||||
Step: "9s",
|
||||
MaxLookback: "65s",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
Result: []*at.QueryResult{
|
||||
{
|
||||
Metric: map[string]string{},
|
||||
Samples: []*at.Sample{
|
||||
at.NewSample(t, "2025-06-16T21:28:40.700Z", 0),
|
||||
at.NewSample(t, "2025-06-16T21:28:49.700Z", 0),
|
||||
at.NewSample(t, "2025-06-16T21:28:58.700Z", 1),
|
||||
at.NewSample(t, "2025-06-16T21:29:07.700Z", 1),
|
||||
at.NewSample(t, "2025-06-16T21:29:16.700Z", 0),
|
||||
at.NewSample(t, "2025-06-16T21:29:25.700Z", 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func testComparisonNotInfNotNan(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testComparisonNotInfNotNan(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// comparison-not-inf-not-nan
|
||||
@@ -306,13 +259,13 @@ func testComparisonNotInfNotNan(tc *at.TestCase, sut at.PrometheusWriteQuerier)
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1QueryRange(t, `1/(not_nan_not_inf-1)!=inf!=nan`, at.QueryOpts{
|
||||
return sut.APIV1QueryRange(t, `1/(not_nan_not_inf-1)!=inf!=nan`, at.QueryOpts{
|
||||
Start: "2024-02-05T06:50:36.000Z",
|
||||
End: "2024-02-05T09:58:37.000Z",
|
||||
Step: "60",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -329,7 +282,7 @@ func testComparisonNotInfNotNan(tc *at.TestCase, sut at.PrometheusWriteQuerier)
|
||||
})
|
||||
}
|
||||
|
||||
func testEmptyLabelMatch(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testEmptyLabelMatch(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// empty-label-match
|
||||
@@ -352,13 +305,13 @@ func testEmptyLabelMatch(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1QueryRange(t, `empty_label_match{foo=~'bar|'}`, at.QueryOpts{
|
||||
return sut.APIV1QueryRange(t, `empty_label_match{foo=~'bar|'}`, at.QueryOpts{
|
||||
Start: "2024-02-05T08:55:36.000Z",
|
||||
End: "2024-02-05T08:57:36.000Z",
|
||||
Step: "60s",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -381,7 +334,7 @@ func testEmptyLabelMatch(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
})
|
||||
}
|
||||
|
||||
func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testMaxLookbehind(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// max_lookback_set
|
||||
@@ -405,14 +358,14 @@ func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1QueryRange(t, `max_lookback_set{foo=~'bar|'}`, at.QueryOpts{
|
||||
return sut.APIV1QueryRange(t, `max_lookback_set{foo=~'bar|'}`, at.QueryOpts{
|
||||
Start: "2024-02-05T08:55:06.000Z",
|
||||
End: "2024-02-05T08:57:37.000Z",
|
||||
Step: "10s",
|
||||
MaxLookback: "1s",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -452,13 +405,13 @@ func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1QueryRange(t, `max_lookback_unset{foo=~'bar|'}`, at.QueryOpts{
|
||||
return sut.APIV1QueryRange(t, `max_lookback_unset{foo=~'bar|'}`, at.QueryOpts{
|
||||
Start: "2024-02-05T08:55:06.000Z",
|
||||
End: "2024-02-05T08:57:37.000Z",
|
||||
Step: "10s",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -489,7 +442,7 @@ func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
})
|
||||
}
|
||||
|
||||
func testNonNanAsMissingData(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testNonNanAsMissingData(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// not-nan-as-missing-data
|
||||
@@ -513,13 +466,13 @@ func testNonNanAsMissingData(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query_range response",
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1QueryRange(t, `not_nan_as_missing_data>1`, at.QueryOpts{
|
||||
return sut.APIV1QueryRange(t, `not_nan_as_missing_data>1`, at.QueryOpts{
|
||||
Start: "2024-02-05T08:57:34.000Z",
|
||||
End: "2024-02-05T08:57:36.000Z",
|
||||
Step: "1s",
|
||||
})
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "matrix",
|
||||
@@ -544,7 +497,7 @@ func testNonNanAsMissingData(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
})
|
||||
}
|
||||
|
||||
func testSubqueryAggregation(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
func testSubqueryAggregation(tc *at.TestCase, sut at.WriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// subquery-aggregation
|
||||
@@ -568,14 +521,14 @@ func testSubqueryAggregation(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
Got: func() any {
|
||||
got := sut.PrometheusAPIV1Query(t, `min by (item) (min_over_time(forms_daily_count[10m:1m]))`, at.QueryOpts{
|
||||
got := sut.APIV1Query(t, `min by (item) (min_over_time(forms_daily_count[10m:1m]))`, at.QueryOpts{
|
||||
Time: "2024-02-05T08:56:35.000Z",
|
||||
LatencyOffset: "1ms",
|
||||
})
|
||||
got.Sort()
|
||||
return got
|
||||
},
|
||||
Want: &at.PrometheusAPIV1QueryResponse{
|
||||
Want: &at.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &at.QueryData{
|
||||
ResultType: "vector",
|
||||
@@ -594,7 +547,7 @@ func testSubqueryAggregation(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
|
||||
})
|
||||
}
|
||||
|
||||
func getRowsInsertedTotal(t *testing.T, sut at.PrometheusWriteQuerier) int {
|
||||
func getRowsInsertedTotal(t *testing.T, sut at.WriteQuerier) int {
|
||||
t.Helper()
|
||||
|
||||
selector := `vm_rows_inserted_total{type="graphite"}`
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
)
|
||||
|
||||
// TestSingleVlagentRemoteWrite performs tests for remote write data ingestion
|
||||
// by vlagent application
|
||||
func TestSingleVlagentRemoteWrite(t *testing.T) {
|
||||
os.RemoveAll(t.Name())
|
||||
tc := at.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
// test data ingestion into
|
||||
const instance = "vlsingle"
|
||||
const r1Port = "50425"
|
||||
sutFlags := []string{
|
||||
"-httpListenAddr=127.0.0.1:" + r1Port,
|
||||
"-storageDataPath=" + tc.Dir() + "/" + instance,
|
||||
"-retentionPeriod=100y",
|
||||
}
|
||||
|
||||
sut := tc.MustStartVlsingle(instance, sutFlags)
|
||||
remoteWriteURL := fmt.Sprintf("http://%s/internal/insert", sut.HTTPAddr())
|
||||
|
||||
vlagent := tc.MustStartDefaultVlagent([]string{remoteWriteURL})
|
||||
vlagent.JSONLineWrite(t, []string{
|
||||
`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
|
||||
`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
|
||||
}, at.QueryOptsLogs{})
|
||||
|
||||
sut.ForceFlush(t)
|
||||
got := sut.LogsQLQuery(t, "ingest jsonline", at.QueryOptsLogs{})
|
||||
wantLogLines := []string{
|
||||
`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
|
||||
`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
|
||||
}
|
||||
assertLogsQLResponseEqual(t, got, &at.LogsQLQueryResponse{LogLines: wantLogLines})
|
||||
|
||||
// stop log storage and check data buffering works correctly
|
||||
tc.StopApp(instance)
|
||||
|
||||
// ingest some data vlagent must hold it in memory
|
||||
vlagent.JSONLineWrite(t, []string{
|
||||
`{"_msg":"ingest jsonline2","_time": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
|
||||
`{"_msg":"ingest jsonline2","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
|
||||
}, at.QueryOptsLogs{})
|
||||
|
||||
vlagent.WaitQueueEmptyAfter(t, func() {
|
||||
// start storage and check if buffered data correctly ingested
|
||||
sut = tc.MustStartVlsingle(instance, sutFlags)
|
||||
})
|
||||
|
||||
sut.ForceFlush(t)
|
||||
got = sut.LogsQLQuery(t, "ingest jsonline2", at.QueryOptsLogs{})
|
||||
wantLogLines = []string{
|
||||
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
|
||||
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
|
||||
}
|
||||
assertLogsQLResponseEqual(t, got, &at.LogsQLQueryResponse{LogLines: wantLogLines})
|
||||
}
|
||||
|
||||
func TestSingleVlagentRemoteWriteReplication(t *testing.T) {
|
||||
os.RemoveAll(t.Name())
|
||||
tc := at.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
const (
|
||||
instanceReplica0 = "vlsingle-0"
|
||||
vlsinglePortR0 = "53541"
|
||||
instanceReplica1 = "vlsingle-1"
|
||||
vlsinglePortR1 = "53124"
|
||||
vlagentInstance = "vlagent"
|
||||
)
|
||||
sutFlagsR0 := []string{
|
||||
"-httpListenAddr=127.0.0.1:" + vlsinglePortR0,
|
||||
"-storageDataPath=" + path.Join(tc.Dir(), instanceReplica0),
|
||||
"-retentionPeriod=100y",
|
||||
}
|
||||
sutFlagsR1 := []string{
|
||||
"-httpListenAddr=127.0.0.1:" + vlsinglePortR1,
|
||||
"-storageDataPath=" + path.Join(tc.Dir(), instanceReplica1),
|
||||
"-retentionPeriod=100y",
|
||||
}
|
||||
|
||||
sutR0 := tc.MustStartVlsingle(instanceReplica0, sutFlagsR0)
|
||||
sutR1 := tc.MustStartVlsingle(instanceReplica1, sutFlagsR1)
|
||||
|
||||
vlagentRemoteWriteURLs := []string{
|
||||
fmt.Sprintf("http://%s/internal/insert", sutR0.HTTPAddr()),
|
||||
fmt.Sprintf("http://%s/internal/insert", sutR1.HTTPAddr()),
|
||||
}
|
||||
vlagentFlags := []string{
|
||||
"-remoteWrite.tmpDataPath=" + fmt.Sprintf("%s/%s-%d", os.TempDir(), vlagentInstance, time.Now().UnixNano()),
|
||||
}
|
||||
vlagent := tc.MustStartVlagent(vlagentInstance, vlagentRemoteWriteURLs, vlagentFlags)
|
||||
|
||||
// ingest data and check if it properly replicated to the vlsingles
|
||||
vlagent.JSONLineWrite(t, []string{
|
||||
`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
|
||||
`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
|
||||
}, at.QueryOptsLogs{})
|
||||
|
||||
wantLogLines := []string{
|
||||
`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
|
||||
`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
|
||||
}
|
||||
|
||||
sutR0.ForceFlush(t)
|
||||
gotR0 := sutR0.LogsQLQuery(t, "ingest jsonline", at.QueryOptsLogs{})
|
||||
assertLogsQLResponseEqual(t, gotR0, &at.LogsQLQueryResponse{LogLines: wantLogLines})
|
||||
|
||||
sutR1.ForceFlush(t)
|
||||
gotR1 := sutR1.LogsQLQuery(t, "ingest jsonline", at.QueryOptsLogs{})
|
||||
assertLogsQLResponseEqual(t, gotR1, &at.LogsQLQueryResponse{LogLines: wantLogLines})
|
||||
|
||||
// stop log storage and check data buffering works correctly at vlagent
|
||||
tc.StopApp(instanceReplica0)
|
||||
|
||||
// ingest some data vlagent must hold it in memory
|
||||
vlagent.JSONLineWrite(t, []string{
|
||||
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
|
||||
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
|
||||
}, at.QueryOptsLogs{})
|
||||
|
||||
// check alive storage received data
|
||||
wantLogLines = []string{
|
||||
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
|
||||
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
|
||||
}
|
||||
|
||||
sutR1.ForceFlush(t)
|
||||
gotR1 = sutR1.LogsQLQuery(t, "ingest jsonline2", at.QueryOptsLogs{})
|
||||
assertLogsQLResponseEqual(t, gotR1, &at.LogsQLQueryResponse{LogLines: wantLogLines})
|
||||
|
||||
// stop vmagent, it must buffer data on-disk
|
||||
tc.StopApp(vlagentInstance)
|
||||
|
||||
vlagent = tc.MustStartVlagent(vlagentInstance, vlagentRemoteWriteURLs, vlagentFlags)
|
||||
vlagent.WaitQueueEmptyAfter(t, func() {
|
||||
// start storage and check if buffered data correctly ingested
|
||||
sutR0 = tc.MustStartVlsingle(instanceReplica0, sutFlagsR0)
|
||||
})
|
||||
|
||||
sutR0.ForceFlush(t)
|
||||
gotR0 = sutR0.LogsQLQuery(t, "ingest jsonline2", at.QueryOptsLogs{})
|
||||
assertLogsQLResponseEqual(t, gotR0, &at.LogsQLQueryResponse{LogLines: wantLogLines})
|
||||
}
|
||||
@@ -45,12 +45,12 @@ func TestSingleVMAgentReloadConfigs(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: `unexpected metrics stored on vmagent remote write`,
|
||||
Got: func() any {
|
||||
return vmsingle.PrometheusAPIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
|
||||
return vmsingle.APIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
|
||||
Start: "2022-05-10T00:00:00Z",
|
||||
End: "2022-05-10T23:59:59Z",
|
||||
}).Sort()
|
||||
},
|
||||
Want: &at.PrometheusAPIV1SeriesResponse{
|
||||
Want: &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
Data: []map[string]string{{"__name__": "foo_bar", "label1": "value1"}},
|
||||
},
|
||||
@@ -76,12 +76,12 @@ func TestSingleVMAgentReloadConfigs(t *testing.T) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: `unexpected metrics stored on vmagent remote write`,
|
||||
Got: func() any {
|
||||
return vmsingle.PrometheusAPIV1Series(t, `{__name__="bar_foo"}`, at.QueryOpts{
|
||||
return vmsingle.APIV1Series(t, `{__name__="bar_foo"}`, at.QueryOpts{
|
||||
Start: "2022-05-10T00:00:00Z",
|
||||
End: "2022-05-10T23:59:59Z",
|
||||
}).Sort()
|
||||
},
|
||||
Want: &at.PrometheusAPIV1SeriesResponse{
|
||||
Want: &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
Data: []map[string]string{{"__name__": "bar_foo", "label1": "value2"}},
|
||||
},
|
||||
@@ -122,12 +122,12 @@ func testSingleVMAgentRemoteWrite(t *testing.T, forcePromProto bool) {
|
||||
tc.Assert(&at.AssertOptions{
|
||||
Msg: `unexpected metrics stored on vmagent remote write`,
|
||||
Got: func() any {
|
||||
return vmsingle.PrometheusAPIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
|
||||
return vmsingle.APIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
|
||||
Start: "2022-05-10T00:00:00Z",
|
||||
End: "2022-05-10T23:59:59Z",
|
||||
}).Sort()
|
||||
},
|
||||
Want: &at.PrometheusAPIV1SeriesResponse{
|
||||
Want: &at.APIV1SeriesResponse{
|
||||
Status: "success",
|
||||
Data: []map[string]string{{"__name__": "foo_bar"}},
|
||||
},
|
||||
|
||||
@@ -77,19 +77,19 @@ func TestClusterTenantsToTenantsVmctlNativeProtocol(t *testing.T) {
|
||||
testVmctlNativeProtocol(tc, clusterSrc, clusterDst, flags)
|
||||
}
|
||||
|
||||
func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.PrometheusWriteQuerier, dstSut apptest.PrometheusWriteQuerier, vmctlFlags []string) {
|
||||
func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.WriteQuerier, dstSut apptest.WriteQuerier, vmctlFlags []string) {
|
||||
t := tc.T()
|
||||
t.Helper()
|
||||
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
|
||||
// test for empty data request in the source
|
||||
got := srcSut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
got := srcSut.APIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: "2025-05-30T12:45:00Z",
|
||||
})
|
||||
|
||||
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
@@ -116,14 +116,14 @@ func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.PrometheusWrit
|
||||
dataSet[i] = fmt.Sprintf("%s %d %s", metricsName, i, ingestTimestamp)
|
||||
}
|
||||
|
||||
wantResponse := apptest.PrometheusAPIV1QueryResponse{
|
||||
wantResponse := apptest.APIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &expectedQueryData,
|
||||
}
|
||||
|
||||
wantResponse.Sort()
|
||||
|
||||
srcSut.PrometheusAPIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{})
|
||||
srcSut.APIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{})
|
||||
srcSut.ForceFlush(t)
|
||||
|
||||
tc.MustStartVmctl("vmctl", vmctlFlags)
|
||||
@@ -134,7 +134,7 @@ func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.PrometheusWrit
|
||||
Retries: 300,
|
||||
Msg: `unexpected metrics stored on vmsingle via the native protocol`,
|
||||
Got: func() any {
|
||||
exported := dstSut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
exported := dstSut.APIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
Start: "2025-05-30T16:39:36Z",
|
||||
End: "2025-05-30T16:39:37Z",
|
||||
})
|
||||
@@ -143,7 +143,7 @@ func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.PrometheusWrit
|
||||
},
|
||||
Want: wantResponse.Data.Result,
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -55,19 +55,19 @@ func TestClusterVmctlPrometheusProtocol(t *testing.T) {
|
||||
testPrometheusProtocol(tc, cluster, vmctlFlags)
|
||||
}
|
||||
|
||||
func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, vmctlFlags []string) {
|
||||
func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.WriteQuerier, vmctlFlags []string) {
|
||||
t := tc.T()
|
||||
t.Helper()
|
||||
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
|
||||
// test for empty data request
|
||||
got := sut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
got := sut.APIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: "2025-06-02T17:14:00Z",
|
||||
})
|
||||
|
||||
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
@@ -88,7 +88,7 @@ func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
|
||||
t.Fatalf("cannot read expected series response file: %s", err)
|
||||
}
|
||||
|
||||
var wantResponse apptest.PrometheusAPIV1QueryResponse
|
||||
var wantResponse apptest.APIV1QueryResponse
|
||||
if err := json.Unmarshal(bytes, &wantResponse); err != nil {
|
||||
t.Fatalf("cannot unmarshal expected series response file: %s", err)
|
||||
}
|
||||
@@ -99,7 +99,7 @@ func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
|
||||
Retries: 300,
|
||||
Msg: `unexpected metrics stored on vmsingle via the prometheus protocol`,
|
||||
Got: func() any {
|
||||
expected := sut.PrometheusAPIV1Export(t, `{__name__="vm_log_messages_total", location=~"VictoriaMetrics/lib/ingestserver/opentsdb/server.go:(48|59)"}`, apptest.QueryOpts{
|
||||
expected := sut.APIV1Export(t, `{__name__="vm_log_messages_total", location=~"VictoriaMetrics/lib/ingestserver/opentsdb/server.go:(48|59)"}`, apptest.QueryOpts{
|
||||
Start: "2025-06-02T00:00:00Z",
|
||||
End: "2025-06-02T23:59:59Z",
|
||||
})
|
||||
@@ -108,7 +108,7 @@ func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
|
||||
},
|
||||
Want: wantResponse.Data.Result,
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestClusterVmctlRemoteReadProtocol(t *testing.T) {
|
||||
testRemoteReadProtocol(tc, clusterDst, newRemoteReadServer, vmctlFlags)
|
||||
}
|
||||
|
||||
func testRemoteReadProtocol(tc *at.TestCase, sut at.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
|
||||
func testRemoteReadProtocol(tc *at.TestCase, sut at.WriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
|
||||
t := tc.T()
|
||||
t.Helper()
|
||||
|
||||
@@ -84,14 +84,14 @@ func testRemoteReadProtocol(tc *at.TestCase, sut at.PrometheusWriteQuerier, newR
|
||||
|
||||
expectedResult := transformSeriesToQueryResult(rrs.storage.store)
|
||||
|
||||
cmpOpt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
cmpOpt := cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
// test for empty data request
|
||||
got := sut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
got := sut.APIV1Query(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: "2025-06-02T17:14:00Z",
|
||||
})
|
||||
|
||||
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
want := at.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
@@ -106,7 +106,7 @@ func testRemoteReadProtocol(tc *at.TestCase, sut at.PrometheusWriteQuerier, newR
|
||||
Retries: 300,
|
||||
Msg: `unexpected metrics stored on vmsingle via the prometheus protocol`,
|
||||
Got: func() any {
|
||||
expected := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
expected := sut.APIV1Export(t, `{__name__=~".*"}`, at.QueryOpts{
|
||||
Start: "2025-06-11T15:31:10Z",
|
||||
End: "2025-06-11T15:32:20Z",
|
||||
})
|
||||
@@ -115,7 +115,7 @@ func testRemoteReadProtocol(tc *at.TestCase, sut at.PrometheusWriteQuerier, newR
|
||||
},
|
||||
Want: expectedResult,
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
package apptest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Vlagent holds the state of a vlagent app and provides vlagent-specific functions
|
||||
type Vlagent struct {
|
||||
*app
|
||||
*ServesMetrics
|
||||
|
||||
remoteStoragesCount int
|
||||
httpListenAddr string
|
||||
}
|
||||
|
||||
// StartVlagent starts an instance of vlagent with the given flags.
|
||||
// It also sets the default flags and populates the app instance state with runtime
|
||||
// values extracted from the application log (such as httpListenAddr)
|
||||
func StartVlagent(instance string, remoteWriteURLs []string, flags []string, cli *Client) (*Vlagent, error) {
|
||||
extractREs := []*regexp.Regexp{
|
||||
httpListenAddrRE,
|
||||
}
|
||||
|
||||
app, stderrExtracts, err := startApp(instance, "../../bin/vlagent", flags, &appOptions{
|
||||
defaultFlags: map[string]string{
|
||||
"-httpListenAddr": "127.0.0.1:0",
|
||||
"-remoteWrite.url": strings.Join(remoteWriteURLs, ","),
|
||||
"-remoteWrite.tmpDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
|
||||
"-remoteWrite.flushInterval": "10ms",
|
||||
"-remoteWrite.showURL": "true",
|
||||
},
|
||||
extractREs: extractREs,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Vlagent{
|
||||
app: app,
|
||||
remoteStoragesCount: len(remoteWriteURLs),
|
||||
ServesMetrics: &ServesMetrics{
|
||||
metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
|
||||
cli: cli,
|
||||
},
|
||||
httpListenAddr: stderrExtracts[0],
|
||||
}, nil
|
||||
}
|
||||
|
||||
// JSONLineWrite is a test helper function that inserts a
|
||||
// collection of records in json line format by sending a HTTP
|
||||
// POST request to /insert/jsonline vlagent endpoint.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api
|
||||
func (app *Vlagent) JSONLineWrite(t *testing.T, records []string, opts QueryOptsLogs) {
|
||||
t.Helper()
|
||||
|
||||
data := []byte(strings.Join(records, "\n"))
|
||||
|
||||
url := fmt.Sprintf("http://%s/insert/jsonline", app.httpListenAddr)
|
||||
uv := opts.asURLValues()
|
||||
uvs := uv.Encode()
|
||||
if len(uvs) > 0 {
|
||||
url += "?" + uvs
|
||||
}
|
||||
app.sendBlocking(t, len(records), func() {
|
||||
_, statusCode := app.cli.Post(t, url, "text/plain", data)
|
||||
if statusCode != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// WaitQueueEmptyAfter checks that persistent queue is empty
|
||||
// after execution of provided callback
|
||||
func (app *Vlagent) WaitQueueEmptyAfter(t *testing.T, cb func()) {
|
||||
t.Helper()
|
||||
const (
|
||||
retries = 70
|
||||
period = 100 * time.Millisecond
|
||||
)
|
||||
// vlagent_remotewrite_blocks_sent_total
|
||||
// take in account data replication
|
||||
blocksSent := app.remoteWriteBlocksSent(t)
|
||||
cb()
|
||||
for range retries {
|
||||
if app.remoteWriteBlocksSent(t) > blocksSent && app.persistentQueueSize(t) == 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(period)
|
||||
}
|
||||
t.Fatalf("timed out while waiting for inserted logs to be flushed to remote storage")
|
||||
|
||||
}
|
||||
|
||||
// sendBlocking sends the data to remote write url by executing `send` function and
|
||||
// waits until the data is actually sent.
|
||||
//
|
||||
// vlagent does not send the data immediately. It first puts the data into a
|
||||
// buffer. Then a background goroutine takes the data from the buffer sends it
|
||||
// to the vmstorage. This happens every 1s by default.
|
||||
//
|
||||
// Waiting is implemented a retrieving the value of `vlagent_remotewrite_block_size_rows_sum`
|
||||
// metric and checking whether it is equal or greater than the wanted value.
|
||||
// If it is, then the data has been sent to remote storage.
|
||||
//
|
||||
// Unreliable if the records are inserted concurrently.
|
||||
func (app *Vlagent) sendBlocking(t *testing.T, numRecordsToSend int, send func()) {
|
||||
t.Helper()
|
||||
|
||||
send()
|
||||
|
||||
const (
|
||||
retries = 50
|
||||
period = 100 * time.Millisecond
|
||||
)
|
||||
// take in account data replication
|
||||
wantRowsSentCount := app.remoteWriteRowsPushed(t) + numRecordsToSend*app.remoteStoragesCount
|
||||
for range retries {
|
||||
if app.remoteWriteRowsPushed(t) >= wantRowsSentCount {
|
||||
return
|
||||
}
|
||||
time.Sleep(period)
|
||||
}
|
||||
t.Fatalf("timed out while waiting for inserted rows to be sent to remote storage")
|
||||
}
|
||||
|
||||
func (app *Vlagent) remoteWriteBlocksSent(t *testing.T) int {
|
||||
total := 0.0
|
||||
for _, v := range app.GetMetricsByPrefix(t, "vlagent_remotewrite_blocks_sent_total") {
|
||||
total += v
|
||||
}
|
||||
return int(total)
|
||||
}
|
||||
|
||||
func (app *Vlagent) remoteWriteRowsPushed(t *testing.T) int {
|
||||
total := 0.0
|
||||
// vlagent_remotewrite_blocks_sent_total
|
||||
for _, v := range app.GetMetricsByPrefix(t, "vlagent_remotewrite_block_size_rows_sum") {
|
||||
total += v
|
||||
}
|
||||
return int(total)
|
||||
}
|
||||
|
||||
func (app *Vlagent) persistentQueueSize(t *testing.T) int {
|
||||
total := 0.0
|
||||
for _, v := range app.GetMetricsByPrefix(t, "vlagent_remotewrite_pending_data_bytes") {
|
||||
total += v
|
||||
}
|
||||
for _, v := range app.GetMetricsByPrefix(t, "vlagent_remotewrite_pending_inmemory_blocks") {
|
||||
total += v
|
||||
}
|
||||
return int(total)
|
||||
}
|
||||
@@ -127,12 +127,12 @@ func (app *Vminsert) GraphiteWrite(t *testing.T, records []string, _ QueryOpts)
|
||||
app.cli.Write(t, app.graphiteListenAddr, records)
|
||||
}
|
||||
|
||||
// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
|
||||
// APIV1ImportCSV is a test helper function that inserts a collection
|
||||
// of records in CSV format for the given tenant by sending an HTTP POST
|
||||
// request to prometheus/api/v1/import/csv vminsert endpoint.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
|
||||
func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
|
||||
func (app *Vminsert) APIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
|
||||
t.Helper()
|
||||
|
||||
url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/csv", app.httpListenAddr, opts.getTenant())
|
||||
@@ -150,12 +150,12 @@ func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, op
|
||||
})
|
||||
}
|
||||
|
||||
// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
|
||||
// APIV1ImportNative is a test helper function that inserts a collection
|
||||
// of records in Native format for the given tenant by sending an HTTP POST
|
||||
// request to prometheus/api/v1/import/native vminsert endpoint.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
|
||||
func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
|
||||
func (app *Vminsert) APIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
|
||||
t.Helper()
|
||||
|
||||
url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/native", app.httpListenAddr, opts.getTenant())
|
||||
@@ -195,10 +195,10 @@ func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOp
|
||||
})
|
||||
}
|
||||
|
||||
// PrometheusAPIV1Write is a test helper function that inserts a
|
||||
// APIV1Write is a test helper function that inserts a
|
||||
// collection of records in Prometheus remote-write format by sending a HTTP
|
||||
// POST request to /prometheus/api/v1/write vminsert endpoint.
|
||||
func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts) {
|
||||
func (app *Vminsert) APIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts) {
|
||||
t.Helper()
|
||||
|
||||
url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.getTenant())
|
||||
@@ -212,13 +212,13 @@ func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries,
|
||||
})
|
||||
}
|
||||
|
||||
// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
|
||||
// APIV1ImportPrometheus is a test helper function that inserts a
|
||||
// collection of records in Prometheus text exposition format for the given
|
||||
// tenant by sending a HTTP POST request to
|
||||
// /prometheus/api/v1/import/prometheus vminsert endpoint.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
|
||||
func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
|
||||
func (app *Vminsert) APIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
|
||||
t.Helper()
|
||||
|
||||
url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, opts.getTenant())
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user