Compare commits

..

2 Commits

Author SHA1 Message Date
dmitryk-dk
ba0eb6f71d apptest: rename prometheus from stopwritequerirer 2025-06-19 12:20:07 +02:00
dmitryk-dk
f5bd54842a apptest: update tests model by removing Prometheus prefix
apptest: make linter happy
2025-06-19 12:18:11 +02:00
303 changed files with 7546 additions and 13546 deletions

1
.gitignore vendored
View File

@@ -12,7 +12,6 @@
/victoria-logs-data
/victoria-metrics-data
/vmagent-remotewrite-data
/vlagent-remotewritewrite
/vmstorage-data
/vmselect-cache
/package/temp-deb-*

View File

@@ -545,7 +545,7 @@ test-full:
test-full-386:
GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore victoria-logs vlagent
integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore victoria-logs
go test ./apptest/... -skip="^TestCluster.*"
benchmark:

View File

@@ -3,12 +3,12 @@
![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)
![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)
![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)
[![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml)
![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)
![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)
![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)
![Slack](https://img.shields.io/badge/Join-4A154B?logo=slack&link=https%3A%2F%2Fslack.victoriametrics.com)
[![X](https://img.shields.io/twitter/follow/VictoriaMetrics?style=flat&label=Follow&color=black&logo=x&labelColor=black&link=https%3A%2F%2Fx.com%2FVictoriaMetrics)](https://x.com/VictoriaMetrics/)
[![Reddit](https://img.shields.io/reddit/subreddit-subscribers/VictoriaMetrics?style=flat&label=Join&labelColor=red&logoColor=white&logo=reddit&link=https%3A%2F%2Fwww.reddit.com%2Fr%2FVictoriaMetrics)](https://www.reddit.com/r/VictoriaMetrics/)
![X](https://img.shields.io/twitter/follow/VictoriaMetrics?style=flat&label=Follow&color=black&logo=x&labelColor=black&link=https%3A%2F%2Fx.com%2FVictoriaMetrics)
![Reddit](https://img.shields.io/reddit/subreddit-subscribers/VictoriaMetrics?style=flat&label=Join&labelColor=red&logoColor=white&logo=reddit&link=https%3A%2F%2Fwww.reddit.com%2Fr%2FVictoriaMetrics)
<picture>
<source srcset="docs/victoriametrics/logo_white.webp" media="(prefers-color-scheme: dark)">

View File

@@ -1,106 +0,0 @@
# All these commands must run from repository root.
vlagent:
APP_NAME=vlagent $(MAKE) app-local
vlagent-race:
APP_NAME=vlagent RACE=-race $(MAKE) app-local
vlagent-prod:
APP_NAME=vlagent $(MAKE) app-via-docker
vlagent-pure-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-pure
vlagent-linux-amd64-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-linux-amd64
vlagent-linux-arm-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-linux-arm
vlagent-linux-arm64-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-linux-arm64
vlagent-linux-ppc64le-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-linux-ppc64le
vlagent-linux-386-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-linux-386
vlagent-darwin-amd64-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-darwin-amd64
vlagent-darwin-arm64-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-darwin-arm64
vlagent-freebsd-amd64-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-freebsd-amd64
vlagent-openbsd-amd64-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-openbsd-amd64
vlagent-windows-amd64-prod:
APP_NAME=vlagent $(MAKE) app-via-docker-windows-amd64
package-vlagent:
APP_NAME=vlagent $(MAKE) package-via-docker
package-vlagent-pure:
APP_NAME=vlagent $(MAKE) package-via-docker-pure
package-vlagent-amd64:
APP_NAME=vlagent $(MAKE) package-via-docker-amd64
package-vlagent-arm:
APP_NAME=vlagent $(MAKE) package-via-docker-arm
package-vlagent-arm64:
APP_NAME=vlagent $(MAKE) package-via-docker-arm64
package-vlagent-ppc64le:
APP_NAME=vlagent $(MAKE) package-via-docker-ppc64le
package-vlagent-386:
APP_NAME=vlagent $(MAKE) package-via-docker-386
publish-vlagent:
APP_NAME=vlagent $(MAKE) publish-via-docker
vlagent-linux-amd64:
APP_NAME=vlagent CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
vlagent-linux-arm:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
vlagent-linux-arm64:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
vlagent-linux-ppc64le:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
vlagent-linux-s390x:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
vlagent-linux-loong64:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
vlagent-linux-386:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
vlagent-darwin-amd64:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
vlagent-darwin-arm64:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
vlagent-freebsd-amd64:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
vlagent-openbsd-amd64:
APP_NAME=vlagent CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
vlagent-windows-amd64:
GOARCH=amd64 APP_NAME=vlagent $(MAKE) app-local-windows-goarch
vlagent-pure:
APP_NAME=vlagent $(MAKE) app-local-pure

View File

@@ -1,3 +0,0 @@
See vlagent docs [here](https://docs.victoriametrics.com/victorialogs/vlagent/).
vlagent docs can be edited at [docs/vlagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victorialogs/vlagent.md).

View File

@@ -1,8 +0,0 @@
ARG base_image=non-existing
FROM $base_image
EXPOSE 9429
ENTRYPOINT ["/vlagent-prod"]
ARG src_binary=non-existing
COPY $src_binary ./vlagent-prod

View File

@@ -1,97 +0,0 @@
package main
import (
"flag"
"fmt"
"net/http"
"os"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlagent/remotewrite"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
)
var (
httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vlagent instances on the same server. "+
"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
)
func main() {
// Write flags and help message to stdout, since it is easier to grep or pipe.
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage = usage
envflag.Parse()
buildinfo.Init()
remotewrite.InitSecretFlags()
logger.Init()
remotewrite.Init()
vlinsert.Init()
insertutil.SetLogRowsStorage(&remotewrite.Storage{})
listenAddrs := *httpListenAddrs
if len(listenAddrs) == 0 {
listenAddrs = []string{":9429"}
}
logger.Infof("starting vlagent at %q...", listenAddrs)
startTime := time.Now()
go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
UseProxyProtocol: useProxyProtocol,
})
logger.Infof("started vlagent in %.3f seconds", time.Since(startTime).Seconds())
pushmetrics.Init()
sig := procutil.WaitForSigterm()
logger.Infof("received signal %s", sig)
pushmetrics.Stop()
startTime = time.Now()
logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
if err := httpserver.Stop(listenAddrs); err != nil {
logger.Fatalf("cannot stop the webservice: %s", err)
}
vlinsert.Stop()
remotewrite.Stop()
logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
logger.Infof("successfully stopped vlagent in %.3f seconds", time.Since(startTime).Seconds())
}
// RequestHandler handles insert requests for VictoriaLogs
func requestHandler(w http.ResponseWriter, r *http.Request) bool {
if r.URL.Path == "/" {
if r.Method != http.MethodGet {
return false
}
w.Header().Add("Content-Type", "text/html; charset=utf-8")
fmt.Fprintf(w, "<h2>vlagent</h2>")
fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victorialogs/vlagent/'>https://docs.victoriametrics.com/victorialogs/vlagent/</a></br>")
fmt.Fprintf(w, "Useful endpoints:</br>")
httpserver.WriteAPIHelp(w, [][2]string{
{"metrics", "available service metrics"},
{"flags", "command-line flags"},
})
return true
}
return vlinsert.RequestHandler(w, r)
}
func usage() {
const s = `
vlagent collects logs via popular data ingestion protocols and routes it to VictoriaLogs.
See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ .
`
flagutil.Usage(s)
}

View File

@@ -1,12 +0,0 @@
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
ARG certs_image=non-existing
ARG root_image=non-existing
FROM $certs_image AS certs
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
FROM $root_image
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
EXPOSE 9429
ENTRYPOINT ["/vlagent-prod"]
ARG TARGETARCH
COPY vlagent-linux-${TARGETARCH}-prod ./vlagent-prod

View File

@@ -1,462 +0,0 @@
package remotewrite
import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/VictoriaMetrics/metrics"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
)
var (
rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data ")
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxTime")
retryMaxTime = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
tlsHandshakeTimeout = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
tlsCertFile = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
"to the corresponding -remoteWrite.url")
tlsKeyFile = flagutil.NewArrayString("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
tlsCAFile = flagutil.NewArrayString("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
"By default, system CA is used")
tlsServerName = flagutil.NewArrayString("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
"By default, the server name from -remoteWrite.url is used")
headers = flagutil.NewArrayString("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
basicAuthUsername = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
basicAuthPassword = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
"The file is re-read every second")
bearerToken = flagutil.NewArrayString("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
bearerTokenFile = flagutil.NewArrayString("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
"The token is re-read from the file every second")
oauth2ClientID = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
oauth2ClientSecret = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
oauth2EndpointParams = flagutil.NewArrayString("remoteWrite.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -remoteWrite.url . "+
`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
oauth2TokenURL = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
oauth2Scopes = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
)
type client struct {
sanitizedURL string
remoteWriteURL string
fq *persistentqueue.FastQueue
hc *http.Client
retryMinInterval time.Duration
retryMaxTime time.Duration
sendBlock func(block []byte) bool
authCfg *promauth.Config
rl *ratelimiter.RateLimiter
bytesSent *metrics.Counter
blocksSent *metrics.Counter
requestDuration *metrics.Histogram
requestsOKCount *metrics.Counter
errorsCount *metrics.Counter
packetsDropped *metrics.Counter
rateLimit *metrics.Gauge
retriesCount *metrics.Counter
sendDuration *metrics.FloatCounter
wg sync.WaitGroup
stopCh chan struct{}
}
func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
authCfg, err := getAuthConfig(argIdx)
if err != nil {
logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
}
tr := httputil.NewTransport(false, "vlagent_remotewrite")
tr.TLSHandshakeTimeout = tlsHandshakeTimeout.GetOptionalArg(argIdx)
tr.MaxConnsPerHost = 2 * concurrency
tr.MaxIdleConnsPerHost = 2 * concurrency
tr.IdleConnTimeout = time.Minute
tr.WriteBufferSize = 64 * 1024
pURL := proxyURL.GetOptionalArg(argIdx)
if len(pURL) > 0 {
if !strings.Contains(pURL, "://") {
logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
}
pu, err := url.Parse(pURL)
if err != nil {
logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
}
tr.Proxy = http.ProxyURL(pu)
}
hc := &http.Client{
Transport: authCfg.NewRoundTripper(tr),
Timeout: sendTimeout.GetOptionalArg(argIdx),
}
c := &client{
sanitizedURL: sanitizedURL,
remoteWriteURL: remoteWriteURL,
authCfg: authCfg,
fq: fq,
hc: hc,
retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
retryMaxTime: retryMaxTime.GetOptionalArg(argIdx),
stopCh: make(chan struct{}),
}
c.sendBlock = c.sendBlockHTTP
return c
}
func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
c.rl = ratelimiter.New(int64(bytesPerSec), limitReached, c.stopCh)
}
c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
return float64(rateLimit.GetOptionalArg(argIdx))
})
c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vlagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vlagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
return float64(*queues)
})
for i := 0; i < concurrency; i++ {
c.wg.Add(1)
go func() {
defer c.wg.Done()
c.runWorker()
}()
}
logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
}
func (c *client) MustStop() {
close(c.stopCh)
c.wg.Wait()
logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
}
func getAuthConfig(argIdx int) (*promauth.Config, error) {
headersValue := headers.GetOptionalArg(argIdx)
var hdrs []string
if headersValue != "" {
hdrs = strings.Split(headersValue, "^^")
}
username := basicAuthUsername.GetOptionalArg(argIdx)
password := basicAuthPassword.GetOptionalArg(argIdx)
passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
var basicAuthCfg *promauth.BasicAuthConfig
if username != "" || password != "" || passwordFile != "" {
basicAuthCfg = &promauth.BasicAuthConfig{
Username: username,
Password: promauth.NewSecret(password),
PasswordFile: passwordFile,
}
}
token := bearerToken.GetOptionalArg(argIdx)
tokenFile := bearerTokenFile.GetOptionalArg(argIdx)
var oauth2Cfg *promauth.OAuth2Config
clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
if clientSecretFile != "" || clientSecret != "" {
endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(argIdx)
endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
if err != nil {
return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
}
oauth2Cfg = &promauth.OAuth2Config{
ClientID: oauth2ClientID.GetOptionalArg(argIdx),
ClientSecret: promauth.NewSecret(clientSecret),
ClientSecretFile: clientSecretFile,
EndpointParams: endpointParams,
TokenURL: oauth2TokenURL.GetOptionalArg(argIdx),
Scopes: strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
}
}
tlsCfg := &promauth.TLSConfig{
CAFile: tlsCAFile.GetOptionalArg(argIdx),
CertFile: tlsCertFile.GetOptionalArg(argIdx),
KeyFile: tlsKeyFile.GetOptionalArg(argIdx),
ServerName: tlsServerName.GetOptionalArg(argIdx),
InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
}
opts := &promauth.Options{
BasicAuth: basicAuthCfg,
BearerToken: token,
BearerTokenFile: tokenFile,
OAuth2: oauth2Cfg,
TLSConfig: tlsCfg,
Headers: hdrs,
}
authCfg, err := opts.NewConfig()
if err != nil {
return nil, fmt.Errorf("cannot populate auth config for remoteWrite idx: %d, err: %w", argIdx, err)
}
return authCfg, nil
}
func (c *client) runWorker() {
var ok bool
var block []byte
ch := make(chan bool, 1)
for {
block, ok = c.fq.MustReadBlock(block[:0])
if !ok {
return
}
if len(block) == 0 {
// skip empty data blocks from sending
continue
}
go func() {
startTime := time.Now()
ch <- c.sendBlock(block)
c.sendDuration.Add(time.Since(startTime).Seconds())
}()
select {
case ok := <-ch:
if ok {
// The block has been sent successfully
continue
}
// Return unsent block to the queue.
c.fq.MustWriteBlockIgnoreDisabledPQ(block)
return
case <-c.stopCh:
// c must be stopped. Wait for a while in the hope the block will be sent.
graceDuration := 5 * time.Second
select {
case ok := <-ch:
if !ok {
// Return unsent block to the queue.
c.fq.MustWriteBlockIgnoreDisabledPQ(block)
}
case <-time.After(graceDuration):
// Return unsent block to the queue.
c.fq.MustWriteBlockIgnoreDisabledPQ(block)
}
return
}
}
}
func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
req, err := c.newRequest(url, body)
if err != nil {
return nil, err
}
resp, err := c.hc.Do(req)
if err == nil {
return resp, nil
}
if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
return nil, err
}
// It is likely connection became stale or timed out during the first request.
// Make another attempt in hope request will succeed.
// If not, the error should be handled by the caller as usual.
// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
req, err = c.newRequest(url, body)
if err != nil {
return nil, fmt.Errorf("second attempt: %w", err)
}
resp, err = c.hc.Do(req)
if err != nil {
return nil, fmt.Errorf("second attempt: %w", err)
}
return resp, nil
}
func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
reqBody := bytes.NewBuffer(body)
req, err := http.NewRequest(http.MethodPost, url, reqBody)
if err != nil {
logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
}
err = c.authCfg.SetHeaders(req, true)
if err != nil {
return nil, err
}
h := req.Header
h.Set("User-Agent", "vlagent")
h.Set("Content-Encoding", "zstd")
h.Set("Content-Type", "application/octet-stream")
return req, nil
}
// sendBlockHTTP sends the given block to c.remoteWriteURL.
//
// The function returns false only if c.stopCh is closed.
// Otherwise, it tries sending the block to remote storage indefinitely.
func (c *client) sendBlockHTTP(block []byte) bool {
c.rl.Register(len(block))
maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
retriesCount := 0
again:
startTime := time.Now()
resp, err := c.doRequest(c.remoteWriteURL, block)
c.requestDuration.UpdateDuration(startTime)
if err != nil {
c.errorsCount.Inc()
retryDuration *= 2
if retryDuration > maxRetryDuration {
retryDuration = maxRetryDuration
}
remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
len(block), c.sanitizedURL, err, retryDuration.Seconds())
t := timerpool.Get(retryDuration)
select {
case <-c.stopCh:
timerpool.Put(t)
return false
case <-t.C:
timerpool.Put(t)
}
c.retriesCount.Inc()
goto again
}
statusCode := resp.StatusCode
if statusCode/100 == 2 {
_ = resp.Body.Close()
c.requestsOKCount.Inc()
c.bytesSent.Add(len(block))
c.blocksSent.Inc()
return true
}
metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
if statusCode == 400 || statusCode == 404 {
logBlockRejected(block, c.sanitizedURL, resp)
_ = resp.Body.Close()
c.packetsDropped.Inc()
return true
}
// Unexpected status code returned
retriesCount++
retryAfterHeader := parseRetryAfterHeader(resp.Header.Get("Retry-After"))
retryDuration = getRetryDuration(retryAfterHeader, retryDuration, maxRetryDuration)
// Handle response
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
} else {
logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
"re-sending the block in %.3f seconds", len(block), c.sanitizedURL, retriesCount, statusCode, body, retryDuration.Seconds())
}
t := timerpool.Get(retryDuration)
select {
case <-c.stopCh:
timerpool.Put(t)
return false
case <-t.C:
timerpool.Put(t)
}
c.retriesCount.Inc()
goto again
}
var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
// getRetryDuration returns retry duration.
// retryAfterDuration has the highest priority.
// If retryAfterDuration is not specified, retryDuration gets doubled.
// retryDuration can't exceed maxRetryDuration.
//
// Also see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6097
func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.Duration) time.Duration {
// retryAfterDuration has the highest priority duration
if retryAfterDuration > 0 {
return timeutil.AddJitterToDuration(retryAfterDuration)
}
// default backoff retry policy
retryDuration *= 2
if retryDuration > maxRetryDuration {
retryDuration = maxRetryDuration
}
return retryDuration
}
func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
body, err := io.ReadAll(resp.Body)
if err != nil {
remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
"failed to read response body: %s",
len(block), sanitizedURL, resp.StatusCode, err)
} else {
remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
len(block), sanitizedURL, resp.StatusCode, string(body))
}
}
// parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
// retryAfterString should be in either HTTP-date or a number of seconds.
// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
func parseRetryAfterHeader(retryAfterString string) (retryAfterDuration time.Duration) {
if retryAfterString == "" {
return retryAfterDuration
}
defer func() {
v := retryAfterDuration.Seconds()
logger.Infof("'Retry-After: %s' parsed into %.2f second(s)", retryAfterString, v)
}()
// Retry-After could be in "Mon, 02 Jan 2006 15:04:05 GMT" format.
if parsedTime, err := time.Parse(http.TimeFormat, retryAfterString); err == nil {
return time.Duration(time.Until(parsedTime).Seconds()) * time.Second
}
// Retry-After could be in seconds.
if seconds, err := strconv.Atoi(retryAfterString); err == nil {
return time.Duration(seconds) * time.Second
}
return 0
}

View File

@@ -1,99 +0,0 @@
package remotewrite
import (
"math"
"net/http"
"testing"
"time"
)
func TestCalculateRetryDuration(t *testing.T) {
// `testFunc` call `calculateRetryDuration` for `n` times
// and evaluate if the result of `calculateRetryDuration` is
// 1. >= expectMinDuration
// 2. <= expectMinDuration + 10% (see timeutil.AddJitterToDuration)
f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
t.Helper()
for i := 0; i < n; i++ {
retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
}
expectMaxDuration := helper(expectMinDuration)
expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
if !(retryDuration >= expectMinDuration && retryDuration <= expectMaxDuration) {
t.Fatalf(
"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
retryDuration.Milliseconds(),
)
}
}
// Call calculateRetryDuration for 1 time.
{
// default backoff policy
f(0, time.Second, 1, 2*time.Second)
// default backoff policy exceed max limit"
f(0, 10*time.Minute, 1, time.Minute)
// retry after > default backoff policy
f(10*time.Second, 1*time.Second, 1, 10*time.Second)
// retry after < default backoff policy
f(1*time.Second, 10*time.Second, 1, 1*time.Second)
// retry after invalid and < default backoff policy
f(0, time.Second, 1, 2*time.Second)
}
// Call calculateRetryDuration for multiple times.
{
// default backoff policy 2 times
f(0, time.Second, 2, 4*time.Second)
// default backoff policy 3 times
f(0, time.Second, 3, 8*time.Second)
// default backoff policy N times exceed max limit
f(0, time.Second, 10, time.Minute)
// retry after 120s 1 times
f(120*time.Second, time.Second, 1, 120*time.Second)
// retry after 120s 2 times
f(120*time.Second, time.Second, 2, 120*time.Second)
}
}
func TestParseRetryAfterHeader(t *testing.T) {
f := func(retryAfterString string, expectResult time.Duration) {
t.Helper()
result := parseRetryAfterHeader(retryAfterString)
// expect `expectResult == result` when retryAfterString is in seconds or invalid
// expect the difference between result and expectResult to be lower than 10%
if !(expectResult == result || math.Abs(float64(expectResult-result))/float64(expectResult) < 0.10) {
t.Fatalf(
"incorrect retry after duration, want (ms): %d, got (ms): %d",
expectResult.Milliseconds(), result.Milliseconds(),
)
}
}
// retry after header in seconds
f("10", 10*time.Second)
// retry after header in date time
f(time.Now().Add(30*time.Second).UTC().Format(http.TimeFormat), 30*time.Second)
// retry after header invalid
f("invalid-retry-after", 0)
// retry after header not in GMT
f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
}
// helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
func helper(d time.Duration) time.Duration {
dv := d / 10
if dv > 10*time.Second {
dv = 10 * time.Second
}
return d + dv
}

View File

@@ -1,158 +0,0 @@
package remotewrite
import (
"flag"
"sync"
"sync/atomic"
"time"
"github.com/VictoriaMetrics/metrics"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
)
var (
maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage.")
flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
"This option takes effect only when less than 2MB of data per second are pushed to -remoteWrite.url")
)
type pendingLogs struct {
lastFlushTime atomic.Uint64
// The queue to send blocks to.
fq *persistentqueue.FastQueue
// mu protects wr
mu sync.Mutex
wr writeRequest
stopCh chan struct{}
periodicFlusherWG sync.WaitGroup
}
func newPendingLogs(fq *persistentqueue.FastQueue) *pendingLogs {
pl := &pendingLogs{
fq: fq,
stopCh: make(chan struct{}),
}
pl.periodicFlusherWG.Add(1)
go func() {
defer pl.periodicFlusherWG.Done()
pl.periodicFlusher()
}()
return pl
}
func (pl *pendingLogs) add(lr *logstorage.LogRows) {
lr.ForEachRow(func(_ uint64, r *logstorage.InsertRow) {
pl.addLogRow(r)
})
}
func (pl *pendingLogs) addLogRow(r *logstorage.InsertRow) {
bb := bbPool.Get()
bb.B = r.Marshal(bb.B)
pl.mu.Lock()
_, _ = pl.wr.pendingData.Write(bb.B)
pl.wr.pendingLogRowsCount++
if len(pl.wr.pendingData.B) > maxUnpackedBlockSize.IntN() {
pl.mustFlushLocked()
}
pl.mu.Unlock()
bbPool.Put(bb)
}
func (pl *pendingLogs) mustFlushLocked() {
pl.lastFlushTime.Store(fasttime.UnixTimestamp())
pl.wr.push(func(b []byte) {
if !pl.fq.TryWriteBlock(b) {
logger.Fatalf("BUG: TryWriteBlock cannot return false")
}
})
pl.wr.reset()
}
func (pl *pendingLogs) periodicFlusher() {
flushSeconds := int64(flushInterval.Seconds())
if flushSeconds <= 0 {
flushSeconds = 1
}
d := timeutil.AddJitterToDuration(*flushInterval)
ticker := time.NewTicker(d)
defer ticker.Stop()
for {
select {
case <-pl.stopCh:
pl.mu.Lock()
pl.mustFlushOnStop()
pl.mu.Unlock()
return
case <-ticker.C:
if fasttime.UnixTimestamp()-pl.lastFlushTime.Load() < uint64(flushSeconds) {
continue
}
}
pl.mu.Lock()
pl.mustFlushLocked()
pl.mu.Unlock()
}
}
// mustFlushOnStop force pushes wr data
//
// This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
func (pl *pendingLogs) mustFlushOnStop() {
pl.wr.push(pl.fq.MustWriteBlockIgnoreDisabledPQ)
pl.wr.reset()
}
func (pl *pendingLogs) mustStop() {
close(pl.stopCh)
pl.periodicFlusherWG.Wait()
}
type writeRequest struct {
pendingData bytesutil.ByteBuffer
pendingLogRowsCount int64
}
func (wr *writeRequest) push(pushBlock func([]byte)) {
if len(wr.pendingData.B) == 0 {
return
}
b := wr.pendingData.B
zb := compressBufPool.Get()
zb.B = zstd.CompressLevel(zb.B[:0], b, 1)
zbLen := len(zb.B)
pushBlock(zb.B)
compressBufPool.Put(zb)
blockSizeBytes.Update(float64(zbLen))
blockSizeLogRows.Update(float64(wr.pendingLogRowsCount))
}
func (wr *writeRequest) reset() {
wr.pendingData.Reset()
wr.pendingLogRowsCount = 0
}
var (
blockSizeBytes = metrics.NewHistogram(`vlagent_remotewrite_block_size_bytes`)
blockSizeLogRows = metrics.NewHistogram(`vlagent_remotewrite_block_size_rows`)
)
var (
compressBufPool bytesutil.ByteBufferPool
bbPool bytesutil.ByteBufferPool
)

View File

@@ -1,277 +0,0 @@
package remotewrite
import (
"flag"
"fmt"
"net/url"
"path/filepath"
"sync"
"sync/atomic"
"github.com/VictoriaMetrics/metrics"
"github.com/cespare/xxhash/v2"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
)
var (
remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support VictoriaLogs native protocol. "+
"Example url: http://<victorialogs-host>:9428/internal/insert. "+
"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems.")
maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
"Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. "+
"Disk usage is unlimited if the value is set to 0")
tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vlagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
"See also -remoteWrite.maxDiskUsagePerURL")
queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
"isn't enough for sending high volume of collected data to remote storage. "+
"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
"It is hidden by default, since it can contain sensitive info such as auth key")
)
// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
var rwctxsGlobal []*remoteWriteCtx
// Storage implements insertutil.LogRowsStorage interface
type Storage struct{}
// MustAddRows implements insertutil.LogRowsStorage interface
func (*Storage) MustAddRows(lr *logstorage.LogRows) {
pushToRemoteStorages(lr)
}
// CanWriteData implements insertutil.LogRowsStorage interface
func (*Storage) CanWriteData() error {
return nil
}
// maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
// since it may lead to high memory usage due to big number of buffers.
var maxQueues = cgroup.AvailableCPUs() * 16
const persistentQueueDirname = "persistent-queue"
// InitSecretFlags must be called after flag.Parse and before any logging.
func InitSecretFlags() {
if !*showRemoteWriteURL {
// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
flagutil.RegisterSecretFlag("remoteWrite.url")
}
}
// Init initializes remotewrite.
//
// It must be called after flag.Parse().
//
// Stop must be called for graceful shutdown.
func Init() {
if len(*remoteWriteURLs) == 0 {
logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
}
if *queues > maxQueues {
*queues = maxQueues
}
if *queues <= 0 {
*queues = 1
}
initRemoteWriteCtxs(*remoteWriteURLs)
dropDanglingQueues()
}
// Stop stops remotewrite.
//
// It is expected that nobody calls TryPush during and after the call to this func.
func Stop() {
for _, rwctx := range rwctxsGlobal {
rwctx.mustStop()
}
rwctxsGlobal = nil
}
func dropDanglingQueues() {
// Remove dangling persistent queues, if any.
// This is required for the case when the number of queues has been changed or URL have been changed.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
//
// In case if there were many persistent queues with identical *remoteWriteURLs
// the queue with the last index will be dropped.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
existingQueues := make(map[string]struct{}, len(rwctxsGlobal))
for _, rwctx := range rwctxsGlobal {
existingQueues[rwctx.fq.Dirname()] = struct{}{}
}
queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
files := fs.MustReadDir(queuesDir)
removed := 0
for _, f := range files {
dirname := f.Name()
if _, ok := existingQueues[dirname]; !ok {
logger.Infof("removing dangling queue %q", dirname)
fullPath := filepath.Join(queuesDir, dirname)
fs.MustRemoveAll(fullPath)
removed++
}
}
if removed > 0 {
logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsGlobal))
}
}
func initRemoteWriteCtxs(urls []string) {
if len(urls) == 0 {
logger.Panicf("BUG: urls must be non-empty")
}
maxInmemoryBlocks := memory.Allowed() / len(urls) / 10000
if maxInmemoryBlocks / *queues > 100 {
// There is no much sense in keeping higher number of blocks in memory,
// since this means that the producer outperforms consumer and the queue
// will continue growing. It is better storing the queue to file.
maxInmemoryBlocks = 100 * *queues
}
if maxInmemoryBlocks < 2 {
maxInmemoryBlocks = 2
}
rwctxs := make([]*remoteWriteCtx, len(urls))
rwctxIdx := make([]int, len(urls))
for i, remoteWriteURLRaw := range urls {
remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
if err != nil {
logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
}
sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
if *showRemoteWriteURL {
sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
}
rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
rwctxIdx[i] = i
}
rwctxsGlobal = rwctxs
}
func pushToRemoteStorages(lr *logstorage.LogRows) {
rwctxs := rwctxsGlobal
if len(rwctxs) == 1 {
// fast path
rwctxs[0].push(lr)
return
}
// Push samples to remote storage systems in parallel in order to reduce
// the time needed for sending the data to multiple remote storage systems.
var wg sync.WaitGroup
for _, rwctx := range rwctxs {
wg.Add(1)
go func(rwctx *remoteWriteCtx) {
defer wg.Done()
rwctx.push(lr)
}(rwctx)
}
wg.Wait()
}
type remoteWriteCtx struct {
idx int
fq *persistentqueue.FastQueue
c *client
pls []*pendingLogs
pssNextIdx atomic.Uint64
}
func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
// protocol version is required by victoria-logs
q := remoteWriteURL.Query()
q.Set("version", netinsert.ProtocolVersion)
remoteWriteURL.RawQuery = q.Encode()
// strip query params, otherwise changing params resets pq
pqURL := *remoteWriteURL
pqURL.RawQuery = ""
pqURL.Fragment = ""
h := xxhash.Sum64([]byte(pqURL.String()))
queuePath := filepath.Join(*tmpDataPath, persistentQueueDirname, fmt.Sprintf("%d_%016X", argIdx+1, h))
maxPendingBytes := maxPendingBytesPerURL.GetOptionalArg(argIdx)
if maxPendingBytes != 0 && maxPendingBytes < persistentqueue.DefaultChunkFileSize {
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4195
logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
maxPendingBytes = persistentqueue.DefaultChunkFileSize
}
fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, false)
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
return float64(fq.GetPendingBytes())
})
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
return float64(fq.GetInmemoryQueueLen())
})
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_queue_blocked{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
if fq.IsWriteBlocked() {
return 1
}
return 0
})
var c *client
switch remoteWriteURL.Scheme {
case "http", "https":
c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
default:
logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
}
c.init(argIdx, *queues, sanitizedURL)
// Initialize pss
plsLen := *queues
if n := cgroup.AvailableCPUs(); plsLen > n {
// There is no sense in running more than availableCPUs concurrent pendingLogs,
// since every pendingLogs can saturate up to a single CPU.
plsLen = n
}
pls := make([]*pendingLogs, plsLen)
for i := range pls {
pls[i] = newPendingLogs(fq)
}
rwctx := &remoteWriteCtx{
idx: argIdx,
fq: fq,
c: c,
pls: pls,
}
return rwctx
}
func (rwctx *remoteWriteCtx) push(lr *logstorage.LogRows) {
pls := rwctx.pls
idx := rwctx.pssNextIdx.Add(1) % uint64(len(pls))
pls[idx].add(lr)
}
func (rwctx *remoteWriteCtx) mustStop() {
for _, ps := range rwctx.pls {
ps.mustStop()
}
rwctx.idx = 0
rwctx.pls = nil
rwctx.fq.UnblockAllReaders()
rwctx.c.MustStop()
rwctx.c = nil
rwctx.fq.MustClose()
rwctx.fq = nil
}

View File

@@ -101,7 +101,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
var (
v2LogsRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
v2LogsRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
v2LogsRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
)
// datadog message field has two formats:

View File

@@ -129,7 +129,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
var (
bulkRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
bulkRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
)
func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {

View File

@@ -84,5 +84,5 @@ var (
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/internal/insert"}`)
requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/internal/insert"}`)
)

View File

@@ -28,6 +28,24 @@ import (
// See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
const maxFieldNameLen = 64
func isValidJournaldFieldName(s string) bool {
if len(s) == 0 {
return false
}
c := s[0]
if !(c >= 'A' && c <= 'Z' || c == '_') {
return false
}
for i := 1; i < len(s); i++ {
c := s[i]
if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
return false
}
}
return true
}
var (
journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields")
@@ -145,7 +163,7 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
var (
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
)
func processStreamInternal(streamName string, r io.Reader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
@@ -196,18 +214,6 @@ func (fb *fieldsBuf) addField(name, value string) {
})
}
func (fb *fieldsBuf) appendNextLineToValue(lr *insertutil.LineReader) error {
if !lr.NextLine() {
if err := lr.Err(); err != nil {
return err
}
return fmt.Errorf("unexpected end of stream")
}
fb.value = append(fb.value, lr.Line...)
fb.value = append(fb.value, '\n')
return nil
}
func getFieldsBuf() *fieldsBuf {
fb := fieldsBufPool.Get()
if fb == nil {
@@ -253,35 +259,42 @@ func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp inse
return nil
}
// line could be either "key=value" or "key"
// line could be either "key=value\n" or "key\n<little_endian_size_64>value\n"
// according to https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
if n := bytes.IndexByte(line, '='); n >= 0 {
// line = "key=value"
// "key=value\n"
fb.name = append(fb.name[:0], line[:n]...)
name = bytesutil.ToUnsafeString(fb.name)
fb.value = append(fb.value[:0], line[n+1:]...)
value = bytesutil.ToUnsafeString(fb.value)
} else {
// line = "key"
// Parse the binary-encoded value from the next line according to "key\n<little_endian_size_64>value\n" format
// "key\n<little_endian_size_64>value\n"
fb.name = append(fb.name[:0], line...)
name = bytesutil.ToUnsafeString(fb.name)
fb.value = fb.value[:0]
for len(fb.value) < 8 {
if err := fb.appendNextLineToValue(lr); err != nil {
return fmt.Errorf("cannot read value size: %w", err)
if !lr.NextLine() {
if err := lr.Err(); err != nil {
return fmt.Errorf("cannot read value size: %w", err)
}
return fmt.Errorf("unexpected end of stream while reading value size")
}
fb.value = append(fb.value, lr.Line...)
fb.value = append(fb.value, '\n')
}
size := binary.LittleEndian.Uint64(fb.value[:8])
// Read the value until its length exceeds the given size - the last char in the read value will always be '\n'
// because it is appended by appendNextLineToValue().
for uint64(len(fb.value[8:])) <= size {
if err := fb.appendNextLineToValue(lr); err != nil {
return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
for size > uint64(len(fb.value[8:])) {
if !lr.NextLine() {
if err := lr.Err(); err != nil {
return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
}
return fmt.Errorf("unexpected end of stream while reading %q value with size %d bytes; read only %d bytes", fb.name, size, len(fb.value[8:]))
}
fb.value = append(fb.value, lr.Line...)
fb.value = append(fb.value, '\n')
}
value = bytesutil.ToUnsafeString(fb.value[8 : len(fb.value)-1])
if uint64(len(value)) != size {
@@ -301,7 +314,7 @@ func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp inse
logger.Errorf("%s: field name size should not exceed %d bytes; got %d bytes: %q; skipping this field", streamName, maxFieldNameLen, len(name), name)
continue
}
if !isValidFieldName(name) {
if !isValidJournaldFieldName(name) {
logger.Errorf("%s: invalid field name %q; it must consist of `A-Z0-9_` chars and must start from non-digit char; skipping this field", streamName, name)
continue
}
@@ -337,19 +350,13 @@ func journaldPriorityToLevel(priority string) string {
// See https://wiki.archlinux.org/title/Systemd/Journal#Priority_level
// and https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level
switch priority {
case "0":
return "emerg"
case "1":
return "alert"
case "2":
case "0", "1", "2":
return "critical"
case "3":
return "error"
case "4":
return "warning"
case "5":
return "notice"
case "6":
case "5", "6":
return "info"
case "7":
return "debug"
@@ -357,21 +364,3 @@ func journaldPriorityToLevel(priority string) string {
return priority
}
}
func isValidFieldName(s string) bool {
if len(s) == 0 {
return false
}
c := s[0]
if !(c >= 'A' && c <= 'Z' || c == '_') {
return false
}
for i := 1; i < len(s); i++ {
c := s[i]
if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
return false
}
}
return true
}

View File

@@ -8,11 +8,11 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
)
func TestIsValidFieldName(t *testing.T) {
func TestIsValidJournaldFieldName(t *testing.T) {
f := func(name string, resultExpected bool) {
t.Helper()
result := isValidFieldName(name)
result := isValidJournaldFieldName(name)
if result != resultExpected {
t.Fatalf("unexpected result for isValidJournaldFieldName(%q); got %v; want %v", name, result, resultExpected)
}
@@ -102,24 +102,6 @@ func TestPushJournald_Success(t *testing.T) {
"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
)
// Parse binary data with trailing newline
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x14\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n\n_PID=2763\n\n",
[]int64{1729698775704404000},
`{"_CMDLINE":"python3","_msg":"foo\nbar\n\n\nasda\nasda\n","_PID":"2763"}`,
)
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00\n_PID=2763\n\n",
[]int64{1729698775704404000},
`{"_CMDLINE":"python3","_PID":"2763"}`,
)
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n\n_PID=2763\n\n",
[]int64{1729698775704404000},
`{"_CMDLINE":"python3","_msg":"123456789\n","_PID":"2763"}`,
)
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890\n_PID=2763\n\n",
[]int64{1729698775704404000},
`{"_CMDLINE":"python3","_msg":"1234567890","_PID":"2763"}`,
)
// Empty field name must be ignored
f("__REALTIME_TIMESTAMP=91723819283\na=b\n=Test message", nil, "")
f("__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n__REALTIME_TIMESTAMP=91723819283\n=Test message\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
@@ -156,10 +138,7 @@ func TestPushJournald_Failure(t *testing.T) {
}
// too short binary encoded message
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasd")
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00_PID=2763\n\n")
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890_PID=2763\n\n")
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n_PID=2763\n\n")
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasda")
// too long binary encoded message
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasdakljlsfd")

View File

@@ -4,55 +4,12 @@ import (
"bytes"
"encoding/binary"
"fmt"
"strings"
"testing"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
)
func BenchmarkIsValidFieldName(b *testing.B) {
b.ReportAllocs()
b.SetBytes(int64(len(benchmarkFields)))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
for _, field := range benchmarkFields {
if !isValidFieldName(field) {
panic(fmt.Errorf("cannot validate field %q", field))
}
}
}
})
}
var benchmarkFields = strings.Split(
"E,_BOOT_ID,_UID,_GID,_MACHINE_ID,_HOSTNAME,_RUNTIME_SCOPE,_TRANSPORT,_CAP_EFFECTIVE,_SYSTEMD_CGROUP,_SYSTEMD_UNIT,"+
"_SYSTEMD_SLICE,CODE_FILE,CODE_LINE,CODE_FUNC,SYSLOG_IDENTIFIER,_COMM,_EXE,_CMDLINE,MESSAGE,_PID,_SOURCE_REALTIME_TIMESTAMP,_REALTIME_TIMESTAMP",
",")
func BenchmarkPushJournaldPerformance(b *testing.B) {
cp := &insertutil.CommonParams{
TimeFields: []string{"__REALTIME_TIMESTAMP"},
MsgFields: []string{"MESSAGE"},
}
const dataChunkSize = 1024 * 1024
data := generateJournaldData(dataChunkSize)
b.ReportAllocs()
b.SetBytes(int64(len(data)))
b.RunParallel(func(pb *testing.PB) {
r := &bytes.Reader{}
blp := &insertutil.BenchmarkLogMessageProcessor{}
for pb.Next() {
r.Reset(data)
if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
panic(fmt.Errorf("unexpected error: %w", err))
}
}
})
}
func generateJournaldData(size int) []byte {
var buf []byte
timestamp := time.Now().UnixMicro()
@@ -80,3 +37,26 @@ func generateJournaldData(size int) []byte {
}
return buf
}
func BenchmarkPushJournaldPerformance(b *testing.B) {
cp := &insertutil.CommonParams{
TimeFields: []string{"__REALTIME_TIMESTAMP"},
MsgFields: []string{"MESSAGE"},
}
const dataChunkSize = 1024 * 1024
data := generateJournaldData(dataChunkSize)
b.ReportAllocs()
b.SetBytes(int64(len(data)))
b.RunParallel(func(pb *testing.PB) {
r := &bytes.Reader{}
blp := &insertutil.BenchmarkLogMessageProcessor{}
for pb.Next() {
r.Reset(data)
if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
panic(fmt.Errorf("unexpected error: %w", err))
}
}
})
}

View File

@@ -119,5 +119,5 @@ var (
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
)

View File

@@ -58,7 +58,7 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
var (
requestsJSONTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
requestJSONDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
)
func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {

View File

@@ -62,7 +62,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
var (
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
)
func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {

View File

@@ -70,7 +70,7 @@ var (
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
)
func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) error {

View File

@@ -101,7 +101,7 @@ func TestProcessStreamInternal_Success(t *testing.T) {
currentYear := 2023
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
{"priority":"165","facility_keyword":"local4","level":"notice","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
{"priority":"165","facility_keyword":"local4","level":"info","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
{"priority":"123","facility_keyword":"solaris-cron","level":"error","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
f(data, currentYear, timestampsExpected, resultExpected)
}

View File

@@ -742,23 +742,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
This function is supported by PromQL.
See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
#### rate_prometheus
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
See also [irate](#irate) and [rollup_rate](#rollup_rate).
#### rate_over_sum

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -35,10 +35,10 @@
<meta property="og:title" content="UI for VictoriaLogs">
<meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
<meta property="og:description" content="Explore your log data with VictoriaLogs UI">
<script type="module" crossorigin src="./assets/index-721xTF8u.js"></script>
<link rel="modulepreload" crossorigin href="./assets/vendor-V4vnRsM-.js">
<script type="module" crossorigin src="./assets/index-DhqzKCNf.js"></script>
<link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
<link rel="stylesheet" crossorigin href="./assets/index-C36SC0pJ.css">
<link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>

View File

@@ -89,18 +89,15 @@ func (t *Type) ValidateExpr(expr string) error {
return nil
}
// SupportedType is true if given datasource type is supported
func SupportedType(dsType string) bool {
return dsType == "graphite" || dsType == "prometheus" || dsType == "vlogs"
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
var s string
if err := unmarshal(&s); err != nil {
return err
}
if !SupportedType(s) {
switch s {
case "graphite", "prometheus", "vlogs":
default:
return fmt.Errorf("unknown datasource type=%q, want prometheus, graphite or vlogs", s)
}
t.Name = s

View File

@@ -148,13 +148,9 @@ func main() {
if err != nil {
logger.Fatalf("failed to init datasource: %s", err)
}
totalRows, droppedRows, err := replay(groupsCfg, q, rw)
if err != nil {
if err := replay(groupsCfg, q, rw); err != nil {
logger.Fatalf("replay failed: %s", err)
}
if droppedRows > 0 {
logger.Fatalf("failed to push all generated samples to remote write url, dropped %d samples out of %d", droppedRows, totalRows)
}
logger.Infof("replay succeed!")
return
}

View File

@@ -194,10 +194,9 @@ func mergeLabels(target string, metaLabels *promutil.Labels, cfg *Config) *promu
alertsPath = address[n:]
address = address[:n]
}
m.Add("__address__", address)
m.Add("__scheme__", scheme)
m.Add("__alerts_path__", alertsPath)
m.AddFrom(metaLabels)
// force labels
m.Set("__address__", address)
m.Set("__scheme__", scheme)
m.Set("__alerts_path__", alertsPath)
return m
}

View File

@@ -2,7 +2,6 @@ package notifier
import (
"fmt"
"sort"
"sync"
"time"
@@ -54,11 +53,8 @@ func (cw *configWatcher) notifiers() []Notifier {
for _, n := range ns {
notifiers = append(notifiers, n.Notifier)
}
}
// deterministically sort the output
sort.Slice(notifiers, func(i, j int) bool {
return notifiers[i].Addr() < notifiers[j].Addr()
})
return notifiers
}
@@ -89,12 +85,12 @@ func (cw *configWatcher) reload(path string) error {
}
func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn getLabels) error {
targetMetadata, errors := getTargetMetadata(labelsFn, cw.cfg)
targets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
for _, err := range errors {
return fmt.Errorf("failed to init notifier for %q: %w", typeK, err)
}
cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
cw.setTargets(typeK, targets)
cw.wg.Add(1)
go func() {
@@ -109,22 +105,22 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn
return
case <-ticker.C:
}
targetMetadata, errors := getTargetMetadata(labelsFn, cw.cfg)
updateTargets, errors := targetsFromLabels(labelsFn, cw.cfg, cw.genFn)
for _, err := range errors {
logger.Errorf("failed to init notifier for %q: %w", typeK, err)
}
cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
cw.setTargets(typeK, updateTargets)
}
}()
return nil
}
func getTargetMetadata(labelsFn getLabels, cfg *Config) (map[string]*promutil.Labels, []error) {
func targetsFromLabels(labelsFn getLabels, cfg *Config, genFn AlertURLGenerator) ([]Target, []error) {
metaLabels, err := labelsFn()
if err != nil {
return nil, []error{fmt.Errorf("failed to get labels: %w", err)}
}
targetMetadata := make(map[string]*promutil.Labels, len(metaLabels))
var targets []Target
var errors []error
duplicates := make(map[string]struct{})
for _, labels := range metaLabels {
@@ -147,9 +143,18 @@ func getTargetMetadata(labelsFn getLabels, cfg *Config) (map[string]*promutil.La
continue
}
duplicates[u] = struct{}{}
targetMetadata[u] = processedLabels
am, err := NewAlertManager(u, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
if err != nil {
errors = append(errors, err)
continue
}
targets = append(targets, Target{
Notifier: am,
Labels: processedLabels,
})
}
return targetMetadata, errors
return targets, errors
}
type getLabels func() ([]*promutil.Labels, error)
@@ -236,40 +241,21 @@ func (cw *configWatcher) mustStop() {
func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
cw.targetsMu.Lock()
newT := make(map[string]Target)
for _, t := range targets {
newT[t.Addr()] = t
}
oldT := cw.targets[key]
for _, ot := range oldT {
if _, ok := newT[ot.Addr()]; !ok {
ot.Notifier.Close()
}
}
cw.targets[key] = targets
cw.targetsMu.Unlock()
}
func (cw *configWatcher) updateTargets(key TargetType, targetMetadata map[string]*promutil.Labels, cfg *Config, genFn AlertURLGenerator) {
cw.targetsMu.Lock()
defer cw.targetsMu.Unlock()
oldTargets := cw.targets[key]
var updatedTargets []Target
for _, ot := range oldTargets {
if _, ok := targetMetadata[ot.Addr()]; !ok {
// if target not exists in currentTargets, close it
ot.Notifier.Close()
} else {
updatedTargets = append(updatedTargets, ot)
delete(targetMetadata, ot.Addr())
}
}
// create new resources for the new targets
for addr, labels := range targetMetadata {
am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
if err != nil {
logger.Errorf("failed to init %s notifier with addr %q: %w", key, addr, err)
continue
}
updatedTargets = append(updatedTargets, Target{
Notifier: am,
Labels: labels,
})
}
cw.targets[key] = updatedTargets
}
// mergeHTTPClientConfigs merges fields between child and parent params
// by populating child from parent params if they're missing.
func mergeHTTPClientConfigs(parent, child promauth.HTTPClientConfig) promauth.HTTPClientConfig {

View File

@@ -8,10 +8,8 @@ import (
"os"
"sync"
"testing"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
)
func TestConfigWatcherReload(t *testing.T) {
@@ -61,11 +59,6 @@ static_configs:
}
func TestConfigWatcherStart(t *testing.T) {
oldSDCheckInterval := consul.SDCheckInterval
defer func() { consul.SDCheckInterval = oldSDCheckInterval }()
consulCheckInterval := 100 * time.Millisecond
consul.SDCheckInterval = &consulCheckInterval
consulSDServer := newFakeConsulServer()
defer consulSDServer.Close()
@@ -104,11 +97,6 @@ consul_sd_configs:
if n2.Addr() != expAddr2 {
t.Fatalf("exp address %q; got %q", expAddr2, n2.Addr())
}
f := func() bool { return len(cw.notifiers()) == 1 }
if !waitFor(f, time.Second) {
t.Fatalf("expected to get 1 notifiers; got %d", len(cw.notifiers()))
}
}
// TestConfigWatcherReloadConcurrent supposed to test concurrent
@@ -205,7 +193,6 @@ const (
)
func newFakeConsulServer() *httptest.Server {
requestCount := 0
mux := http.NewServeMux()
mux.HandleFunc("/v1/agent/self", func(rw http.ResponseWriter, _ *http.Request) {
rw.Write([]byte(`{"Config": {"Datacenter": "dc1"}}`))
@@ -220,9 +207,8 @@ func newFakeConsulServer() *httptest.Server {
}`))
})
mux.HandleFunc("/v1/health/service/alertmanager", func(rw http.ResponseWriter, _ *http.Request) {
if requestCount == 0 {
rw.Header().Set("X-Consul-Index", "1")
rw.Write([]byte(`
rw.Header().Set("X-Consul-Index", "1")
rw.Write([]byte(`
[
{
"Node": {
@@ -311,56 +297,6 @@ func newFakeConsulServer() *httptest.Server {
}
}
]`))
} else {
rw.Header().Set("X-Consul-Index", "2")
rw.Write([]byte(`
[
{
"Node": {
"ID": "e8e3629a-3f50-9d6e-aaf8-f173b5b05c72",
"Node": "machine",
"Address": "127.0.0.1",
"Datacenter": "dc1",
"TaggedAddresses": {
"lan": "127.0.0.1",
"lan_ipv4": "127.0.0.1",
"wan": "127.0.0.1",
"wan_ipv4": "127.0.0.1"
},
"Meta": {
"consul-network-segment": ""
},
"CreateIndex": 13,
"ModifyIndex": 14
},
"Service": {
"ID": "am3",
"Service": "alertmanager",
"Tags": [
"alertmanager",
"__scheme__=http"
],
"Address": "",
"Meta": null,
"Port": 9097,
"Weights": {
"Passing": 1,
"Warning": 1
},
"EnableTagOverride": false,
"Proxy": {
"Mode": "",
"MeshGateway": {},
"Expose": {}
},
"Connect": {},
"CreateIndex": 16,
"ModifyIndex": 16
}
}
]`))
}
requestCount++
})
return httptest.NewServer(mux)
@@ -421,13 +357,3 @@ func TestParseLabels_Success(t *testing.T) {
PathPrefix: "test",
}, "https://alertmanager:9093/api/v1/alerts")
}
func waitFor(f func() bool, timeout time.Duration) bool {
for start := time.Now(); time.Since(start) < timeout; {
if f() == true {
return true
}
time.Sleep(50 * time.Millisecond)
}
return false
}

View File

@@ -216,7 +216,7 @@ var (
)
// GetDroppedRows returns value of droppedRows metric
func GetDroppedRows() int { return int(droppedRows.Get()) }
func GetDroppedRows() int64 { return int64(droppedRows.Get()) }
// flush is a blocking function that marshals WriteRequest and sends
// it to remote-write endpoint. Flush performs limited amount of retries

View File

@@ -20,8 +20,9 @@ var (
"The time filter in RFC3339 format to finish the replay by. E.g. '2020-01-01T20:07:00Z'. "+
"By default, is set to the current time.")
replayRulesDelay = flag.Duration("replay.rulesDelay", time.Second,
"Delay before evaluating the next rule within the group. Is important for chained rules. "+
"Keep it equal or bigger than -remoteWrite.flushInterval. When set to >0, replay ignores group's concurrency setting.")
"Delay between rules evaluation within the group. Could be important if there are chained rules inside the group "+
"and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule."+
"Keep it equal or bigger than -remoteWrite.flushInterval.")
replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
@@ -30,13 +31,13 @@ var (
"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
)
func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {
func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) error {
if *replayMaxDatapoints < 1 {
return 0, 0, fmt.Errorf("replay.maxDatapointsPerQuery can't be lower than 1")
return fmt.Errorf("replay.maxDatapointsPerQuery can't be lower than 1")
}
tFrom, err := time.Parse(time.RFC3339, *replayFrom)
if err != nil {
return 0, 0, fmt.Errorf("failed to parse replay.timeFrom=%q: %w", *replayFrom, err)
return fmt.Errorf("failed to parse replay.timeFrom=%q: %w", *replayFrom, err)
}
// use tFrom location for default value, otherwise filters could have different locations
@@ -44,12 +45,12 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
if *replayTo != "" {
tTo, err = time.Parse(time.RFC3339, *replayTo)
if err != nil {
return 0, 0, fmt.Errorf("failed to parse replay.timeTo=%q: %w", *replayTo, err)
return fmt.Errorf("failed to parse replay.timeTo=%q: %w", *replayTo, err)
}
}
if !tTo.After(tFrom) {
return 0, 0, fmt.Errorf("replay.timeTo=%v must be bigger than replay.timeFrom=%v", tTo, tFrom)
return fmt.Errorf("replay.timeTo=%v must be bigger than replay.timeFrom=%v", tTo, tFrom)
}
labels := make(map[string]string)
for _, s := range *externalLabels {
@@ -58,7 +59,7 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
}
n := strings.IndexByte(s, '=')
if n < 0 {
return 0, 0, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
return fmt.Errorf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
}
labels[s[:n]] = s[n+1:]
}
@@ -69,14 +70,18 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
"\nmax data points per request: %d\n",
tFrom, tTo, *replayMaxDatapoints)
var total int
for _, cfg := range groupsCfg {
ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar)
total += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar)
}
logger.Infof("replay evaluation finished, generated %d samples", totalRows)
logger.Infof("replay evaluation finished, generated %d samples", total)
if err := rw.Close(); err != nil {
return 0, 0, err
return err
}
droppedRows = remotewrite.GetDroppedRows()
return totalRows, droppedRows, nil
droppedRows := remotewrite.GetDroppedRows()
if droppedRows > 0 {
return fmt.Errorf("failed to push all generated samples to remote write url, dropped %d samples out of %d", droppedRows, total)
}
return nil
}

View File

@@ -8,45 +8,38 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
)
type fakeReplayQuerier struct {
datasource.FakeQuerier
registry map[string]map[string][]datasource.Metric
registry map[string]map[string]struct{}
}
func (fr *fakeReplayQuerier) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
return fr
}
type fakeRWClient struct{}
func (fc *fakeRWClient) Push(_ prompbmarshal.TimeSeries) error {
return nil
}
func (fc *fakeRWClient) Close() error {
return nil
}
func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to time.Time) (res datasource.Result, err error) {
key := fmt.Sprintf("%s+%s", from.Format("15:04:05"), to.Format("15:04:05"))
dps, ok := fr.registry[q]
if !ok {
return res, fmt.Errorf("unexpected query received: %q", q)
}
metrics, ok := dps[key]
_, ok = dps[key]
if !ok {
return res, fmt.Errorf("unexpected time range received: %q", key)
}
res.Data = metrics
delete(dps, key)
if len(fr.registry[q]) < 1 {
delete(fr.registry, q)
}
return res, nil
}
func TestReplay(t *testing.T) {
f := func(from, to string, maxDP int, ruleDelay time.Duration, cfg []config.Group, qb *fakeReplayQuerier, expectTotalRows int) {
f := func(from, to string, maxDP int, cfg []config.Group, qb *fakeReplayQuerier) {
t.Helper()
fromOrig, toOrig, maxDatapointsOrig := *replayFrom, *replayTo, *replayMaxDatapoints
@@ -58,172 +51,90 @@ func TestReplay(t *testing.T) {
}()
*replayRuleRetryAttempts = 1
*replayRulesDelay = ruleDelay
rwb := &fakeRWClient{}
*replayRulesDelay = time.Millisecond
rwb := &remotewrite.DebugClient{}
*replayFrom = from
*replayTo = to
*replayMaxDatapoints = maxDP
totalRows, _, err := replay(cfg, qb, rwb)
if err != nil {
if err := replay(cfg, qb, rwb); err != nil {
t.Fatalf("replay failed: %s", err)
}
if totalRows != expectTotalRows {
t.Fatalf("unexpected total rows count: got %d, want %d", totalRows, expectTotalRows)
if len(qb.registry) > 0 {
t.Fatalf("not all requests were sent: %#v", qb.registry)
}
}
// one rule + one response
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, time.Millisecond, []config.Group{
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, []config.Group{
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
}, &fakeReplayQuerier{
registry: map[string]map[string][]datasource.Metric{
"sum(up)": {"12:00:00+12:02:00": {
{
Timestamps: []int64{1},
Values: []float64{1},
},
}},
registry: map[string]map[string]struct{}{
"sum(up)": {"12:00:00+12:02:00": {}},
},
}, 1)
})
// one rule + multiple responses
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
}, &fakeReplayQuerier{
registry: map[string]map[string][]datasource.Metric{
registry: map[string]map[string]struct{}{
"sum(up)": {
"12:00:00+12:01:00": {
{
Timestamps: []int64{1},
Values: []float64{1},
},
},
"12:00:00+12:01:00": {},
"12:01:00+12:02:00": {},
"12:02:00+12:02:30": {
{
Timestamps: []int64{1},
Values: []float64{1},
},
},
"12:02:00+12:02:30": {},
},
},
}, 2)
})
// datapoints per step
f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, time.Millisecond, []config.Group{
f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, []config.Group{
{Interval: promutil.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
}, &fakeReplayQuerier{
registry: map[string]map[string][]datasource.Metric{
registry: map[string]map[string]struct{}{
"sum(up)": {
"12:00:00+13:00:00": {
{
Timestamps: []int64{1, 2},
Values: []float64{1, 2},
},
},
"13:00:00+14:00:00": {
{
Timestamps: []int64{1},
Values: []float64{1},
},
},
"12:00:00+13:00:00": {},
"13:00:00+14:00:00": {},
"14:00:00+15:00:00": {},
"15:00:00+15:02:30": {},
},
},
}, 3)
})
// multiple recording rules + multiple responses
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
}, &fakeReplayQuerier{
registry: map[string]map[string][]datasource.Metric{
registry: map[string]map[string]struct{}{
"sum(up)": {
"12:00:00+12:01:00": {
{
Timestamps: []int64{1, 2},
Values: []float64{1, 2},
},
},
"12:00:00+12:01:00": {},
"12:01:00+12:02:00": {},
"12:02:00+12:02:30": {},
},
"max(up)": {
"12:00:00+12:01:00": {},
"12:01:00+12:02:00": {
{
Timestamps: []int64{1, 2},
Values: []float64{1, 2},
},
},
"12:01:00+12:02:00": {},
"12:02:00+12:02:30": {},
},
},
}, 4)
})
// multiple alerting rules + multiple responses
// alerting rule generates two series `ALERTS` and `ALERTS_FOR_STATE` when triggered
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
}, &fakeReplayQuerier{
registry: map[string]map[string][]datasource.Metric{
registry: map[string]map[string]struct{}{
"sum(up) > 1": {
"12:00:00+12:01:00": {
{
Timestamps: []int64{1, 2},
Values: []float64{1, 2},
},
},
"12:00:00+12:01:00": {},
"12:01:00+12:02:00": {},
"12:02:00+12:02:30": {},
},
"max(up) < 1": {
"12:00:00+12:01:00": {
{
Timestamps: []int64{1},
Values: []float64{1},
},
},
"12:01:00+12:02:00": {},
"12:02:00+12:02:30": {},
},
},
}, 6)
// multiple recording rules in one group+ multiple responses + concurrency
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 0, []config.Group{
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up) > 1"}, {Record: "bar", Expr: "max(up) < 1"}}, Concurrency: 2}}, &fakeReplayQuerier{
registry: map[string]map[string][]datasource.Metric{
"sum(up) > 1": {
"12:00:00+12:01:00": {
{
Timestamps: []int64{1},
Values: []float64{1},
},
},
"12:01:00+12:02:00": {
{
Timestamps: []int64{1},
Values: []float64{1},
},
},
"12:02:00+12:02:30": {
{
Timestamps: []int64{1},
Values: []float64{1},
},
},
},
"max(up) < 1": {
"12:00:00+12:01:00": {},
"12:01:00+12:02:00": {{
Timestamps: []int64{1},
Values: []float64{1},
}},
"12:01:00+12:02:00": {},
"12:02:00+12:02:30": {},
},
},
}, 4)
})
}

View File

@@ -445,17 +445,11 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
g.infof("re-started")
case <-t.C:
// calculate the real wall clock offset by stripping the monotonic clock first,
// then evalTS can be corrected when wall clock is adjusted.
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8790#issuecomment-2986541829
offset := time.Now().Round(0).Sub(evalTS.Round(0))
missed := (offset / g.Interval) - 1
missed := (time.Since(evalTS) / g.Interval) - 1
if missed < 0 {
// missed can become < 0 due to irregular delays during evaluation
// which can result in time.Since(evalTS) < g.Interval;
// or the system wall clock was changed backward
// which can result in time.Since(evalTS) < g.Interval
missed = 0
evalTS = time.Now()
}
if missed > 0 {
g.metrics.iterationMissed.Inc()
@@ -520,84 +514,36 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
iterations := int(end.Sub(start)/step) + 1
fmt.Printf("\nGroup %q"+
"\ninterval: \t%v"+
"\nconcurrency: \t %d"+
"\nrequests to make per rule: \t%d"+
"\nrequests to make: \t%d"+
"\nmax range per request: \t%v\n",
g.Name, g.Interval, g.Concurrency, iterations, step)
g.Name, g.Interval, iterations, step)
if g.Limit > 0 {
fmt.Printf("\nWarning: `limit: %d` param has no effect during replay.\n",
fmt.Printf("\nPlease note, `limit: %d` param has no effect during replay.\n",
g.Limit)
}
concurrency := g.Concurrency
if g.Concurrency > 1 && replayDelay > 0 {
fmt.Printf("\nWarning: group concurrency %d will be ignored since `-replay.rulesDelay` is %.3f seconds."+
" Set -replay.rulesDelay=0 to enable concurrency for replay.\n", g.Concurrency, replayDelay.Seconds())
concurrency = 1
}
if concurrency == 1 {
for _, rule := range g.Rules {
var bar *pb.ProgressBar
if !disableProgressBar {
bar = pb.StartNew(iterations)
}
// pass ri as a copy, so it can be modified within the replayRuleRange
total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts)
if bar != nil {
bar.Finish()
}
// sleep to let remote storage to flush data on-disk
// so chained rules could be calculated correctly
time.Sleep(replayDelay)
for _, rule := range g.Rules {
fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
var bar *pb.ProgressBar
if !disableProgressBar {
bar = pb.StartNew(iterations)
}
return total
}
sem := make(chan struct{}, g.Concurrency)
res := make(chan int, len(g.Rules)*iterations)
wg := sync.WaitGroup{}
var bar *pb.ProgressBar
if !disableProgressBar {
bar = pb.StartNew(iterations * len(g.Rules))
}
for _, r := range g.Rules {
sem <- struct{}{}
wg.Add(1)
go func(r Rule, ri rangeIterator) {
// pass ri as a copy, so it can be modified within the replayRuleRange
res <- replayRuleRange(r, ri, bar, rw, replayRuleRetryAttempts)
<-sem
wg.Done()
}(r, ri)
}
wg.Wait()
close(res)
close(sem)
if bar != nil {
bar.Finish()
}
total = 0
for n := range res {
total += n
}
return total
}
func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts int) int {
fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
total := 0
for ri.next() {
n, err := replayRule(r, ri.s, ri.e, rw, replayRuleRetryAttempts)
if err != nil {
logger.Fatalf("rule %q: %s", r, err)
ri.reset()
for ri.next() {
n, err := replayRule(rule, ri.s, ri.e, rw, replayRuleRetryAttempts)
if err != nil {
logger.Fatalf("rule %q: %s", rule, err)
}
total += n
if bar != nil {
bar.Increment()
}
}
if bar != nil {
bar.Increment()
bar.Finish()
}
total += n
// sleep to let remote storage to flush data on-disk
// so chained rules could be calculated correctly
time.Sleep(replayDelay)
}
return total
}
@@ -624,10 +570,11 @@ type rangeIterator struct {
s, e time.Time
}
// next iterates with given step between start and end
// by modifying iter, s and e.
// Returns true until it reaches end.
// next modifies ri and isn't thread-safe.
func (ri *rangeIterator) reset() {
ri.iter = 0
ri.s, ri.e = time.Time{}, time.Time{}
}
func (ri *rangeIterator) next() bool {
ri.s = ri.start.Add(ri.step * time.Duration(ri.iter))
if !ri.end.After(ri.s) {

View File

@@ -99,15 +99,3 @@ textarea.curl-area {
padding: 0;
overflow: scroll;
}
.w-10 {
width: 10%;
}
.w-20 {
width: 20%;
}
.w-60 {
width: 60%;
}

View File

@@ -9,7 +9,6 @@ import (
"strconv"
"strings"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
@@ -28,7 +27,6 @@ var (
// such as Grafana, and proxied via vmselect.
{"api/v1/rules", "list all loaded groups and rules"},
{"api/v1/alerts", "list all active alerts"},
{"api/v1/notifiers", "list all notifiers"},
{fmt.Sprintf("api/v1/alert?%s=<int>&%s=<int>", paramGroupID, paramAlertID), "get alert status by group and alert ID"},
}
systemLinks = [][2]string{
@@ -44,10 +42,6 @@ var (
{Name: "Notifiers", URL: "notifiers"},
{Name: "Docs", URL: "https://docs.victoriametrics.com/victoriametrics/vmalert/"},
}
ruleTypeMap = map[string]string{
"alert": ruleTypeAlerting,
"record": ruleTypeRecording,
}
)
type requestHandler struct {
@@ -95,13 +89,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
WriteRuleDetails(w, r, rule)
return true
case "/vmalert/groups":
rf, err := newRulesFilter(r)
if err != nil {
httpserver.Errorf(w, r, "%s", err)
return true
}
filter := r.URL.Query().Get("filter")
rf := extractRulesFilter(r, filter)
data := rh.groups(rf)
WriteListGroups(w, r, data, rf.filter)
WriteListGroups(w, r, data, filter)
return true
case "/vmalert/notifiers":
WriteListTargets(w, r, notifier.GetTargets())
@@ -111,35 +102,23 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
// served without `vmalert` prefix:
case "/rules":
// Grafana makes an extra request to `/rules`
// handler in addition to `/api/v1/rules` calls in alerts UI
var data []*apiGroup
rf, err := newRulesFilter(r)
if err != nil {
httpserver.Errorf(w, r, "%s", err)
return true
}
// handler in addition to `/api/v1/rules` calls in alerts UI,
var data []apiGroup
filter := r.URL.Query().Get("filter")
rf := extractRulesFilter(r, filter)
data = rh.groups(rf)
WriteListGroups(w, r, data, rf.filter)
WriteListGroups(w, r, data, filter)
return true
case "/vmalert/api/v1/notifiers", "/api/v1/notifiers":
data, err := rh.listNotifiers()
if err != nil {
httpserver.Errorf(w, r, "%s", err)
return true
}
w.Header().Set("Content-Type", "application/json")
w.Write(data)
return true
case "/vmalert/api/v1/rules", "/api/v1/rules":
// path used by Grafana for ng alerting
var data []byte
rf, err := newRulesFilter(r)
if err != nil {
httpserver.Errorf(w, r, "%s", err)
return true
}
var err error
filter := r.URL.Query().Get("filter")
rf := extractRulesFilter(r, filter)
data, err = rh.listGroups(rf)
if err != nil {
httpserver.Errorf(w, r, "%s", err)
return true
@@ -150,12 +129,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
case "/vmalert/api/v1/alerts", "/api/v1/alerts":
// path used by Grafana for ng alerting
rf, err := newRulesFilter(r)
if err != nil {
httpserver.Errorf(w, r, "%s", err)
return true
}
data, err := rh.listAlerts(rf)
data, err := rh.listAlerts()
if err != nil {
httpserver.Errorf(w, r, "%s", err)
return true
@@ -244,7 +218,7 @@ func (rh *requestHandler) getAlert(r *http.Request) (*apiAlert, error) {
type listGroupsResponse struct {
Status string `json:"status"`
Data struct {
Groups []*apiGroup `json:"groups"`
Groups []apiGroup `json:"groups"`
} `json:"data"`
}
@@ -255,102 +229,82 @@ type rulesFilter struct {
ruleNames []string
ruleType string
excludeAlerts bool
filter string
dsType config.Type
onlyUnhealthy bool
onlyNoMatch bool
}
func newRulesFilter(r *http.Request) (*rulesFilter, error) {
rf := &rulesFilter{}
query := r.URL.Query()
func extractRulesFilter(r *http.Request, filter string) rulesFilter {
rf := rulesFilter{}
ruleTypeParam := query.Get("type")
if len(ruleTypeParam) > 0 {
if ruleType, ok := ruleTypeMap[ruleTypeParam]; ok {
rf.ruleType = ruleType
} else {
return nil, errResponse(fmt.Errorf(`invalid parameter "type": not supported value %q`, ruleTypeParam), http.StatusBadRequest)
}
}
dsType := query.Get("datasource_type")
if len(dsType) > 0 {
if config.SupportedType(dsType) {
rf.dsType = config.NewRawType(dsType)
} else {
return nil, errResponse(fmt.Errorf(`invalid parameter "datasource_type": not supported value %q`, dsType), http.StatusBadRequest)
}
}
filter := strings.ToLower(query.Get("filter"))
if len(filter) > 0 {
if filter == "nomatch" || filter == "unhealthy" {
rf.filter = filter
} else {
return nil, errResponse(fmt.Errorf(`invalid parameter "filter": not supported value %q`, filter), http.StatusBadRequest)
}
var ruleType string
ruleTypeParam := r.URL.Query().Get("type")
// for some reason, `type` in filter doesn't match `type` in response,
// so we use this matching here
if ruleTypeParam == "alert" {
ruleType = ruleTypeAlerting
} else if ruleTypeParam == "record" {
ruleType = ruleTypeRecording
}
rf.ruleType = ruleType
rf.excludeAlerts = httputil.GetBool(r, "exclude_alerts")
rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
rf.files = append([]string{}, r.Form["file[]"]...)
return rf, nil
switch filter {
case "unhealthy":
rf.onlyUnhealthy = true
case "noMatch":
rf.onlyNoMatch = true
}
return rf
}
func (rf *rulesFilter) matchesGroup(group *rule.Group) bool {
if len(rf.groupNames) > 0 && !slices.Contains(rf.groupNames, group.Name) {
return false
}
if len(rf.files) > 0 && !slices.Contains(rf.files, group.File) {
return false
}
if len(rf.dsType.Name) > 0 && rf.dsType.String() != group.Type.String() {
return false
}
return true
}
func (rh *requestHandler) groups(rf *rulesFilter) []*apiGroup {
func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
rh.m.groupsMu.RLock()
defer rh.m.groupsMu.RUnlock()
groups := make([]*apiGroup, 0)
groups := make([]apiGroup, 0)
for _, group := range rh.m.groups {
if !rf.matchesGroup(group) {
if len(rf.groupNames) > 0 && !slices.Contains(rf.groupNames, group.Name) {
continue
}
if len(rf.files) > 0 && !slices.Contains(rf.files, group.File) {
continue
}
g := groupToAPI(group)
// the returned list should always be non-nil
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
filteredRules := make([]apiRule, 0)
for _, rule := range g.Rules {
if rf.ruleType != "" && rf.ruleType != rule.Type {
for _, r := range g.Rules {
if rf.ruleType != "" && rf.ruleType != r.Type {
continue
}
if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, rule.Name) {
continue
}
if (rule.LastError == "" && rf.filter == "unhealthy") || (!isNoMatch(rule) && rf.filter == "nomatch") {
if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, r.Name) {
continue
}
if rf.excludeAlerts {
rule.Alerts = nil
r.Alerts = nil
}
if rule.LastError != "" {
if (r.LastError == "" && rf.onlyUnhealthy) || (!isNoMatch(r) && rf.onlyNoMatch) {
continue
}
if r.LastError != "" {
g.Unhealthy++
} else {
g.Healthy++
}
if isNoMatch(rule) {
if isNoMatch(r) {
g.NoMatch++
}
filteredRules = append(filteredRules, rule)
filteredRules = append(filteredRules, r)
}
g.Rules = filteredRules
groups = append(groups, g)
}
// sort list of groups for deterministic output
slices.SortFunc(groups, func(a, b *apiGroup) int {
slices.SortFunc(groups, func(a, b apiGroup) int {
if a.Name != b.Name {
return strings.Compare(a.Name, b.Name)
}
@@ -359,7 +313,7 @@ func (rh *requestHandler) groups(rf *rulesFilter) []*apiGroup {
return groups
}
func (rh *requestHandler) listGroups(rf *rulesFilter) ([]byte, error) {
func (rh *requestHandler) listGroups(rf rulesFilter) ([]byte, error) {
lr := listGroupsResponse{Status: "success"}
lr.Data.Groups = rh.groups(rf)
b, err := json.Marshal(lr)
@@ -406,17 +360,14 @@ func (rh *requestHandler) groupAlerts() []groupAlerts {
return gAlerts
}
func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
func (rh *requestHandler) listAlerts() ([]byte, error) {
rh.m.groupsMu.RLock()
defer rh.m.groupsMu.RUnlock()
lr := listAlertsResponse{Status: "success"}
lr.Data.Alerts = make([]*apiAlert, 0)
for _, group := range rh.m.groups {
if !rf.matchesGroup(group) {
continue
}
for _, r := range group.Rules {
for _, g := range rh.m.groups {
for _, r := range g.Rules {
a, ok := r.(*rule.AlertingRule)
if !ok {
continue
@@ -440,42 +391,6 @@ func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
return b, nil
}
type listNotifiersResponse struct {
Status string `json:"status"`
Data struct {
Notifiers []*apiNotifier `json:"notifiers"`
} `json:"data"`
}
func (rh *requestHandler) listNotifiers() ([]byte, error) {
targets := notifier.GetTargets()
lr := listNotifiersResponse{Status: "success"}
lr.Data.Notifiers = make([]*apiNotifier, 0)
for protoName, protoTargets := range targets {
notifier := &apiNotifier{
Kind: string(protoName),
Targets: make([]*apiTarget, 0, len(protoTargets)),
}
for _, target := range protoTargets {
notifier.Targets = append(notifier.Targets, &apiTarget{
Address: target.Notifier.Addr(),
Labels: target.Labels.ToMap(),
})
}
lr.Data.Notifiers = append(lr.Data.Notifiers, notifier)
}
b, err := json.Marshal(lr)
if err != nil {
return nil, &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf(`error encoding list of notifiers: %w`, err),
StatusCode: http.StatusInternalServerError,
}
}
return b, nil
}
func errResponse(err error, sc int) *httpserver.ErrorWithStatusCode {
return &httpserver.ErrorWithStatusCode{
Err: err,

View File

@@ -93,18 +93,18 @@
{%= tpl.Footer(r) %}
{% endfunc %}
{% func ListGroups(r *http.Request, groups []*apiGroup, filter string) %}
{% func ListGroups(r *http.Request, groups []apiGroup, filter string) %}
{%code
prefix := vmalertutil.Prefix(r.URL.Path)
filters := map[string]string{
"": "All",
"unhealthy": "Unhealthy",
"nomatch": "No Match",
"noMatch": "No Match",
}
icons := map[string]string{
"": "all",
"unhealthy": "unhealthy",
"nomatch": "nomatch",
"noMatch": "nomatch",
}
currentText := filters[filter]
currentIcon := icons[filter]
@@ -161,9 +161,9 @@
<table class="table table-striped table-hover table-sm">
<thead>
<tr>
<th scope="col" class="w-60">Rule</th>
<th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series</th>
<th scope="col" class="w-20" class="text-center" title="How many seconds ago rule was executed">Updated</th>
<th scope="col" style="width: 60%">Rule</th>
<th scope="col" style="width: 20%" class="text-center" title="How many series were produced by the rule">Series</th>
<th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
</tr>
</thead>
<tbody>
@@ -594,9 +594,9 @@
<thead>
<tr>
<th scope="col" title="The time when event was created">Updated at</th>
<th scope="col" class="w-10 text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
{% if seriesFetchedEnabled %}<th scope="col" class="w-10 text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>{% endif %}
<th scope="col" class="w-10 text-center" title="How many seconds request took">Duration</th>
<th scope="col" style="width: 10%" class="text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
{% if seriesFetchedEnabled %}<th scope="col" style="width: 10%" class="text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>{% endif %}
<th scope="col" style="width: 10%" class="text-center" title="How many seconds request took">Duration</th>
<th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>
<th scope="col" class="text-center" title="cURL command with request example">cURL</th>
</tr>

View File

@@ -316,7 +316,7 @@ func Welcome(r *http.Request) string {
}
//line app/vmalert/web.qtpl:96
func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*apiGroup, filter string) {
func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []apiGroup, filter string) {
//line app/vmalert/web.qtpl:96
qw422016.N().S(`
`)
@@ -325,12 +325,12 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*apiG
filters := map[string]string{
"": "All",
"unhealthy": "Unhealthy",
"nomatch": "No Match",
"noMatch": "No Match",
}
icons := map[string]string{
"": "all",
"unhealthy": "unhealthy",
"nomatch": "nomatch",
"noMatch": "nomatch",
}
currentText := filters[filter]
currentIcon := icons[filter]
@@ -523,9 +523,9 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*apiG
<table class="table table-striped table-hover table-sm">
<thead>
<tr>
<th scope="col" class="w-60">Rule</th>
<th scope="col" class="w-20" class="text-center" title="How many series were produced by the rule">Series</th>
<th scope="col" class="w-20" class="text-center" title="How many seconds ago rule was executed">Updated</th>
<th scope="col" style="width: 60%">Rule</th>
<th scope="col" style="width: 20%" class="text-center" title="How many series were produced by the rule">Series</th>
<th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
</tr>
</thead>
<tbody>
@@ -722,7 +722,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*apiG
}
//line app/vmalert/web.qtpl:222
func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []*apiGroup, filter string) {
func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []apiGroup, filter string) {
//line app/vmalert/web.qtpl:222
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmalert/web.qtpl:222
@@ -733,7 +733,7 @@ func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []*apiG
}
//line app/vmalert/web.qtpl:222
func ListGroups(r *http.Request, groups []*apiGroup, filter string) string {
func ListGroups(r *http.Request, groups []apiGroup, filter string) string {
//line app/vmalert/web.qtpl:222
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmalert/web.qtpl:222
@@ -1697,17 +1697,17 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule apiRule)
<thead>
<tr>
<th scope="col" title="The time when event was created">Updated at</th>
<th scope="col" class="w-10 text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
<th scope="col" style="width: 10%" class="text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
`)
//line app/vmalert/web.qtpl:598
if seriesFetchedEnabled {
//line app/vmalert/web.qtpl:598
qw422016.N().S(`<th scope="col" class="w-10 text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>`)
qw422016.N().S(`<th scope="col" style="width: 10%" class="text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>`)
//line app/vmalert/web.qtpl:598
}
//line app/vmalert/web.qtpl:598
qw422016.N().S(`
<th scope="col" class="w-10 text-center" title="How many seconds request took">Duration</th>
<th scope="col" style="width: 10%" class="text-center" title="How many seconds request took">Duration</th>
<th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>
<th scope="col" class="text-center" title="cURL command with request example">cURL</th>
</tr>

View File

@@ -19,34 +19,25 @@ import (
func TestHandler(t *testing.T) {
fq := &datasource.FakeQuerier{}
fq.Add(datasource.Metric{
Values: []float64{1},
Timestamps: []int64{0},
Values: []float64{1}, Timestamps: []int64{0},
})
m := &manager{groups: map[uint64]*rule.Group{}}
var ar *rule.AlertingRule
var rr *rule.RecordingRule
for _, dsType := range []string{"prometheus", "", "graphite"} {
g := rule.NewGroup(config.Group{
Name: "group",
File: "rules.yaml",
Type: config.NewRawType(dsType),
Concurrency: 1,
Rules: []config.Rule{
{
ID: 0,
Alert: "alert",
},
{
ID: 1,
Record: "record",
},
},
}, fq, 1*time.Minute, nil)
ar = g.Rules[0].(*rule.AlertingRule)
rr = g.Rules[1].(*rule.RecordingRule)
g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
m.groups[g.CreateID()] = g
}
g := rule.NewGroup(config.Group{
Name: "group",
File: "rules.yaml",
Concurrency: 1,
Rules: []config.Rule{
{ID: 0, Alert: "alert"},
{ID: 1, Record: "record"},
},
}, fq, 1*time.Minute, nil)
ar := g.Rules[0].(*rule.AlertingRule)
rr := g.Rules[1].(*rule.RecordingRule)
g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
m := &manager{groups: map[uint64]*rule.Group{
g.CreateID(): g,
}}
rh := &requestHandler{m: m}
getResp := func(t *testing.T, url string, to any, code int) {
@@ -63,7 +54,7 @@ func TestHandler(t *testing.T) {
t.Fatalf("err closing body %s", err)
}
}()
if to != nil && code < 300 {
if to != nil {
if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
t.Fatalf("unexpected err %s", err)
}
@@ -104,23 +95,14 @@ func TestHandler(t *testing.T) {
t.Run("/api/v1/alerts", func(t *testing.T) {
lr := listAlertsResponse{}
getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
if length := len(lr.Data.Alerts); length != 3 {
t.Fatalf("expected 3 alert got %d", length)
if length := len(lr.Data.Alerts); length != 1 {
t.Fatalf("expected 1 alert got %d", length)
}
lr = listAlertsResponse{}
getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
if length := len(lr.Data.Alerts); length != 3 {
t.Fatalf("expected 3 alert got %d", length)
}
lr = listAlertsResponse{}
getResp(t, ts.URL+"/api/v1/alerts?datasource_type=test", &lr, 400)
lr = listAlertsResponse{}
getResp(t, ts.URL+"/api/v1/alerts?datasource_type=prometheus", &lr, 200)
if length := len(lr.Data.Alerts); length != 2 {
t.Fatalf("expected 2 alert got %d", length)
if length := len(lr.Data.Alerts); length != 1 {
t.Fatalf("expected 1 alert got %d", length)
}
})
t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
@@ -156,14 +138,14 @@ func TestHandler(t *testing.T) {
t.Run("/api/v1/rules", func(t *testing.T) {
lr := listGroupsResponse{}
getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
if length := len(lr.Data.Groups); length != 3 {
t.Fatalf("expected 3 group got %d", length)
if length := len(lr.Data.Groups); length != 1 {
t.Fatalf("expected 1 group got %d", length)
}
lr = listGroupsResponse{}
getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
if length := len(lr.Data.Groups); length != 3 {
t.Fatalf("expected 3 group got %d", length)
if length := len(lr.Data.Groups); length != 1 {
t.Fatalf("expected 1 group got %d", length)
}
})
t.Run("/api/v1/rule?ruleID&groupID", func(t *testing.T) {
@@ -190,10 +172,10 @@ func TestHandler(t *testing.T) {
})
t.Run("/api/v1/rules&filters", func(t *testing.T) {
check := func(url string, statusCode, expGroups, expRules int) {
check := func(url string, expGroups, expRules int) {
t.Helper()
lr := listGroupsResponse{}
getResp(t, ts.URL+url, &lr, statusCode)
getResp(t, ts.URL+url, &lr, 200)
if length := len(lr.Data.Groups); length != expGroups {
t.Fatalf("expected %d groups got %d", expGroups, length)
}
@@ -209,31 +191,25 @@ func TestHandler(t *testing.T) {
}
}
check("/api/v1/rules?type=alert", 200, 3, 3)
check("/api/v1/rules?type=record", 200, 3, 3)
check("/api/v1/rules?type=records", 400, 0, 0)
check("/api/v1/rules?type=alert", 1, 1)
check("/api/v1/rules?type=record", 1, 1)
check("/vmalert/api/v1/rules?type=alert", 200, 3, 3)
check("/vmalert/api/v1/rules?type=record", 200, 3, 3)
check("/vmalert/api/v1/rules?type=recording", 400, 0, 0)
check("/vmalert/api/v1/rules?datasource_type=prometheus", 200, 2, 4)
check("/vmalert/api/v1/rules?datasource_type=graphite", 200, 1, 2)
check("/vmalert/api/v1/rules?datasource_type=graphiti", 400, 0, 0)
check("/vmalert/api/v1/rules?type=alert", 1, 1)
check("/vmalert/api/v1/rules?type=record", 1, 1)
// no filtering expected due to bad params
check("/api/v1/rules?type=badParam", 400, 0, 0)
check("/api/v1/rules?foo=bar", 200, 3, 6)
check("/api/v1/rules?type=badParam", 1, 2)
check("/api/v1/rules?foo=bar", 1, 2)
check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 200, 0, 0)
check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 200, 3, 6)
check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 0, 0)
check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 1, 2)
check("/api/v1/rules?rule_group[]=group&file[]=foo", 200, 0, 0)
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 200, 3, 6)
check("/api/v1/rules?rule_group[]=group&file[]=foo", 0, 0)
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 1, 2)
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 200, 3, 0)
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 200, 3, 3)
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 200, 3, 6)
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 1, 0)
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 1, 1)
check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 1, 2)
})
t.Run("/api/v1/rules&exclude_alerts=true", func(t *testing.T) {
// check if response returns active alerts by default
@@ -283,7 +259,7 @@ func TestEmptyResponse(t *testing.T) {
t.Fatalf("err closing body %s", err)
}
}()
if to != nil && code < 300 {
if to != nil {
if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
t.Fatalf("unexpected err %s", err)
}

View File

@@ -20,16 +20,6 @@ const (
paramRuleID = "rule_id"
)
type apiNotifier struct {
Kind string `json:"kind"`
Targets []*apiTarget `json:"targets"`
}
type apiTarget struct {
Address string `json:"address"`
Labels map[string]string `json:"labels"`
}
// apiAlert represents a notifier.AlertingRule state
// for WEB view
// https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
@@ -118,7 +108,7 @@ type apiGroup struct {
// groupAlerts represents a group of alerts for WEB view
type groupAlerts struct {
Group *apiGroup
Group apiGroup
Alerts []*apiAlert
}
@@ -337,7 +327,7 @@ func newAlertAPI(ar *rule.AlertingRule, a *notifier.Alert) *apiAlert {
return aa
}
func groupToAPI(g *rule.Group) *apiGroup {
func groupToAPI(g *rule.Group) apiGroup {
g = g.DeepCopy()
ag := apiGroup{
// encode as string to avoid rounding
@@ -363,7 +353,7 @@ func groupToAPI(g *rule.Group) *apiGroup {
for _, r := range g.Rules {
ag.Rules = append(ag.Rules, ruleToAPI(r))
}
return &ag
return ag
}
func urlValuesToStrings(values url.Values) []string {

View File

@@ -70,7 +70,7 @@ var (
Usage: "VictoriaMetrics address to perform import requests. \n" +
"Should be the same as --httpListenAddr value for single-node version or vminsert component. \n" +
"When importing into the clustered version do not forget to set additionally --vm-account-id flag. \n" +
"Please note, that vmctl performs initial readiness check for the given address by checking /health endpoint.",
"Please note, that `vmctl` performs initial readiness check for the given address by checking `/health` endpoint.",
},
&cli.StringFlag{
Name: vmUser,
@@ -514,27 +514,27 @@ var (
},
&cli.StringFlag{
Name: vmNativeSrcBearerToken,
Usage: "Optional bearer auth token to use for the corresponding --vm-native-src-addr",
Usage: "Optional bearer auth token to use for the corresponding `--vm-native-src-addr`",
},
&cli.StringFlag{
Name: vmNativeSrcCertFile,
Usage: "Optional path to client-side TLS certificate file to use when connecting to --vm-native-src-addr",
Usage: "Optional path to client-side TLS certificate file to use when connecting to `--vm-native-src-addr`",
},
&cli.StringFlag{
Name: vmNativeSrcKeyFile,
Usage: "Optional path to client-side TLS key to use when connecting to --vm-native-src-addr",
Usage: "Optional path to client-side TLS key to use when connecting to `--vm-native-src-addr`",
},
&cli.StringFlag{
Name: vmNativeSrcCAFile,
Usage: "Optional path to TLS CA file to use for verifying connections to --vm-native-src-addr. By default, system CA is used",
Usage: "Optional path to TLS CA file to use for verifying connections to `--vm-native-src-addr`. By default, system CA is used",
},
&cli.StringFlag{
Name: vmNativeSrcServerName,
Usage: "Optional TLS server name to use for connections to --vm-native-src-addr. By default, the server name from --vm-native-src-addr is used",
Usage: "Optional TLS server name to use for connections to `--vm-native-src-addr`. By default, the server name from `--vm-native-src-addr` is used",
},
&cli.BoolFlag{
Name: vmNativeSrcInsecureSkipVerify,
Usage: "Whether to skip TLS certificate verification when connecting to --vm-native-src-addr",
Usage: "Whether to skip TLS certificate verification when connecting to `--vm-native-src-addr`",
Value: false,
},
@@ -563,27 +563,27 @@ var (
},
&cli.StringFlag{
Name: vmNativeDstBearerToken,
Usage: "Optional bearer auth token to use for the corresponding --vm-native-dst-addr",
Usage: "Optional bearer auth token to use for the corresponding `--vm-native-dst-addr`",
},
&cli.StringFlag{
Name: vmNativeDstCertFile,
Usage: "Optional path to client-side TLS certificate file to use when connecting to --vm-native-dst-addr",
Usage: "Optional path to client-side TLS certificate file to use when connecting to `--vm-native-dst-addr`",
},
&cli.StringFlag{
Name: vmNativeDstKeyFile,
Usage: "Optional path to client-side TLS key to use when connecting to --vm-native-dst-addr",
Usage: "Optional path to client-side TLS key to use when connecting to `--vm-native-dst-addr`",
},
&cli.StringFlag{
Name: vmNativeDstCAFile,
Usage: "Optional path to TLS CA file to use for verifying connections to --vm-native-dst-addr. By default, system CA is used",
Usage: "Optional path to TLS CA file to use for verifying connections to `--vm-native-dst-addr`. By default, system CA is used",
},
&cli.StringFlag{
Name: vmNativeDstServerName,
Usage: "Optional TLS server name to use for connections to --vm-native-dst-addr. By default, the server name from --vm-native-dst-addr is used",
Usage: "Optional TLS server name to use for connections to `--vm-native-dst-addr`. By default, the server name from `--vm-native-dst-addr` is used",
},
&cli.BoolFlag{
Name: vmNativeDstInsecureSkipVerify,
Usage: "Whether to skip TLS certificate verification when connecting to --vm-native-dst-addr",
Usage: "Whether to skip TLS certificate verification when connecting to `--vm-native-dst-addr`",
Value: false,
},
@@ -597,7 +597,7 @@ var (
Name: vmRateLimit,
Usage: "Optional data transfer rate limit in bytes per second.\n" +
"By default, the rate limit is disabled. It can be useful for limiting load on source or destination databases. \n" +
"Rate limit is applied per worker, see --vm-concurrency.",
"Rate limit is applied per worker, see `--vm-concurrency`.",
},
&cli.BoolFlag{
Name: vmInterCluster,

View File

@@ -140,15 +140,6 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU
written, err := io.Copy(w, reader)
if err != nil {
// io.Copy could fail if ImportPipe will fail before and close the pr
// so we check if that's the case and to not ignore importErr if it exists.
select {
case importErr := <-importCh:
if importErr != nil {
return fmt.Errorf("failed to import %s: %w", p.dst.Addr, importErr)
}
default:
}
return fmt.Errorf("failed to write into %q: %s", p.dst.Addr, err)
}

View File

@@ -562,15 +562,6 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"status":"success","data":{"alerts":[]}}`)
return true
case "/api/v1/notifiers", "/notifiers":
notifiersRequests.Inc()
if len(*vmalertProxyURL) > 0 {
proxyVMAlertRequests(w, r)
return true
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"status":"success","data":{"notifiers":[]}}`)
return true
case "/api/v1/metadata":
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
metadataRequests.Inc()
@@ -700,10 +691,9 @@ var (
expandWithExprsRequests = metrics.NewCounter(`vm_http_requests_total{path="/expand-with-exprs"}`)
prettifyQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/prettify-query"}`)
vmalertRequests = metrics.NewCounter(`vm_http_requests_total{path="/vmalert"}`)
rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
notifiersRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/notifiers"}`)
vmalertRequests = metrics.NewCounter(`vm_http_requests_total{path="/vmalert"}`)
rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
buildInfoRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/buildinfo"}`)

View File

@@ -377,13 +377,11 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
preFunc := func(_ []float64, _ []int64) {}
funcName = strings.ToLower(funcName)
// window > lookbackDelta could result in negative delta.
// See issue: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8342
stalenessInterval := lookbackDelta
if stalenessInterval != 0 {
// If stalenessInterval was set, it should additionally account for [window] range to cover following cases:
// * window > stalenessInterval, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8342
// * window captures prevValue in doInternal while removeCounterResets does not,
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8935#issuecomment-3000735468
stalenessInterval += window
if stalenessInterval != 0 && stalenessInterval < window {
stalenessInterval = window
}
if rollupFuncsRemoveCounterResets[funcName] {

View File

@@ -742,23 +742,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
This function is supported by PromQL.
See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
#### rate_prometheus
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
See also [irate](#irate) and [rollup_rate](#rollup_rate).
#### rate_over_sum

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -36,10 +36,10 @@
<meta property="og:title" content="UI for VictoriaMetrics">
<meta property="og:url" content="https://victoriametrics.com/">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<script type="module" crossorigin src="./assets/index-BiQY-19a.js"></script>
<script type="module" crossorigin src="./assets/index-D-ssBbZq.js"></script>
<link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
<link rel="stylesheet" crossorigin href="./assets/index-ojCMu5lE.css">
<link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>

View File

@@ -63,8 +63,6 @@ var (
cacheSizeStorageTSID = flagutil.NewBytes("storage.cacheSizeStorageTSID", 0, "Overrides max size for storage/tsid cache. "+
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
cacheSizeStorageMetricName = flagutil.NewBytes("storage.cacheSizeStorageMetricName", 0, "Overrides max size for storage/metricName cache. "+
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
cacheSizeIndexDBIndexBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBIndexBlocks", 0, "Overrides max size for indexdb/indexBlocks cache. "+
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
cacheSizeIndexDBDataBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBDataBlocks", 0, "Overrides max size for indexdb/dataBlocks cache. "+
@@ -113,7 +111,6 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
storage.SetMetricNamesStatsCacheSize(cacheSizeMetricNamesStats.IntN())
storage.SetMetricNameCacheSize(cacheSizeStorageMetricName.IntN())
mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
mergeset.SetDataBlocksSparseCacheSize(cacheSizeIndexDBDataBlocksSparse.IntN())

View File

@@ -13,7 +13,7 @@
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
<link rel="manifest" href="/manifest.json"/>
<!--
Notice the use of in the tags above.
It will be replaced with the URL of the `public` folder during the build.

View File

@@ -2,9 +2,9 @@
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="/favicon.victorialogs.svg" />
<link rel="apple-touch-icon" href="/favicon.victorialogs.svg" />
<link rel="mask-icon" href="/favicon.victorialogs.svg" color="#000000">
<link rel="icon" href="/favicon.svg" />
<link rel="apple-touch-icon" href="/favicon.svg" />
<link rel="mask-icon" href="/favicon.svg" color="#000000">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
@@ -13,7 +13,7 @@
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
<link rel="manifest" href="/manifest.json"/>
<!--
Notice the use of in the tags above.
It will be replaced with the URL of the `public` folder during the build.

View File

@@ -13,7 +13,7 @@
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
<link rel="manifest" href="/manifest.json"/>
<!--
Notice the use of in the tags above.
It will be replaced with the URL of the `public` folder during the build.

View File

@@ -10,8 +10,6 @@
"dependencies": {
"@types/lodash.debounce": "^4.0.9",
"@types/lodash.get": "^4.4.9",
"@types/lodash.orderBy": "^4.6.9",
"@types/lodash.throttle": "^4.1.9",
"@types/qs": "^6.9.18",
"@types/react": "^19.1.2",
"@types/react-input-mask": "^3.0.6",
@@ -20,8 +18,6 @@
"dayjs": "^1.11.13",
"lodash.debounce": "^4.0.8",
"lodash.get": "^4.4.2",
"lodash.orderBy": "^4.6.0",
"lodash.throttle": "^4.1.1",
"marked": "^15.0.8",
"marked-emoji": "^2.0.0",
"preact": "^10.26.5",
@@ -2195,24 +2191,6 @@
"@types/lodash": "*"
}
},
"node_modules/@types/lodash.orderBy": {
"version": "4.6.9",
"resolved": "https://registry.npmjs.org/@types/lodash.orderby/-/lodash.orderby-4.6.9.tgz",
"integrity": "sha512-T9o2wkIJOmxXwVTPTmwJ59W6eTi2FseiLR369fxszG649Po/xe9vqFNhf/MtnvT5jrbDiyWKxPFPZbpSVK0SVQ==",
"license": "MIT",
"dependencies": {
"@types/lodash": "*"
}
},
"node_modules/@types/lodash.throttle": {
"version": "4.1.9",
"resolved": "https://registry.npmjs.org/@types/lodash.throttle/-/lodash.throttle-4.1.9.tgz",
"integrity": "sha512-PCPVfpfueguWZQB7pJQK890F2scYKoDUL3iM522AptHWn7d5NQmeS/LTEHIcLr5PaTzl3dK2Z0xSUHHTHwaL5g==",
"license": "MIT",
"dependencies": {
"@types/lodash": "*"
}
},
"node_modules/@types/node": {
"version": "22.14.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz",
@@ -5764,18 +5742,6 @@
"dev": true,
"license": "MIT"
},
"node_modules/lodash.orderBy": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/lodash.orderby/-/lodash.orderby-4.6.0.tgz",
"integrity": "sha512-T0rZxKmghOOf5YPnn8EY5iLYeWCpZq8G41FfqoVHH5QDTAFaghJRmAdLiadEDq+ztgM2q5PjA+Z1fOwGrLgmtg==",
"license": "MIT"
},
"node_modules/lodash.throttle": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz",
"integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==",
"license": "MIT"
},
"node_modules/loose-envify": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",

View File

@@ -7,8 +7,6 @@
"dependencies": {
"@types/lodash.debounce": "^4.0.9",
"@types/lodash.get": "^4.4.9",
"@types/lodash.orderBy": "^4.6.9",
"@types/lodash.throttle": "^4.1.9",
"@types/qs": "^6.9.18",
"@types/react": "^19.1.2",
"@types/react-input-mask": "^3.0.6",
@@ -17,8 +15,6 @@
"dayjs": "^1.11.13",
"lodash.debounce": "^4.0.8",
"lodash.get": "^4.4.2",
"lodash.orderBy": "^4.6.0",
"lodash.throttle": "^4.1.1",
"marked": "^15.0.8",
"marked-emoji": "^2.0.0",
"preact": "^10.26.5",

View File

@@ -1,5 +0,0 @@
<svg width="48" height="48" fill="#e94600" xmlns="http://www.w3.org/2000/svg">
<path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0"/>
<path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z"/>
<path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z"/>
</svg>

Before

Width:  |  Height:  |  Size: 1.3 KiB

View File

@@ -742,23 +742,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
This function is supported by PromQL.
See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
#### rate_prometheus
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
See also [irate](#irate) and [rollup_rate](#rollup_rate).
#### rate_over_sum

View File

@@ -1,4 +1,4 @@
import { FC } from "preact/compat";
import React, { FC } from "preact/compat";
import classNames from "classnames";
import { MouseEvent as ReactMouseEvent, ReactNode } from "react";
import "./style.scss";
@@ -14,7 +14,6 @@ interface ButtonProps {
disabled?: boolean
children?: ReactNode
className?: string
"data-id"?: string
onClick?: (e: ReactMouseEvent<HTMLButtonElement>) => void
onMouseDown?: (e: ReactMouseEvent<HTMLButtonElement>) => void
}
@@ -32,7 +31,6 @@ const Button: FC<ButtonProps> = ({
disabled,
onClick,
onMouseDown,
"data-id": dataId
}) => {
const classesButton = classNames({
@@ -52,7 +50,6 @@ const Button: FC<ButtonProps> = ({
aria-label={ariaLabel}
onClick={onClick}
onMouseDown={onMouseDown}
data-id={dataId}
>
{startIcon}{children}{endIcon}
</button>

View File

@@ -656,33 +656,3 @@ export const ScrollToTopIcon = () => (
/>
</svg>
);
export const SortIcon = () => (
<svg
viewBox="0 0 24 24"
fill="currentColor"
>
<path d="M4 3 L4 15 L1.5 15 L5.5 21 L9.5 15 L7 15 L7 3 Z"/>
<path d="M13 21 L13 9 L10.5 9 L14.5 3 L18.5 9 L16 9 L16 21 Z"/>
</svg>
);
export const SortArrowDownIcon = () => (
<svg
viewBox="0 0 24 24"
fill="currentColor"
>
<path d="M10.5 3 L10.5 15 L8 15 L12 21 L16 15 L13.5 15 L13.5 3 Z"/>
</svg>
);
export const SortArrowUpIcon = () => (
<svg
viewBox="0 0 24 24"
fill="currentColor"
>
<path d="M10.5 21 L10.5 9 L8 9 L12 3 L16 9 L13.5 9 L13.5 21 Z"/>
</svg>
);

View File

@@ -18,8 +18,8 @@ const title = "Table settings";
interface TableSettingsProps {
columns: string[];
selectedColumns?: string[];
tableCompact?: boolean;
toggleTableCompact?: () => void;
tableCompact: boolean;
toggleTableCompact: () => void;
onChangeColumns: (arr: string[]) => void
}
@@ -195,20 +195,18 @@ const TableSettings: FC<TableSettingsProps> = ({
</div>
</div>
</div>
{toggleTableCompact && tableCompact !== undefined && (
<div className="vm-table-settings-modal-section">
<div className="vm-table-settings-modal-section__title">
<div className="vm-table-settings-modal-section">
<div className="vm-table-settings-modal-section__title">
Table view
</div>
<div className="vm-table-settings-modal-columns-list__item">
<Switch
label={"Compact view"}
value={tableCompact}
onChange={toggleTableCompact}
/>
</div>
</div>
)}
<div className="vm-table-settings-modal-columns-list__item">
<Switch
label={"Compact view"}
value={tableCompact}
onChange={toggleTableCompact}
/>
</div>
</div>
</Modal>)}
</div>
);

View File

@@ -5,7 +5,7 @@
&-header {
background-color: $color-background-block;
z-index: 3;
z-index: 1;
margin: -$padding-medium 0-$padding-medium 0;
position: sticky;
top: 0;

View File

@@ -1,67 +1,21 @@
import React, { FC, useMemo, useCallback, createPortal } from "preact/compat";
import React, { FC } from "preact/compat";
import DownloadLogsButton from "../../../DownloadLogsButton/DownloadLogsButton";
import { createPortal } from "preact/compat";
import JsonViewComponent from "../../../../../components/Views/JsonView/JsonView";
import { ViewProps } from "../../types";
import EmptyLogs from "../components/EmptyLogs/EmptyLogs";
import JsonViewSettings from "./JsonViewSettings/JsonViewSettings";
import { useSearchParams } from "react-router-dom";
import orderBy from "lodash.orderBy";
import "./style.scss";
import { Logs } from "../../../../../api/types";
import { SortDirection } from "./types";
import { useCallback } from "react";
const MemoizedJsonView = React.memo(JsonViewComponent);
const jsonQuerySortParam = "json_sort";
const fieldSortQueryParamName = "json_field_sort";
const JsonView: FC<ViewProps> = ({ data, settingsRef }) => {
const getLogs = useCallback(() => data, [data]);
const [searchParams] = useSearchParams();
const sortParam = searchParams.get(jsonQuerySortParam);
const fieldSortParam = searchParams.get(fieldSortQueryParamName) as SortDirection;
const [sortField, sortDirection] = useMemo(() => {
const [sortField, sortDirection] = sortParam?.split(":").map(decodeURIComponent) || [];
return [sortField, sortDirection as "asc" | "desc" | undefined];
}, [sortParam]);
const fields = useMemo(() => {
const keys = new Set(data.flatMap(Object.keys));
return Array.from(keys);
}, [data]);
const orderedFieldsData = useMemo(() => {
if (!fieldSortParam) return data;
const orderedFields = fields.toSorted((a, b) => fieldSortParam === "asc" ? a.localeCompare(b): b.localeCompare(a));
return data.map((item) => {
return orderedFields.reduce((acc, field) => {
if (item[field]) acc[field] = item[field];
return acc;
}, {} as Logs);
});
}, [fields, fieldSortParam, data]);
const sortedData = useMemo(() => {
if (!sortField || !sortDirection) return orderedFieldsData;
return orderBy(orderedFieldsData, [sortField], [sortDirection]);
}, [orderedFieldsData, sortField, sortDirection]);
const renderSettings = () => {
if (!settingsRef.current) return null;
return createPortal(
data.length > 0 && (
<div className="vm-json-view__settings-container">
<DownloadLogsButton getLogs={getLogs} />
<JsonViewSettings
fields={fields}
sortQueryParamName={jsonQuerySortParam}
fieldSortQueryParamName={fieldSortQueryParamName}
/>
</div>
),
data.length > 0 && <DownloadLogsButton getLogs={getLogs} />,
settingsRef.current
);
};
@@ -71,11 +25,9 @@ const JsonView: FC<ViewProps> = ({ data, settingsRef }) => {
return (
<>
{renderSettings()}
<MemoizedJsonView
data={sortedData}
/>
<MemoizedJsonView data={data} />
</>
);
};
export default JsonView;
export default JsonView;

View File

@@ -1,185 +0,0 @@
import { FC, useMemo, useRef } from "preact/compat";
import Button from "../../../../../../components/Main/Button/Button";
import { SettingsIcon, SortArrowDownIcon, SortArrowUpIcon, SortIcon } from "../../../../../../components/Main/Icons";
import Tooltip from "../../../../../../components/Main/Tooltip/Tooltip";
import Select from "../../../../../../components/Main/Select/Select";
import useBoolean from "../../../../../../hooks/useBoolean";
import { useState, useEffect, useCallback } from "react";
import Modal from "../../../../../../components/Main/Modal/Modal";
import { useSearchParams } from "react-router-dom";
import "./style.scss";
import { SortDirection } from "../types";
const title = "JSON settings";
const directionList = ["asc", "desc"];
interface JsonSettingsProps {
fields: string[];
sortQueryParamName: string;
fieldSortQueryParamName: string;
}
const JsonViewSettings: FC<JsonSettingsProps> = ({
fields,
sortQueryParamName,
fieldSortQueryParamName
}) => {
const [searchParams, setSearchParams] = useSearchParams();
const buttonRef = useRef<HTMLDivElement>(null);
const [fieldSortDirection, setFieldSortDirection] = useState<SortDirection>(null);
const {
value: openSettings,
toggle: toggleOpenSettings,
setFalse: handleClose,
} = useBoolean(false);
const [sortField, setSortField] = useState<string | null>(null);
const [sortDirection, setSortDirection] = useState<SortDirection>(null);
useEffect(() => {
const sortParam = searchParams.get(sortQueryParamName);
const isSortDirection = (value: string) : value is Exclude<SortDirection, null> => directionList.includes(value);
if (sortParam) {
const [field, direction] = sortParam.split(":").map(decodeURIComponent);
if (field && (isSortDirection(direction))) {
setSortField(field);
setSortDirection(direction);
}
}
const fieldSortParam = searchParams.get(fieldSortQueryParamName);
if (fieldSortParam === "asc" || fieldSortParam === "desc") {
setFieldSortDirection(fieldSortParam);
}
}, [searchParams, sortQueryParamName, fieldSortQueryParamName, setSortField, setSortDirection, setFieldSortDirection]);
const updateSortParams = useCallback((field: string | null, direction: SortDirection) => {
const updatedParams = new URLSearchParams(searchParams.toString());
if (!field || !direction) {
updatedParams.delete(sortQueryParamName);
} else {
updatedParams.set(sortQueryParamName, `${field}:${direction || ""}`);
}
setSearchParams(updatedParams);
}, [searchParams, sortQueryParamName]);
const handleSort = (field: string) => {
const newDirection: SortDirection = sortDirection || "asc";
setSortField(field);
setSortDirection(newDirection);
updateSortParams(field, newDirection);
};
const resetSort = () => {
setSortField(null);
setSortDirection(null);
updateSortParams(null, null);
};
const changeFieldSortDirection = useCallback(() => {
let newFieldSortDirection: SortDirection = null;
if (fieldSortDirection === null) {
newFieldSortDirection = "asc";
}else if (fieldSortDirection === "asc") {
newFieldSortDirection = "desc";
}
setFieldSortDirection(newFieldSortDirection);
const updatedParams = new URLSearchParams(searchParams.toString());
if (!newFieldSortDirection) {
updatedParams.delete(fieldSortQueryParamName);
} else {
updatedParams.set(fieldSortQueryParamName, encodeURIComponent(newFieldSortDirection));
}
setSearchParams(updatedParams);
},[fieldSortDirection, searchParams, fieldSortQueryParamName]);
const handleChangeSortDirection = (direction: string) => {
const field = sortField || fields[0];
setSortField(field);
setSortDirection(direction as SortDirection);
updateSortParams(field, direction as SortDirection);
};
const fieldSortMeta = useMemo(() => ({
default: {
title: "Set field sort order. Click to sort in ascending order",
icon: <SortIcon />
},
asc: {
title: "Fields sorted ascending. Click to sort in descending order",
icon: <SortArrowDownIcon />
},
desc: {
title: "Fields sorted descending. Click to reset sort",
icon: <SortArrowUpIcon />
},
}), []);
const fieldSortButton = useMemo(() => {
const { title, icon } = fieldSortMeta[fieldSortDirection ?? "default"];
return <Tooltip title={title}>
<Button
variant="text"
startIcon={icon}
onClick={changeFieldSortDirection}
ariaLabel={title}
/>
</Tooltip>;
}, [fieldSortDirection, toggleOpenSettings, changeFieldSortDirection, fieldSortMeta]);
return (
<div className="vm-json-settings">
{fieldSortButton}
<Tooltip title={title}>
<div ref={buttonRef}>
<Button
variant="text"
startIcon={<SettingsIcon/>}
onClick={toggleOpenSettings}
ariaLabel={title}
/>
</div>
</Tooltip>
{openSettings && (
<Modal
title={title}
className="vm-json-settings-modal"
onClose={handleClose}
>
<div className="vm-json-settings-modal-section">
<div className="vm-json-settings-modal-section__sort-settings-container">
<Select
value={sortField || ""}
onChange={handleSort}
list={fields}
label="Select field"
/>
<Select
value={sortDirection || ""}
onChange={handleChangeSortDirection}
list={directionList}
label="Sort direction"
/>
{(sortField || sortDirection) && (
<Button
variant="outlined"
color="error"
onClick={resetSort}
>
Reset sort
</Button>
)}
</div>
</div>
</Modal>)}
</div>
);
};
export default JsonViewSettings;

View File

@@ -1,34 +0,0 @@
@use "src/styles/variables" as *;
.vm-json-settings {
display: flex;
flex-direction: row;
&-modal {
.vm-modal-content-body {
min-width: clamp(300px, 600px, 90vw);
padding: 0;
}
&-section {
padding-block: $padding-global;
border-top: $border-divider;
&:first-child {
padding-top: 0;
border-top: none;
}
&__sort-settings-container {
display: grid;
padding: $padding-medium;
grid-template-columns: 1fr 1fr 80px;
gap: $padding-medium;
@media (max-width: 500px) {
grid-template-columns: 1fr;
}
}
}
}
}

View File

@@ -1,9 +0,0 @@
@use "src/styles/variables" as *;
.vm-json-view {
&__settings-container {
display: flex;
flex-direction: row;
align-items: center;
}
}

View File

@@ -1 +0,0 @@
export type SortDirection = "asc" | "desc" | null;

View File

@@ -7,7 +7,7 @@ import { useLiveTailingLogs } from "./useLiveTailingLogs";
import { LOGS_DISPLAY_FIELDS, LOGS_URL_PARAMS } from "../../../../../constants/logs";
import { useMemo } from "react";
import { useSearchParams } from "react-router-dom";
import throttle from "lodash.throttle";
import throttle from "lodash/throttle";
import GroupLogsItem from "../../../GroupLogs/GroupLogsItem";
import LiveTailingSettings from "./LiveTailingSettings";
import Alert from "../../../../../components/Main/Alert/Alert";

View File

@@ -3,6 +3,7 @@ import DownloadLogsButton from "../../../DownloadLogsButton/DownloadLogsButton";
import { createPortal } from "preact/compat";
import "./style.scss";
import { ViewProps } from "../../types";
import useBoolean from "../../../../../hooks/useBoolean";
import useStateSearchParams from "../../../../../hooks/useStateSearchParams";
import TableLogs from "../../TableLogs";
import SelectLimit from "../../../../../components/Main/Pagination/SelectLimit/SelectLimit";
@@ -17,6 +18,7 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
const { setSearchParamsFromKeys } = useSearchParamsFromObject();
const [displayColumns, setDisplayColumns] = useState<string[]>([]);
const [rowsPerPage, setRowsPerPage] = useStateSearchParams(100, "rows_per_page");
const { value: tableCompact, toggle: toggleTableCompact } = useBoolean(false);
const columns = useMemo(() => {
const keys = new Set<string>();
@@ -50,6 +52,8 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
columns={columns}
selectedColumns={displayColumns}
onChangeColumns={setDisplayColumns}
tableCompact={tableCompact}
toggleTableCompact={toggleTableCompact}
/>
</div>
</div>,
@@ -65,7 +69,7 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
<MemoizedTableView
logs={data}
displayColumns={displayColumns}
tableCompact={false}
tableCompact={tableCompact}
columns={columns}
rowsPerPage={Number(rowsPerPage)}
/>

View File

@@ -18,22 +18,22 @@ import (
pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
)
// PrometheusQuerier contains methods available to Prometheus-like HTTP API for Querying
type PrometheusQuerier interface {
PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse
PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
// APIQuerier contains methods available to Prometheus-like HTTP API for Querying
type APIQuerier interface {
APIV1Export(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse
APIV1Query(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse
APIV1QueryRange(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse
APIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *APIV1SeriesResponse
APIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
}
// Writer contains methods for writing new data
type Writer interface {
// Prometheus APIs
PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts)
PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts)
PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts)
// APIWriter contains methods for writing new data
type APIWriter interface {
// Prometheus-like APIs
APIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts)
APIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
APIV1ImportCSV(t *testing.T, records []string, opts QueryOpts)
APIV1ImportNative(t *testing.T, data []byte, opts QueryOpts)
// Graphit APIs
GraphiteWrite(t *testing.T, records []string, opts QueryOpts)
@@ -54,11 +54,11 @@ type StorageMerger interface {
ForceMerge(t *testing.T)
}
// PrometheusWriteQuerier encompasses the methods for writing, flushing and
// WriteQuerier encompasses the methods for writing, flushing and
// querying the data.
type PrometheusWriteQuerier interface {
Writer
PrometheusQuerier
type WriteQuerier interface {
APIWriter
APIQuerier
StorageFlusher
StorageMerger
}
@@ -138,9 +138,9 @@ func (qos *QueryOptsLogs) asURLValues() url.Values {
return uv
}
// PrometheusAPIV1QueryResponse is an inmemory representation of the
// APIV1QueryResponse is an inmemory representation of the
// /prometheus/api/v1/query or /prometheus/api/v1/query_range response.
type PrometheusAPIV1QueryResponse struct {
type APIV1QueryResponse struct {
Status string
Data *QueryData
ErrorType string
@@ -148,12 +148,12 @@ type PrometheusAPIV1QueryResponse struct {
IsPartial bool
}
// NewPrometheusAPIV1QueryResponse is a test helper function that creates a new
// instance of PrometheusAPIV1QueryResponse by unmarshalling a json string.
func NewPrometheusAPIV1QueryResponse(t *testing.T, s string) *PrometheusAPIV1QueryResponse {
// NewAPIV1QueryResponse is a test helper function that creates a new
// instance of APIV1QueryResponse by unmarshalling a json string.
func NewAPIV1QueryResponse(t *testing.T, s string) *APIV1QueryResponse {
t.Helper()
res := &PrometheusAPIV1QueryResponse{}
res := &APIV1QueryResponse{}
if err := json.Unmarshal([]byte(s), res); err != nil {
t.Fatalf("could not unmarshal query response data=\n%s\n: %v", string(s), err)
}
@@ -161,7 +161,7 @@ func NewPrometheusAPIV1QueryResponse(t *testing.T, s string) *PrometheusAPIV1Que
}
// Sort performs data.Result sort by metric labels
func (pqr *PrometheusAPIV1QueryResponse) Sort() {
func (pqr *APIV1QueryResponse) Sort() {
if pqr.Data == nil {
return
}
@@ -257,9 +257,9 @@ func (s *Sample) UnmarshalJSON(b []byte) error {
return nil
}
// PrometheusAPIV1SeriesResponse is an inmemory representation of the
// APIV1SeriesResponse is an inmemory representation of the
// /prometheus/api/v1/series response.
type PrometheusAPIV1SeriesResponse struct {
type APIV1SeriesResponse struct {
Status string
IsPartial bool
Data []map[string]string
@@ -268,12 +268,12 @@ type PrometheusAPIV1SeriesResponse struct {
Error string
}
// NewPrometheusAPIV1SeriesResponse is a test helper function that creates a new
// instance of PrometheusAPIV1SeriesResponse by unmarshalling a json string.
func NewPrometheusAPIV1SeriesResponse(t *testing.T, s string) *PrometheusAPIV1SeriesResponse {
// NewAPIV1SeriesResponse is a test helper function that creates a new
// instance of APIV1SeriesResponse by unmarshalling a json string.
func NewAPIV1SeriesResponse(t *testing.T, s string) *APIV1SeriesResponse {
t.Helper()
res := &PrometheusAPIV1SeriesResponse{}
res := &APIV1SeriesResponse{}
if err := json.Unmarshal([]byte(s), res); err != nil {
t.Fatalf("could not unmarshal series response data:\n%s\n err: %v", string(s), err)
}
@@ -281,7 +281,7 @@ func NewPrometheusAPIV1SeriesResponse(t *testing.T, s string) *PrometheusAPIV1Se
}
// Sort sorts the response data.
func (r *PrometheusAPIV1SeriesResponse) Sort() *PrometheusAPIV1SeriesResponse {
func (r *APIV1SeriesResponse) Sort() *APIV1SeriesResponse {
str := func(m map[string]string) string {
s := []string{}
for k, v := range m {

View File

@@ -146,7 +146,7 @@ func (tc *TestCase) MustStartVmagent(instance string, flags []string, promScrape
// Vmcluster represents a typical cluster setup: several vmstorage replicas, one
// vminsert, and one vmselect.
//
// Both Vmsingle and Vmcluster implement the PrometheusWriteQuerier used in
// Both Vmsingle and Vmcluster implement the WriteQuerier used in
// business logic tests to abstract out the infrasture.
//
// This type is not suitable for infrastructure tests where custom cluster
@@ -309,8 +309,8 @@ func (tc *TestCase) StopApp(instance string) {
}
}
// StopPrometheusWriteQuerier stop all apps that are a part of the pwq.
func (tc *TestCase) StopPrometheusWriteQuerier(pwq PrometheusWriteQuerier) {
// StopWriteQuerier stop all apps that are a part of the pwq.
func (tc *TestCase) StopWriteQuerier(pwq WriteQuerier) {
tc.t.Helper()
switch t := pwq.(type) {
case *Vmsingle:
@@ -433,24 +433,3 @@ func (tc *TestCase) MustStartVlsingle(instance string, flags []string) *Vlsingle
tc.addApp(instance, app)
return app
}
// MustStartDefaultVlagent is a test helper function that starts an instance of
// vlagent with defaults suitable for most tests.
func (tc *TestCase) MustStartDefaultVlagent(remoteWriteURLs []string) *Vlagent {
tc.t.Helper()
return tc.MustStartVlagent("vlagent", remoteWriteURLs, nil)
}
// MustStartVlagent is a test helper function that starts an instance of
// vlagent and fails the test if the app fails to start.
func (tc *TestCase) MustStartVlagent(instance string, remoteWriteURLs []string, flags []string) *Vlagent {
tc.t.Helper()
app, err := StartVlagent(instance, remoteWriteURLs, flags, tc.cli)
if err != nil {
tc.t.Fatalf("Could not start %s: %v", instance, err)
}
tc.addApp(instance, app)
return app
}

View File

@@ -11,10 +11,10 @@ import (
)
type testBackupRestoreOpts struct {
startSUT func() at.PrometheusWriteQuerier
startSUT func() at.WriteQuerier
stopSUT func()
storageDataPaths []string
snapshotCreateURLs func(at.PrometheusWriteQuerier) []string
snapshotCreateURLs func(at.WriteQuerier) []string
}
func TestSingleBackupRestore(t *testing.T) {
@@ -24,7 +24,7 @@ func TestSingleBackupRestore(t *testing.T) {
storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
opts := testBackupRestoreOpts{
startSUT: func() at.PrometheusWriteQuerier {
startSUT: func() at.WriteQuerier {
return tc.MustStartVmsingle("vmsingle", []string{
"-storageDataPath=" + storageDataPath,
"-retentionPeriod=100y",
@@ -37,7 +37,7 @@ func TestSingleBackupRestore(t *testing.T) {
storageDataPaths: []string{
storageDataPath,
},
snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
snapshotCreateURLs: func(sut at.WriteQuerier) []string {
return []string{
sut.(*at.Vmsingle).SnapshotCreateURL(),
}
@@ -55,7 +55,7 @@ func TestClusterBackupRestore(t *testing.T) {
storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
opts := testBackupRestoreOpts{
startSUT: func() at.PrometheusWriteQuerier {
startSUT: func() at.WriteQuerier {
return tc.MustStartCluster(&at.ClusterOptions{
Vmstorage1Instance: "vmstorage1",
Vmstorage1Flags: []string{
@@ -85,7 +85,7 @@ func TestClusterBackupRestore(t *testing.T) {
storage1DataPath,
storage2DataPath,
},
snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
snapshotCreateURLs: func(sut at.WriteQuerier) []string {
c := sut.(*at.Vmcluster)
return []string{
c.Vmstorages[0].SnapshotCreateURL(),
@@ -127,18 +127,18 @@ func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
// assertSeries retrieves set of all metric names from the storage and
// compares it with the expected set.
assertSeries := func(app at.PrometheusQuerier, query string, start, end int64, want []map[string]string) {
assertSeries := func(app at.APIQuerier, query string, start, end int64, want []map[string]string) {
t.Helper()
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
return app.APIV1Series(t, query, at.QueryOpts{
Start: fmt.Sprintf("%d", start),
End: fmt.Sprintf("%d", end),
}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Want: &at.APIV1SeriesResponse{
Status: "success",
Data: want,
},
@@ -148,18 +148,18 @@ func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
// assertSeries retrieves all data from the storage and compares it with the
// expected result.
assertQueryResults := func(app at.PrometheusQuerier, query string, start, end int64, want []*at.QueryResult) {
assertQueryResults := func(app at.APIQuerier, query string, start, end int64, want []*at.QueryResult) {
t.Helper()
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
return app.APIV1QueryRange(t, query, at.QueryOpts{
Start: fmt.Sprintf("%d", start),
End: fmt.Sprintf("%d", end),
Step: "60s",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -171,7 +171,7 @@ func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
})
}
createBackup := func(sut at.PrometheusWriteQuerier, name string) {
createBackup := func(sut at.WriteQuerier, name string) {
for i, storageDataPath := range opts.storageDataPaths {
replica := fmt.Sprintf("replica-%d", i)
instance := fmt.Sprintf("vmbackup-%s-%s", name, replica)
@@ -216,13 +216,13 @@ func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
sut := opts.startSUT()
sut.PrometheusAPIV1ImportPrometheus(t, batch1Data, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, batch1Data, at.QueryOpts{})
sut.ForceFlush(t)
assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
createBackup(sut, "batch1")
sut.PrometheusAPIV1ImportPrometheus(t, batch2Data, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, batch2Data, at.QueryOpts{})
sut.ForceFlush(t)
assertSeries(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12Series)
assertQueryResults(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12QueryResults)

View File

@@ -5,12 +5,13 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
)
func TestSingleDeduplication_dedulicationIsOff(t *testing.T) {
@@ -80,7 +81,7 @@ func TestClusterDeduplication_deduplicationIsOn(t *testing.T) {
}
// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication
func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplicationIsOn bool) {
func testDeduplication(tc *at.TestCase, sut at.WriteQuerier, deduplicationIsOn bool) {
t := tc.T()
firstDayOfThisMonth := func() time.Time {
@@ -134,11 +135,11 @@ func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplica
},
}
sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
sut.APIV1Write(t, data, apptest.QueryOpts{})
sut.ForceFlush(t)
sut.ForceMerge(t)
wantDuplicates := &at.PrometheusAPIV1QueryResponse{
wantDuplicates := &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -166,7 +167,7 @@ func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplica
},
},
}
wantDeduped := &at.PrometheusAPIV1QueryResponse{
wantDeduped := &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -207,7 +208,7 @@ func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplica
tc.Assert(&at.AssertOptions{
Msg: "unexpected response",
Got: func() any {
got := sut.PrometheusAPIV1Export(t, `{__name__=~"metric.*"}`, apptest.QueryOpts{
got := sut.APIV1Export(t, `{__name__=~"metric.*"}`, apptest.QueryOpts{
ReduceMemUsage: "1",
Start: fmt.Sprintf("%d", start.UnixMilli()),
End: fmt.Sprintf("%d", end.UnixMilli()),

View File

@@ -33,9 +33,9 @@ func TestClusterExportImportNative(t *testing.T) {
// testExportImportNative test export and import in VictoriaMetrics native format.
// see: https://docs.victoriametrics.com/#how-to-import-data-in-native-format
func testExportImportNative(t *testing.T, sut at.PrometheusWriteQuerier) {
func testExportImportNative(t *testing.T, sut at.WriteQuerier) {
// create test data
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`native_export_import 10 1707123456700`, // 2024-02-05T08:57:36.700Z
}, at.QueryOpts{
ExtraLabels: []string{"el1=elv1", "el2=elv2"},
@@ -43,27 +43,27 @@ func testExportImportNative(t *testing.T, sut at.PrometheusWriteQuerier) {
sut.ForceFlush(t)
// export test data via native export API
exportResult := sut.PrometheusAPIV1ExportNative(t, "native_export_import", at.QueryOpts{
exportResult := sut.APIV1ExportNative(t, "native_export_import", at.QueryOpts{
Start: "2024-02-05T08:50:00.700Z",
End: "2024-02-05T09:00:00.700Z",
})
// re-import test data via native import API
sut.PrometheusAPIV1ImportNative(t, exportResult, at.QueryOpts{})
sut.APIV1ImportNative(t, exportResult, at.QueryOpts{})
sut.ForceFlush(t)
// check query result
got := sut.PrometheusAPIV1QueryRange(t, "native_export_import", at.QueryOpts{
got := sut.APIV1QueryRange(t, "native_export_import", at.QueryOpts{
Start: "2024-02-05T08:57:36.700Z",
End: "2024-02-05T08:57:36.700Z",
Step: "60s",
})
cmpOptions := []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.EquateNaNs(),
}
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "native_export_import", "el1": "elv1", "el2":"elv2"}, "values": []}]}}`)
want := at.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "native_export_import", "el1": "elv1", "el2":"elv2"}, "values": []}]}}`)
want.Data.Result[0].Samples = []*at.Sample{
at.NewSample(t, "2024-02-05T08:57:36.700Z", 10),
}

View File

@@ -22,7 +22,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
wantMetrics []map[string]string
wantSamples []*at.Sample
}
f := func(sut at.PrometheusQuerier, opts *opts) {
f := func(sut at.APIQuerier, opts *opts) {
t.Helper()
wantResult := []*at.QueryResult{}
for idx, wm := range opts.wantMetrics {
@@ -35,16 +35,16 @@ func TestSingleIngestionProtocols(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /export query response",
Got: func() any {
got := sut.PrometheusAPIV1Export(t, opts.query, at.QueryOpts{
got := sut.APIV1Export(t, opts.query, at.QueryOpts{
Start: "2024-02-05T08:50:00.700Z",
End: "2024-02-05T09:00:00.700Z",
})
got.Sort()
return got
},
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}
@@ -109,7 +109,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
})
// CSV import
sut.PrometheusAPIV1ImportCSV(t, []string{
sut.APIV1ImportCSV(t, []string{
`GOOG,1.23,4.56,NYSE,1707123457`,
`MSFT,23,56,NASDAQ,1707123457`,
}, at.QueryOpts{
@@ -158,7 +158,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
})
// prometheus text exposition format
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`importprometheus_series 10 1707123456700`, // 2024-02-05T08:57:36.700Z
`importprometheus_series2{label="foo",label1="value1"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
}, at.QueryOpts{
@@ -227,7 +227,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
},
},
}
sut.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
sut.APIV1Write(t, pbData, at.QueryOpts{})
sut.ForceFlush(t)
f(sut, &opts{
query: `{__name__=~"prometheusrw.+"}`,
@@ -282,22 +282,22 @@ func TestClusterIngestionProtocols(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /export query response",
Got: func() any {
got := vmselect.PrometheusAPIV1Export(t, opts.query, at.QueryOpts{
got := vmselect.APIV1Export(t, opts.query, at.QueryOpts{
Start: "2024-02-05T08:50:00.700Z",
End: "2024-02-05T09:00:00.700Z",
})
got.Sort()
return got
},
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}
// prometheus text exposition format
vminsert.PrometheusAPIV1ImportPrometheus(t, []string{
vminsert.APIV1ImportPrometheus(t, []string{
`importprometheus_series 10 1707123456700`, // 2024-02-05T08:57:36.700Z
`importprometheus_series2{label="foo",label1="value1"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
}, at.QueryOpts{
@@ -358,7 +358,7 @@ func TestClusterIngestionProtocols(t *testing.T) {
})
// CSV import
vminsert.PrometheusAPIV1ImportCSV(t, []string{
vminsert.APIV1ImportCSV(t, []string{
`GOOG,1.23,4.56,NYSE,1707123457`, // 2024-02-05T08:57:37.000Z
`MSFT,23,56,NASDAQ,1707123457`, // 2024-02-05T08:57:37.000Z
}, at.QueryOpts{
@@ -474,7 +474,7 @@ func TestClusterIngestionProtocols(t *testing.T) {
},
},
}
vminsert.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
vminsert.APIV1Write(t, pbData, at.QueryOpts{})
vmstorage.ForceFlush(t)
f(&opts{
query: `{__name__=~"prometheusrw.+"}`,

View File

@@ -55,10 +55,10 @@ func TestClusterKeyConceptsQueryData(t *testing.T) {
}
// testKeyConceptsQueryData verifies cases from https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data
func testKeyConceptsQueryData(t *testing.T, sut at.PrometheusWriteQuerier) {
func testKeyConceptsQueryData(t *testing.T, sut at.WriteQuerier) {
// Insert example data from documentation.
sut.PrometheusAPIV1ImportPrometheus(t, docData, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, docData, at.QueryOpts{})
sut.ForceFlush(t)
testInstantQuery(t, sut)
@@ -69,14 +69,14 @@ func testKeyConceptsQueryData(t *testing.T, sut at.PrometheusWriteQuerier) {
// testInstantQuery verifies the statements made in the `Instant query` section
// of the VictoriaMetrics documentation. See:
// https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query
func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
func testInstantQuery(t *testing.T, q at.APIQuerier) {
// Get the value of the foo_bar time series at 2022-05-10T08:03:00Z with the
// step of 5m and timeout 5s. There is no sample at exactly this timestamp.
// Therefore, VictoriaMetrics will search for the nearest sample within the
// [time-5m..time] interval.
got := q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:03:00.000Z", Step: "5m"})
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
opt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
got := q.APIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:03:00.000Z", Step: "5m"})
want := at.NewAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
opt := cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType")
if diff := cmp.Diff(want, got, opt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
@@ -86,7 +86,7 @@ func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
// Therefore, VictoriaMetrics will search for the nearest sample within the
// [time-1m..time] interval. Since the nearest sample is 2m away and the
// step is 1m, then the VictoriaMetrics must return empty response.
got = q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:18:00.000Z", Step: "1m"})
got = q.APIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:18:00.000Z", Step: "1m"})
if len(got.Data.Result) > 0 {
t.Errorf("unexpected response: got non-empty result, want empty result:\n%v", got)
}
@@ -95,14 +95,14 @@ func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
// testRangeQuery verifies the statements made in the `Range query` section of
// the VictoriaMetrics documentation. See:
// https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query
func testRangeQuery(t *testing.T, q at.PrometheusQuerier) {
func testRangeQuery(t *testing.T, q at.APIQuerier) {
f := func(start, end, step string, wantSamples []*at.Sample) {
t.Helper()
got := q.PrometheusAPIV1QueryRange(t, "foo_bar", at.QueryOpts{Start: start, End: end, Step: step})
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
got := q.APIV1QueryRange(t, "foo_bar", at.QueryOpts{Start: start, End: end, Step: step})
want := at.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
want.Data.Result[0].Samples = wantSamples
opt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
opt := cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType")
if diff := cmp.Diff(want, got, opt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
@@ -168,11 +168,11 @@ func testRangeQuery(t *testing.T, q at.PrometheusQuerier) {
// will not produce ephemeral points.
//
// See: https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query
func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.PrometheusQuerier) {
func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.APIQuerier) {
f := func(timestamp string, want *at.Sample) {
t.Helper()
gotInstant := q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: timestamp, Step: "1m"})
gotInstant := q.APIV1Query(t, "foo_bar", at.QueryOpts{Time: timestamp, Step: "1m"})
if want == nil {
if got, want := len(gotInstant.Data.Result), 0; got != want {
t.Errorf("unexpected instant result size: got %d, want %d", got, want)
@@ -185,7 +185,7 @@ func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.Prometheu
}
}
rangeRes := q.PrometheusAPIV1QueryRange(t, "foo_bar", at.QueryOpts{
rangeRes := q.APIV1QueryRange(t, "foo_bar", at.QueryOpts{
Start: "2022-05-10T07:59:00.000Z",
End: "2022-05-10T08:17:00.000Z",
Step: "1m",
@@ -231,7 +231,7 @@ func TestClusterMillisecondPrecisionInInstantQueries(t *testing.T) {
testMillisecondPrecisionInInstantQueries(tc, sut)
}
func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
type opts struct {
@@ -242,7 +242,7 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
wantSample *at.Sample
wantSamples []*at.Sample
}
f := func(sut at.PrometheusQuerier, opts *opts) {
f := func(sut at.APIQuerier, opts *opts) {
t.Helper()
wantResult := []*at.QueryResult{}
if opts.wantMetric != nil && (opts.wantSample != nil || len(opts.wantSamples) > 0) {
@@ -255,19 +255,19 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
return sut.PrometheusAPIV1Query(t, opts.query, at.QueryOpts{
return sut.APIV1Query(t, opts.query, at.QueryOpts{
Time: opts.qtime,
Step: opts.step,
})
},
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`series1{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
`series1{label="foo"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
}, at.QueryOpts{})
@@ -326,7 +326,7 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
// Insert samples with different dates. The difference in ms between the two
// timestamps is 4236579304.
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`series2{label="foo"} 10 1638564958042`, // 2021-12-03T20:55:58.042Z
`series2{label="foo"} 20 1642801537346`, // 2022-01-21T21:45:37.346Z
}, at.QueryOpts{})

View File

@@ -16,7 +16,7 @@ func TestSingleMaxIngestionRateIncrementsMetric(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartVmsingle("vmsingle", []string{"-maxIngestionRate=1"})
sut.PrometheusAPIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
sut.APIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
if got := sut.GetMetric(t, "vm_max_ingestion_rate_limit_reached_total"); got <= 0 {
t.Fatalf("Unexpected vm_max_ingestion_rate_limit_reached_total: got %f, want >0", got)
}
@@ -26,7 +26,7 @@ func TestSingleMaxIngestionRateDoesNotIncrementMetric(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartVmsingle("vmsingle", []string{"-maxIngestionRate=15"})
sut.PrometheusAPIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
sut.APIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
if got, want := sut.GetMetric(t, "vm_max_ingestion_rate_limit_reached_total"), 0.0; got != want {
t.Fatalf("Unexpected vm_max_ingestion_rate_limit_reached_total: got %f, want >0", got)
}

View File

@@ -37,7 +37,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
}
tsdbMetricNameEntryCmpOpts := cmpopts.IgnoreFields(apptest.TSDBStatusResponseMetricNameEntry{}, "LastRequestTimestamp")
sut.PrometheusAPIV1ImportPrometheus(t, dataSet, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, dataSet, at.QueryOpts{})
sut.ForceFlush(t)
// verify ingest request correctly registered
@@ -55,7 +55,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
}
// verify query request correctly registered
sut.PrometheusAPIV1Query(t, `{__name__!=""}`, at.QueryOpts{Time: ingestDateTime})
sut.APIV1Query(t, `{__name__!=""}`, at.QueryOpts{Time: ingestDateTime})
expected = apptest.MetricNamesStatsResponse{
Records: []at.MetricNamesStatsRecord{
{MetricName: largeMetricName, QueryRequestsCount: 1},
@@ -97,7 +97,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
}
// perform query request for single metric and check counter increase
sut.PrometheusAPIV1Query(t, `metric_name_2`, at.QueryOpts{Time: ingestDateTime})
sut.APIV1Query(t, `metric_name_2`, at.QueryOpts{Time: ingestDateTime})
expected = apptest.MetricNamesStatsResponse{
Records: []at.MetricNamesStatsRecord{
{MetricName: largeMetricName, QueryRequestsCount: 1},
@@ -187,7 +187,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
// ingest per tenant data and verify it with search
tenantIDs := []string{"1:1", "1:15", "15:15"}
for _, tenantID := range tenantIDs {
vminsert.PrometheusAPIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{Tenant: tenantID})
vminsert.APIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{Tenant: tenantID})
vmstorage1.ForceFlush(t)
vmstorage2.ForceFlush(t)
@@ -206,7 +206,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
}
// verify query request registered correctly
vmselect.PrometheusAPIV1Query(t, `{__name__!=""}`, apptest.QueryOpts{
vmselect.APIV1Query(t, `{__name__!=""}`, apptest.QueryOpts{
Tenant: tenantID, Time: ingestDateTime,
})

View File

@@ -46,7 +46,7 @@ func TestClusterInstantQuery(t *testing.T) {
testQueryRangeWithAtModifier(t, sut)
}
func testInstantQueryWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQuerier) {
func testInstantQueryWithUTFNames(t *testing.T, sut apptest.WriteQuerier) {
data := []pb.TimeSeries{
{
Labels: []pb.Label{
@@ -59,18 +59,18 @@ func testInstantQueryWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQueri
},
}
sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
sut.APIV1Write(t, data, apptest.QueryOpts{})
sut.ForceFlush(t)
var got, want *apptest.PrometheusAPIV1QueryResponse
var got, want *apptest.APIV1QueryResponse
cmpOptions := []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.EquateNaNs(),
}
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "3fooµ¥", "3👋tfにちは": "漢©®€£"}}]}}`)
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "3fooµ¥", "3👋tfにちは": "漢©®€£"}}]}}`)
fn := func(query string) {
got = sut.PrometheusAPIV1Query(t, query, apptest.QueryOpts{
got = sut.APIV1Query(t, query, apptest.QueryOpts{
Step: "5m",
Time: "2024-01-01T00:01:00.000Z",
})
@@ -112,24 +112,24 @@ var staleNaNsData = func() []pb.TimeSeries {
}
}()
func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.PrometheusWriteQuerier) {
func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.WriteQuerier) {
sut.PrometheusAPIV1Write(t, staleNaNsData, apptest.QueryOpts{})
sut.APIV1Write(t, staleNaNsData, apptest.QueryOpts{})
sut.ForceFlush(t)
var got, want *apptest.PrometheusAPIV1QueryResponse
var got, want *apptest.APIV1QueryResponse
cmpOptions := []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.EquateNaNs(),
}
// Verify that instant query returns the first point.
got = sut.PrometheusAPIV1Query(t, "metric", apptest.QueryOpts{
got = sut.APIV1Query(t, "metric", apptest.QueryOpts{
Step: "5m",
Time: "2024-01-01T00:01:00.000Z",
})
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}}]}}`)
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}}]}}`)
want.Data.Result[0].Sample = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
@@ -137,11 +137,11 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
// Verify that instant query does not return stale NaN.
got = sut.PrometheusAPIV1Query(t, "metric", apptest.QueryOpts{
got = sut.APIV1Query(t, "metric", apptest.QueryOpts{
Step: "5m",
Time: "2024-01-01T00:02:00.000Z",
})
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": []}}`)
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": []}}`)
// Empty response, stale NaN is not included into response
if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
@@ -151,11 +151,11 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
// while it must not.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5806
got = sut.PrometheusAPIV1Query(t, "metric[2m]", apptest.QueryOpts{
got = sut.APIV1Query(t, "metric[2m]", apptest.QueryOpts{
Step: "5m",
Time: "2024-01-01T00:02:00.000Z",
})
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
s := make([]*apptest.Sample, 2)
s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
@@ -166,11 +166,11 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
// Verify that exported data contains stale NaN.
got = sut.PrometheusAPIV1Export(t, `{__name__="metric"}`, apptest.QueryOpts{
got = sut.APIV1Export(t, `{__name__="metric"}`, apptest.QueryOpts{
Start: "2024-01-01T00:01:00.000Z",
End: "2024-01-01T00:02:00.000Z",
})
want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
s = make([]*apptest.Sample, 2)
s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
@@ -184,7 +184,7 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
// See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8444
// However, conversion of math.NaN to int64 could behave differently depending on platform and Go version.
// Hence, this test could succeed for some platforms even if fix is rolled back.
func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQuerier) {
func testQueryRangeWithAtModifier(t *testing.T, sut apptest.WriteQuerier) {
data := []pb.TimeSeries{
{
Labels: []pb.Label{
@@ -204,10 +204,10 @@ func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQueri
},
}
sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
sut.APIV1Write(t, data, apptest.QueryOpts{})
sut.ForceFlush(t)
resp := sut.PrometheusAPIV1QueryRange(t, `vector(1) @ up`, apptest.QueryOpts{
resp := sut.APIV1QueryRange(t, `vector(1) @ up`, apptest.QueryOpts{
Start: "2025-01-01T00:00:00Z",
End: "2025-01-01T00:02:00Z",
Step: "10s",
@@ -217,7 +217,7 @@ func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQueri
t.Fatalf("unexpected status: %q", resp.Status)
}
resp = sut.PrometheusAPIV1QueryRange(t, `vector(1) @ metricNaN`, apptest.QueryOpts{
resp = sut.APIV1QueryRange(t, `vector(1) @ metricNaN`, apptest.QueryOpts{
Start: "2025-01-01T00:00:00Z",
End: "2025-01-01T00:02:00Z",
Step: "10s",

View File

@@ -37,7 +37,7 @@ func TestClusterMultilevelSelect(t *testing.T) {
const numMetrics = 1000
records := make([]string, numMetrics)
want := &apptest.PrometheusAPIV1SeriesResponse{
want := &apptest.APIV1SeriesResponse{
Status: "success",
IsPartial: false,
Data: make([]map[string]string, numMetrics),
@@ -49,7 +49,7 @@ func TestClusterMultilevelSelect(t *testing.T) {
}
want.Sort()
qopts := apptest.QueryOpts{Tenant: "0"}
vminsert.PrometheusAPIV1ImportPrometheus(t, records, qopts)
vminsert.APIV1ImportPrometheus(t, records, qopts)
vmstorage.ForceFlush(t)
// Retrieve all time series and verify that both vmselect (L1) and
@@ -60,7 +60,7 @@ func TestClusterMultilevelSelect(t *testing.T) {
tc.Assert(&apptest.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, qopts)
res := app.APIV1Series(t, `{__name__=~".*"}`, qopts)
res.Sort()
return res
},

View File

@@ -13,8 +13,8 @@ import (
func TestClusterMultiTenantSelect(t *testing.T) {
os.RemoveAll(t.Name())
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
cmpSROpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Status", "IsPartial")
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
cmpSROpt := cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Status", "IsPartial")
tc := apptest.NewTestCase(t)
defer tc.Stop()
@@ -37,12 +37,12 @@ func TestClusterMultiTenantSelect(t *testing.T) {
}
// test for empty tenants request
got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
got := vmselect.APIV1Query(t, "foo_bar", apptest.QueryOpts{
Tenant: "multitenant",
Step: "5m",
Time: "2022-05-10T08:03:00.000Z",
})
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
@@ -51,12 +51,12 @@ func TestClusterMultiTenantSelect(t *testing.T) {
tenantIDs := []string{"1:1", "1:15"}
instantCT := "2022-05-10T08:05:00.000Z"
for _, tenantID := range tenantIDs {
vminsert.PrometheusAPIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
vminsert.APIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
vmstorage.ForceFlush(t)
got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
got := vmselect.APIV1Query(t, "foo_bar", apptest.QueryOpts{
Tenant: tenantID, Time: instantCT,
})
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}]}}`)
want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}]}}`)
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
@@ -64,7 +64,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
// verify all tenants searchable with multitenant APIs
// /api/v1/query
want = apptest.NewPrometheusAPIV1QueryResponse(t,
want = apptest.NewAPIV1QueryResponse(t,
`{"data":
{"result":[
{"metric":{"__name__":"foo_bar","vm_account_id":"1","vm_project_id": "1"},"value":[1652169900,"3"]},
@@ -73,7 +73,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
}
}`,
)
got = vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
got = vmselect.APIV1Query(t, "foo_bar", apptest.QueryOpts{
Tenant: "multitenant",
Time: instantCT,
})
@@ -83,14 +83,14 @@ func TestClusterMultiTenantSelect(t *testing.T) {
// /api/v1/query_range aggregated by tenant labels
query := "sum(foo_bar) by(vm_account_id,vm_project_id)"
got = vmselect.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
got = vmselect.APIV1QueryRange(t, query, apptest.QueryOpts{
Tenant: "multitenant",
Start: "2022-05-10T07:59:00.000Z",
End: "2022-05-10T08:05:00.000Z",
Step: "1m",
})
want = apptest.NewPrometheusAPIV1QueryResponse(t,
want = apptest.NewAPIV1QueryResponse(t,
`{"data":
{"result": [
{"metric": {"vm_account_id": "1","vm_project_id":"1"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]},
@@ -104,7 +104,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
// verify /api/v1/series response
wantSR := apptest.NewPrometheusAPIV1SeriesResponse(t,
wantSR := apptest.NewAPIV1SeriesResponse(t,
`{"data": [
{"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
{"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}
@@ -112,7 +112,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
}`)
wantSR.Sort()
gotSR := vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
gotSR := vmselect.APIV1Series(t, "foo_bar", apptest.QueryOpts{
Tenant: "multitenant",
Start: "2022-05-10T08:03:00.000Z",
})
@@ -129,11 +129,11 @@ func TestClusterMultiTenantSelect(t *testing.T) {
`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
}
vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
vminsert.APIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
vmstorage.ForceFlush(t)
// /api/v1/query with query filters
want = apptest.NewPrometheusAPIV1QueryResponse(t,
want = apptest.NewAPIV1QueryResponse(t,
`{"data":
{"result":[
{"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"value":[1652169900,"1"]},
@@ -142,7 +142,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
}
}`,
)
got = vmselect.PrometheusAPIV1Query(t, `foo_bar{vm_account_id="5"}`, apptest.QueryOpts{
got = vmselect.APIV1Query(t, `foo_bar{vm_account_id="5"}`, apptest.QueryOpts{
Time: instantCT,
Tenant: "multitenant",
})
@@ -152,14 +152,14 @@ func TestClusterMultiTenantSelect(t *testing.T) {
// /api/v1/series with extra_filters
wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
wantSR = apptest.NewAPIV1SeriesResponse(t,
`{"data": [
{"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"15"},
{"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}
]
}`)
wantSR.Sort()
gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
gotSR = vmselect.APIV1Series(t, "foo_bar", apptest.QueryOpts{
Start: "2022-05-10T08:00:00.000Z",
End: "2022-05-10T08:30:00.000Z",
ExtraFilters: []string{`{vm_project_id="15"}`},
@@ -175,7 +175,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
vmselect.DeleteSeries(t, "foo_bar", apptest.QueryOpts{
Tenant: "5:15",
})
wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
wantSR = apptest.NewAPIV1SeriesResponse(t,
`{"data": [
{"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
{"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
@@ -185,7 +185,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
}`)
wantSR.Sort()
gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
gotSR = vmselect.APIV1Series(t, "foo_bar", apptest.QueryOpts{
Tenant: "multitenant",
Start: "2022-05-10T08:03:00.000Z",
})
@@ -199,7 +199,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
Tenant: "multitenant",
})
wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
wantSR = apptest.NewAPIV1SeriesResponse(t,
`{"data": [
{"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
{"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
@@ -207,7 +207,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
}`)
wantSR.Sort()
gotSR = vmselect.PrometheusAPIV1Series(t, `foo_bar`, apptest.QueryOpts{
gotSR = vmselect.APIV1Series(t, `foo_bar`, apptest.QueryOpts{
Tenant: "multitenant",
Start: "2022-05-10T08:03:00.000Z",
})

View File

@@ -11,7 +11,7 @@ func TestSingleSearchWithDisabledPerDayIndex(t *testing.T) {
tc := at.NewTestCase(t)
defer tc.Stop()
testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.PrometheusWriteQuerier {
testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.WriteQuerier {
return tc.MustStartVmsingle("vmsingle-"+name, []string{
"-storageDataPath=" + tc.Dir() + "/vmsingle",
"-retentionPeriod=100y",
@@ -25,7 +25,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
tc := at.NewTestCase(t)
defer tc.Stop()
testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.PrometheusWriteQuerier {
testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.WriteQuerier {
// Using static ports for vmstorage because random ports may cause
// changes in how data is sharded.
vmstorage1 := tc.MustStartVmstorage("vmstorage1-"+name, []string{
@@ -59,7 +59,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
})
}
type startSUTFunc func(name string, disablePerDayIndex bool) at.PrometheusWriteQuerier
type startSUTFunc func(name string, disablePerDayIndex bool) at.WriteQuerier
// testDisablePerDayIndex_Search shows what search results to expect when data
// is first inserted with per-day index enabled and then with per-day index
@@ -78,17 +78,17 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
wantSeries []map[string]string
wantQueryResults []*at.QueryResult
}
assertSearchResults := func(sut at.PrometheusQuerier, opts *opts) {
assertSearchResults := func(sut at.APIQuerier, opts *opts) {
t.Helper()
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return sut.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
return sut.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
Start: opts.start,
End: opts.end,
}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Want: &at.APIV1SeriesResponse{
Status: "success",
Data: opts.wantSeries,
},
@@ -96,13 +96,13 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return sut.PrometheusAPIV1QueryRange(t, `{__name__=~".*"}`, at.QueryOpts{
return sut.APIV1QueryRange(t, `{__name__=~".*"}`, at.QueryOpts{
Start: opts.start,
End: opts.end,
Step: "1d",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -116,7 +116,7 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
// is searchable.
sut := start("with-per-day-index", false)
sample1 := []string{"metric1 111 1704067200000"} // 2024-01-01T00:00:00Z
sut.PrometheusAPIV1ImportPrometheus(t, sample1, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, sample1, at.QueryOpts{})
sut.ForceFlush(t)
assertSearchResults(sut, &opts{
start: "2024-01-01T00:00:00Z",
@@ -132,10 +132,10 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
// Restart vmsingle with disabled per-day index, insert sample2, and confirm
// that both sample1 and sample2 is searchable.
tc.StopPrometheusWriteQuerier(sut)
tc.StopWriteQuerier(sut)
sut = start("without-per-day-index", true)
sample2 := []string{"metric2 222 1704067200000"} // 2024-01-01T00:00:00Z
sut.PrometheusAPIV1ImportPrometheus(t, sample2, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, sample2, at.QueryOpts{})
sut.ForceFlush(t)
assertSearchResults(sut, &opts{
start: "2024-01-01T00:00:00Z",
@@ -165,9 +165,9 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
// - sample2 is not searchable when the time range is <= 40 days
// - sample2 becomes searchable when the time range is > 40 days
sample3 := []string{"metric1 333 1705708800000"} // 2024-01-20T00:00:00Z
sut.PrometheusAPIV1ImportPrometheus(t, sample3, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, sample3, at.QueryOpts{})
sut.ForceFlush(t)
tc.StopPrometheusWriteQuerier(sut)
tc.StopWriteQuerier(sut)
sut = start("with-per-day-index2", false)
// Time range is 1 day (Jan 1st) <= 40 days
@@ -295,14 +295,14 @@ func testClusterActiveTimeseriesMetric(t *testing.T, disablePerDayIndex bool) {
})
}
func testActiveTimeseriesMetric(tc *at.TestCase, sut at.PrometheusWriteQuerier, getActiveTimeseries func() int) {
func testActiveTimeseriesMetric(tc *at.TestCase, sut at.WriteQuerier, getActiveTimeseries func() int) {
t := tc.T()
const numSamples = 1000
samples := make([]string, numSamples)
for i := range numSamples {
samples[i] = fmt.Sprintf("metric_%03d %d", i, i)
}
sut.PrometheusAPIV1ImportPrometheus(t, samples, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, samples, at.QueryOpts{})
sut.ForceFlush(t)
tc.Assert(&at.AssertOptions{
Msg: `unexpected vm_cache_entries{type="storage/hour_metric_ids"} metric value`,

View File

@@ -13,7 +13,7 @@ import (
func TestClusterMaxUniqueTimeseries(t *testing.T) {
os.RemoveAll(t.Name())
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
tc := apptest.NewTestCase(t)
defer tc.Stop()
@@ -54,14 +54,14 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
// write data to two tenants
tenantIDs := []string{"0:0", "1:15"}
for _, tenantID := range tenantIDs {
vminsert.PrometheusAPIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
vminsert.APIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
vmstorage.ForceFlush(t)
}
instantCT := "2022-05-10T08:05:00.000Z"
// success - `/api/v1/query`
want := apptest.NewPrometheusAPIV1QueryResponse(t,
want := apptest.NewAPIV1QueryResponse(t,
`{"data":
{"result":[
{"metric":{"__name__":"foo_bar1","instance":"a"},"value":[1652169900,"1"]}
@@ -69,7 +69,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
}
}`,
)
queryRes := vmselectSmallLimit.PrometheusAPIV1Query(t, "foo_bar1", apptest.QueryOpts{
queryRes := vmselectSmallLimit.APIV1Query(t, "foo_bar1", apptest.QueryOpts{
Time: instantCT,
})
if diff := cmp.Diff(want, queryRes, cmpOpt); diff != "" {
@@ -78,7 +78,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
// success - multitenant `/api/v1/query`
// query is split into two queries for each tenant, so the final result can exceed the limit.
want = apptest.NewPrometheusAPIV1QueryResponse(t,
want = apptest.NewAPIV1QueryResponse(t,
`{"data":
{"result":[
{"metric":{"__name__":"foo_bar1","instance":"a","vm_account_id":"0","vm_project_id":"0"},"value":[1652169900,"1"]},
@@ -87,7 +87,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
}
}`,
)
queryRes = vmselectSmallLimit.PrometheusAPIV1Query(t, "foo_bar1", apptest.QueryOpts{
queryRes = vmselectSmallLimit.APIV1Query(t, "foo_bar1", apptest.QueryOpts{
Time: instantCT,
Tenant: "multitenant",
})
@@ -96,7 +96,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
}
// fail - `/api/v1/query`, exceed vmselect `maxUniqueTimeseries`
queryRes = vmselectSmallLimit.PrometheusAPIV1Query(t, "foo_bar2", apptest.QueryOpts{
queryRes = vmselectSmallLimit.APIV1Query(t, "foo_bar2", apptest.QueryOpts{
Time: instantCT,
})
if queryRes.ErrorType != "422" {
@@ -104,7 +104,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
}
// fail - `/api/v1/query`, exceed vmstorage `maxUniqueTimeseries`
queryRes = vmselectNoLimit.PrometheusAPIV1Query(t, "foo_bar3", apptest.QueryOpts{
queryRes = vmselectNoLimit.APIV1Query(t, "foo_bar3", apptest.QueryOpts{
Time: instantCT,
})
if queryRes.ErrorType != "422" {
@@ -112,7 +112,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
}
// fail - `/api/v1/query`, vmselect `maxUniqueTimeseries` cannot exceed vmstorage `maxUniqueTimeseries`
queryRes = vmselectBigLimit.PrometheusAPIV1Query(t, "foo_bar3", apptest.QueryOpts{
queryRes = vmselectBigLimit.APIV1Query(t, "foo_bar3", apptest.QueryOpts{
Time: instantCT,
})
if queryRes.ErrorType != "422" {
@@ -123,7 +123,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
func TestClusterMaxSeries(t *testing.T) {
os.RemoveAll(t.Name())
cmpSROpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Status", "IsPartial")
cmpSROpt := cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Status", "IsPartial")
tc := apptest.NewTestCase(t)
defer tc.Stop()
@@ -153,18 +153,18 @@ func TestClusterMaxSeries(t *testing.T) {
}
// write data
vminsert.PrometheusAPIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{})
vminsert.APIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{})
vmstorage.ForceFlush(t)
// success - `/api/v1/series`, vmselect `maxLabelsAPISeries` can exceed vmstorage `maxLabelsAPISeries``
wantSR := apptest.NewPrometheusAPIV1SeriesResponse(t,
wantSR := apptest.NewAPIV1SeriesResponse(t,
`{"data": [
{"__name__":"foo_bar3","instance":"a"},
{"__name__":"foo_bar3","instance":"b"},
{"__name__":"foo_bar3","instance":"c"}
]
}`)
seriesRes := vmselectBigLimit.PrometheusAPIV1Series(t, "foo_bar3", apptest.QueryOpts{
seriesRes := vmselectBigLimit.APIV1Series(t, "foo_bar3", apptest.QueryOpts{
Start: "2022-05-10T08:03:00.000Z",
})
if diff := cmp.Diff(wantSR.Sort(), seriesRes.Sort(), cmpSROpt); diff != "" {
@@ -172,7 +172,7 @@ func TestClusterMaxSeries(t *testing.T) {
}
// fail - `/api/v1/series`, exceed vmselect `maxSeries`
seriesRes1 := vmselectSmallLimit.PrometheusAPIV1Series(t, "foo_bar3", apptest.QueryOpts{
seriesRes1 := vmselectSmallLimit.APIV1Series(t, "foo_bar3", apptest.QueryOpts{
Start: "2022-05-10T08:03:00.000Z",
})
if seriesRes1.ErrorType != "422" {

View File

@@ -57,7 +57,7 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
wantMetrics []map[string]string
wantSamples []*at.Sample
}
f := func(sut at.PrometheusQuerier, opts *opts) {
f := func(sut at.APIQuerier, opts *opts) {
t.Helper()
wantResult := []*at.QueryResult{}
for idx, wm := range opts.wantMetrics {
@@ -70,19 +70,19 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
return sut.PrometheusAPIV1Query(t, opts.query, at.QueryOpts{
return sut.APIV1Query(t, opts.query, at.QueryOpts{
Time: opts.qtime,
Step: opts.step,
})
},
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`importprometheus_series{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
`must_drop_series{label="foo"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
}, at.QueryOpts{})
@@ -180,7 +180,7 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
},
},
}
sut.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
sut.APIV1Write(t, pbData, at.QueryOpts{})
sut.ForceFlush(t)
f(sut, &opts{
query: `{label="foo2"}[120ms]`,

View File

@@ -7,10 +7,11 @@ import (
"testing"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
)
type clusterWithReplication struct {
@@ -94,7 +95,7 @@ func TestClusterReplication_DataIsWrittenSeveralTimes(t *testing.T) {
for i := range numRecs {
recs[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
}
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
tc.ForceFlush(c.vmstorages...)
// Verify that each storage node has metrics and that total metric count across
@@ -151,7 +152,7 @@ func TestClusterReplication_Deduplication(t *testing.T) {
ts = ts.Add(1 * time.Minute)
}
}
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
tc.ForceFlush(c.vmstorages...)
// Check /api/v1/series response.
@@ -164,12 +165,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
Start: "2024-01-01T00:00:00Z",
End: "2024-01-31T00:00:00Z",
}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Want: &at.APIV1SeriesResponse{
Status: "success",
IsPartial: false,
Data: []map[string]string{
@@ -194,12 +195,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
return app.PrometheusAPIV1Query(t, "metric_1", at.QueryOpts{
return app.APIV1Query(t, "metric_1", at.QueryOpts{
Time: "2024-01-01T00:05:00Z",
Step: "5m",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "vector",
@@ -236,12 +237,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
return app.PrometheusAPIV1Query(t, "metric_1[5m]", at.QueryOpts{
return app.APIV1Query(t, "metric_1[5m]", at.QueryOpts{
Time: "2024-01-01T00:05:00Z",
Step: "5m",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -273,13 +274,13 @@ func TestClusterReplication_Deduplication(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return app.PrometheusAPIV1QueryRange(t, "metric_1", at.QueryOpts{
return app.APIV1QueryRange(t, "metric_1", at.QueryOpts{
Start: "2024-01-01T00:00:00Z",
End: "2024-01-01T00:10:00Z",
Step: "5m",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -309,12 +310,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/export response",
Got: func() any {
return app.PrometheusAPIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
return app.APIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
Start: "2024-01-01T00:00:00Z",
End: "2024-01-01T00:03:00Z",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -360,7 +361,7 @@ func TestClusterReplication_PartialResponse(t *testing.T) {
for i := range numRecs {
recs[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
}
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
tc.ForceFlush(c.vmstorages...)
// Verify partial vs full response.
@@ -370,14 +371,14 @@ func TestClusterReplication_PartialResponse(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Want: &at.APIV1SeriesResponse{
Status: "success",
IsPartial: wantPartial,
},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Data"),
},
})
}
@@ -437,7 +438,7 @@ func TestClusterReplication_SkipSlowReplicas(t *testing.T) {
const numRecs = 1000
recs := make([]string, numRecs)
wantSeries := &at.PrometheusAPIV1SeriesResponse{
wantSeries := &at.APIV1SeriesResponse{
Status: "success",
Data: make([]map[string]string, numRecs),
}
@@ -447,7 +448,7 @@ func TestClusterReplication_SkipSlowReplicas(t *testing.T) {
wantSeries.Data[i] = map[string]string{"__name__": name}
}
wantSeries.Sort()
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
tc.ForceFlush(c.vmstorages...)
// Verify skipping slow replicas by counting the number of skipSlowReplicas
@@ -458,12 +459,12 @@ func TestClusterReplication_SkipSlowReplicas(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
},
Want: wantSeries,
})
res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
res := app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
got := res.Trace.Contains("cancel request because -search.skipSlowReplicas is set and every group returned the needed number of responses according to replicationFactor")
if got != want {
t.Errorf("unexpected number of skipSlowReplicas messages in request trace: got %d, want %d (full trace:\n%v)", got, want, res.Trace)
@@ -654,7 +655,7 @@ func TestClusterGroupReplication(t *testing.T) {
numRecs = numMetrics * numSamples
)
var recs []string
wantSeries := &at.PrometheusAPIV1SeriesResponse{
wantSeries := &at.APIV1SeriesResponse{
Status: "success",
Data: make([]map[string]string, numMetrics),
}
@@ -668,7 +669,7 @@ func TestClusterGroupReplication(t *testing.T) {
}
}
wantSeries.Sort()
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
c.forceFlush(tc)
opts := &testGroupReplicationOpts{
@@ -694,7 +695,7 @@ type testGroupReplicationOpts struct {
numGroups int
numNodes int
numRecs int
wantSeries *at.PrometheusAPIV1SeriesResponse
wantSeries *at.APIV1SeriesResponse
}
// testGroupDataIsWrittenSeveralTimes checks that multiple
@@ -747,7 +748,7 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
Start: "2024-01-01T00:00:00Z",
End: "2024-01-31T00:00:00Z",
}).Sort()
@@ -768,12 +769,12 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
return app.PrometheusAPIV1Query(t, "metric_1", at.QueryOpts{
return app.APIV1Query(t, "metric_1", at.QueryOpts{
Time: "2024-01-01T00:05:00Z",
Step: "5m",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "vector",
@@ -810,12 +811,12 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
return app.PrometheusAPIV1Query(t, "metric_1[5m]", at.QueryOpts{
return app.APIV1Query(t, "metric_1[5m]", at.QueryOpts{
Time: "2024-01-01T00:05:00Z",
Step: "5m",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -847,13 +848,13 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return app.PrometheusAPIV1QueryRange(t, "metric_1", at.QueryOpts{
return app.APIV1QueryRange(t, "metric_1", at.QueryOpts{
Start: "2024-01-01T00:00:00Z",
End: "2024-01-01T00:10:00Z",
Step: "5m",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -883,12 +884,12 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/export response",
Got: func() any {
return app.PrometheusAPIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
return app.APIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
Start: "2024-01-01T00:00:00Z",
End: "2024-01-01T00:03:00Z",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -929,7 +930,7 @@ func testGroupSkipSlowReplicas(tc *at.TestCase, opts *testGroupReplicationOpts)
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
Start: "2024-01-01T00:00:00Z",
End: "2024-01-31T00:00:00Z",
}).Sort()
@@ -937,7 +938,7 @@ func testGroupSkipSlowReplicas(tc *at.TestCase, opts *testGroupReplicationOpts)
Want: opts.wantSeries,
})
res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
res := app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
got := res.Trace.Contains("cancel request because -search.skipSlowReplicas is set and every group returned the needed number of responses according to replicationFactor")
if got < wantMin || got > wantMax {
t.Errorf("unexpected number of skipSlowReplicas messages in request trace: got %d, %d <= want <= %d (full trace:\n%v)", got, wantMin, wantMax, res.Trace)
@@ -973,7 +974,7 @@ func testGroupSkipSlowReplicas(tc *at.TestCase, opts *testGroupReplicationOpts)
// The data is replicated across N groups of M nodes. Replication factor is
// globalRF. There is no replication across the nodes within each group or
//it is unknown it there is one.
// it is unknown it there is one.
//
// Max number of nodes to skip is M*(globalRF-1). This corresponds to the
// case when N-globalRF+1 groups have received the response from all of
@@ -1020,17 +1021,17 @@ func testGroupPartialResponse(tc *at.TestCase, opts *testGroupReplicationOpts) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
Start: "2024-01-01T00:00:00Z",
End: "2024-01-31T00:00:00Z",
}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Want: &at.APIV1SeriesResponse{
Status: "success",
IsPartial: wantPartial,
},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Data"),
},
})
}
@@ -1128,10 +1129,10 @@ func TestClusterReplication_PartialResponseMultitenant(t *testing.T) {
recs[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
}
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{
Tenant: "0",
})
c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{
c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{
Tenant: "1",
})
tc.ForceFlush(c.vmstorages...)
@@ -1144,14 +1145,14 @@ func TestClusterReplication_PartialResponseMultitenant(t *testing.T) {
Msg: "unexpected /api/v1/query response",
Got: func() any {
qo := at.QueryOpts{Tenant: "multitenant", Trace: "1"}
return app.PrometheusAPIV1Query(t, `{__name__=~"metric_.*"}`, qo)
return app.APIV1Query(t, `{__name__=~"metric_.*"}`, qo)
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
IsPartial: wantPartial,
},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Data"),
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Data"),
},
})
}

View File

@@ -13,7 +13,7 @@ import (
func TestClusterRollupResultCache(t *testing.T) {
os.RemoveAll(t.Name())
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
tc := apptest.NewTestCase(t)
defer tc.Stop()
@@ -34,10 +34,10 @@ func TestClusterRollupResultCache(t *testing.T) {
`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
}
vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
vminsert.APIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
vmstorage.ForceFlush(t)
want := apptest.NewPrometheusAPIV1QueryResponse(t,
want := apptest.NewAPIV1QueryResponse(t,
`{"data":
{"result":[
{"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"values":[[1652169720,"1"],[1652169780,"1"]]},
@@ -47,7 +47,7 @@ func TestClusterRollupResultCache(t *testing.T) {
}`,
)
got := vmselect.PrometheusAPIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
got := vmselect.APIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
Tenant: "multitenant",
Start: "2022-05-10T07:59:00.000Z",
End: "2022-05-10T08:05:00.000Z",
@@ -58,13 +58,13 @@ func TestClusterRollupResultCache(t *testing.T) {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
want = apptest.NewPrometheusAPIV1QueryResponse(t,
want = apptest.NewAPIV1QueryResponse(t,
`{"data":
{"result":[]}
}`,
)
got = vmselect.PrometheusAPIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
got = vmselect.APIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
Tenant: "multitenant",
Start: "2022-05-10T07:59:00.000Z",
End: "2022-05-10T08:05:00.000Z",

View File

@@ -38,7 +38,7 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
const numMetrics = 1000
records := make([]string, numMetrics)
want := &apptest.PrometheusAPIV1SeriesResponse{
want := &apptest.APIV1SeriesResponse{
Status: "success",
IsPartial: false,
Data: make([]map[string]string, numMetrics),
@@ -49,7 +49,7 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
want.Data[i] = map[string]string{"__name__": name}
}
want.Sort()
vminsert.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{})
vminsert.APIV1ImportPrometheus(t, records, apptest.QueryOpts{})
vmstorage1.ForceFlush(t)
vmstorage2.ForceFlush(t)
@@ -74,7 +74,7 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
tc.Assert(&apptest.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
res := vmselect.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{})
res := vmselect.APIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{})
res.Sort()
return res
},

View File

@@ -28,7 +28,7 @@ func TestSingleSnapshots_CreateListDelete(t *testing.T) {
for i := range numSamples {
samples[i] = fmt.Sprintf("metric_%03d %d", i, i)
}
sut.PrometheusAPIV1ImportPrometheus(t, samples, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, samples, at.QueryOpts{})
sut.ForceFlush(t)
// Create several snapshots using VictoriaMetrics and Prometheus endpoints.
@@ -113,7 +113,7 @@ func TestClusterSnapshots_CreateListDelete(t *testing.T) {
for i := range numSamples {
samples[i] = fmt.Sprintf("metric_%03d %d", i, i)
}
sut.PrometheusAPIV1ImportPrometheus(t, samples, at.QueryOpts{})
sut.APIV1ImportPrometheus(t, samples, at.QueryOpts{})
sut.ForceFlush(t)
// Create several snapshots for both vmstorage replicas using

View File

@@ -4,9 +4,10 @@ import (
"os"
"testing"
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
)
// TestSingleSpecialQueryRegression is used to test queries that have experienced issues for specific data sets.
@@ -35,13 +36,12 @@ func TestClusterSpecialQueryRegression(t *testing.T) {
testSpecialQueryRegression(tc, sut)
}
func testSpecialQueryRegression(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testSpecialQueryRegression(tc *at.TestCase, sut at.WriteQuerier) {
// prometheus
testCaseSensitiveRegex(tc, sut)
testDuplicateLabel(tc, sut)
testTooBigLookbehindWindow(tc, sut)
testMatchSeries(tc, sut)
testNegativeIncrease(tc, sut)
// graphite
testComparisonNotInfNotNan(tc, sut)
@@ -51,12 +51,12 @@ func testSpecialQueryRegression(tc *at.TestCase, sut at.PrometheusWriteQuerier)
testSubqueryAggregation(tc, sut)
}
func testCaseSensitiveRegex(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testCaseSensitiveRegex(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
// case-sensitive-regex
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/161
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`prometheus.sensitiveRegex{label="sensitiveRegex"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
`prometheus.sensitiveRegex{label="SensitiveRegex"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
}, at.QueryOpts{})
@@ -65,12 +65,12 @@ func testCaseSensitiveRegex(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/export response",
Got: func() any {
return sut.PrometheusAPIV1Export(t, `{label=~'(?i)sensitiveregex'}`, at.QueryOpts{
return sut.APIV1Export(t, `{label=~'(?i)sensitiveregex'}`, at.QueryOpts{
Start: "2024-02-05T08:50:00.700Z",
End: "2024-02-05T09:00:00.700Z",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -87,17 +87,17 @@ func testCaseSensitiveRegex(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
},
},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}
func testDuplicateLabel(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testDuplicateLabel(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
// duplicate_label
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/172
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`prometheus.duplicate_label{label="duplicate", label="duplicate"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
}, at.QueryOpts{})
sut.ForceFlush(t)
@@ -105,12 +105,12 @@ func testDuplicateLabel(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/export response",
Got: func() any {
return sut.PrometheusAPIV1Export(t, `{__name__='prometheus.duplicate_label'}`, at.QueryOpts{
return sut.APIV1Export(t, `{__name__='prometheus.duplicate_label'}`, at.QueryOpts{
Start: "2024-02-05T08:50:00.700Z",
End: "2024-02-05T09:00:00.700Z",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -123,17 +123,17 @@ func testDuplicateLabel(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
},
},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}
func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testTooBigLookbehindWindow(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
// too big look-behind window
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`prometheus.too_big_lookbehind{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
}, at.QueryOpts{})
sut.ForceFlush(t)
@@ -141,12 +141,12 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
return sut.PrometheusAPIV1Query(t, `prometheus.too_big_lookbehind{label="foo"}[100y]`, at.QueryOpts{
return sut.APIV1Query(t, `prometheus.too_big_lookbehind{label="foo"}[100y]`, at.QueryOpts{
Step: "5m",
Time: "2024-02-05T08:57:36.700Z",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -164,7 +164,7 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
// too big look-behind window - query range
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`prometheus.too_big_lookbehind_range{label="foo"} 13 1707123496700`, // 2024-02-05T08:58:16.700Z
`prometheus.too_big_lookbehind_range{label="foo"} 12 1707123466700`, // 2024-02-05T08:57:46.700Z
`prometheus.too_big_lookbehind_range{label="foo"} 11 1707123436700`, // 2024-02-05T08:57:16.700Z
@@ -175,13 +175,13 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return sut.PrometheusAPIV1QueryRange(t, `prometheus.too_big_lookbehind_range{label="foo"}`, at.QueryOpts{
return sut.APIV1QueryRange(t, `prometheus.too_big_lookbehind_range{label="foo"}`, at.QueryOpts{
Start: "2024-02-05T08:56:46.700Z",
End: "2024-02-05T08:58:16.700Z",
Step: "30s",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -201,12 +201,12 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
})
}
func testMatchSeries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testMatchSeries(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
// match_series
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/155
sut.PrometheusAPIV1ImportPrometheus(t, []string{
sut.APIV1ImportPrometheus(t, []string{
`GenBearTemp{db="TenMinute",Park="1",TurbineType="V112"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
`GenBearTemp{db="TenMinute",Park="2",TurbineType="V112"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
`GenBearTemp{db="TenMinute",Park="3",TurbineType="V112"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
@@ -217,12 +217,12 @@ func testMatchSeries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return sut.PrometheusAPIV1Series(t, `{__name__="GenBearTemp"}`, at.QueryOpts{
return sut.APIV1Series(t, `{__name__="GenBearTemp"}`, at.QueryOpts{
Start: "2024-02-04T08:57:36.700Z",
End: "2024-02-05T08:57:36.700Z",
}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Want: &at.APIV1SeriesResponse{
Status: "success",
IsPartial: false,
Data: []map[string]string{
@@ -235,54 +235,7 @@ func testMatchSeries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
})
}
func testNegativeIncrease(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
t := tc.T()
// negative increase when user overrides staleness interval
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8935#issuecomment-2978728661
sut.PrometheusAPIV1ImportPrometheus(t, []string{
`foo 108 1750109243514`, // 2025-06-16 21:27:23:514
`foo 108 1750109258514`, // 2025-06-16 21:27:38:514
// gap 75s
`foo 1 1750109333514`, // 2025-06-16 21:28:53:514
`foo 1 1750109348514`, // 2025-06-16 21:29:08:514
}, at.QueryOpts{})
sut.ForceFlush(t)
tc.Assert(&at.AssertOptions{
Msg: "regression for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8935#issuecomment-2978728661",
DoNotRetry: true,
Got: func() any {
return sut.PrometheusAPIV1QueryRange(t, `increase(foo[1m])`, at.QueryOpts{
Start: "2025-06-16T21:28:40.700Z",
End: "2025-06-16T21:29:30.700Z",
Step: "9s",
MaxLookback: "65s",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
Result: []*at.QueryResult{
{
Metric: map[string]string{},
Samples: []*at.Sample{
at.NewSample(t, "2025-06-16T21:28:40.700Z", 0),
at.NewSample(t, "2025-06-16T21:28:49.700Z", 0),
at.NewSample(t, "2025-06-16T21:28:58.700Z", 1),
at.NewSample(t, "2025-06-16T21:29:07.700Z", 1),
at.NewSample(t, "2025-06-16T21:29:16.700Z", 0),
at.NewSample(t, "2025-06-16T21:29:25.700Z", 0),
},
},
},
},
},
})
}
func testComparisonNotInfNotNan(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testComparisonNotInfNotNan(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
// comparison-not-inf-not-nan
@@ -306,13 +259,13 @@ func testComparisonNotInfNotNan(tc *at.TestCase, sut at.PrometheusWriteQuerier)
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return sut.PrometheusAPIV1QueryRange(t, `1/(not_nan_not_inf-1)!=inf!=nan`, at.QueryOpts{
return sut.APIV1QueryRange(t, `1/(not_nan_not_inf-1)!=inf!=nan`, at.QueryOpts{
Start: "2024-02-05T06:50:36.000Z",
End: "2024-02-05T09:58:37.000Z",
Step: "60",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -329,7 +282,7 @@ func testComparisonNotInfNotNan(tc *at.TestCase, sut at.PrometheusWriteQuerier)
})
}
func testEmptyLabelMatch(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testEmptyLabelMatch(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
// empty-label-match
@@ -352,13 +305,13 @@ func testEmptyLabelMatch(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return sut.PrometheusAPIV1QueryRange(t, `empty_label_match{foo=~'bar|'}`, at.QueryOpts{
return sut.APIV1QueryRange(t, `empty_label_match{foo=~'bar|'}`, at.QueryOpts{
Start: "2024-02-05T08:55:36.000Z",
End: "2024-02-05T08:57:36.000Z",
Step: "60s",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -381,7 +334,7 @@ func testEmptyLabelMatch(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
})
}
func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testMaxLookbehind(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
// max_lookback_set
@@ -405,14 +358,14 @@ func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return sut.PrometheusAPIV1QueryRange(t, `max_lookback_set{foo=~'bar|'}`, at.QueryOpts{
return sut.APIV1QueryRange(t, `max_lookback_set{foo=~'bar|'}`, at.QueryOpts{
Start: "2024-02-05T08:55:06.000Z",
End: "2024-02-05T08:57:37.000Z",
Step: "10s",
MaxLookback: "1s",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -452,13 +405,13 @@ func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return sut.PrometheusAPIV1QueryRange(t, `max_lookback_unset{foo=~'bar|'}`, at.QueryOpts{
return sut.APIV1QueryRange(t, `max_lookback_unset{foo=~'bar|'}`, at.QueryOpts{
Start: "2024-02-05T08:55:06.000Z",
End: "2024-02-05T08:57:37.000Z",
Step: "10s",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -489,7 +442,7 @@ func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
})
}
func testNonNanAsMissingData(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testNonNanAsMissingData(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
// not-nan-as-missing-data
@@ -513,13 +466,13 @@ func testNonNanAsMissingData(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query_range response",
Got: func() any {
return sut.PrometheusAPIV1QueryRange(t, `not_nan_as_missing_data>1`, at.QueryOpts{
return sut.APIV1QueryRange(t, `not_nan_as_missing_data>1`, at.QueryOpts{
Start: "2024-02-05T08:57:34.000Z",
End: "2024-02-05T08:57:36.000Z",
Step: "1s",
})
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "matrix",
@@ -544,7 +497,7 @@ func testNonNanAsMissingData(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
})
}
func testSubqueryAggregation(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
func testSubqueryAggregation(tc *at.TestCase, sut at.WriteQuerier) {
t := tc.T()
// subquery-aggregation
@@ -568,14 +521,14 @@ func testSubqueryAggregation(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
got := sut.PrometheusAPIV1Query(t, `min by (item) (min_over_time(forms_daily_count[10m:1m]))`, at.QueryOpts{
got := sut.APIV1Query(t, `min by (item) (min_over_time(forms_daily_count[10m:1m]))`, at.QueryOpts{
Time: "2024-02-05T08:56:35.000Z",
LatencyOffset: "1ms",
})
got.Sort()
return got
},
Want: &at.PrometheusAPIV1QueryResponse{
Want: &at.APIV1QueryResponse{
Status: "success",
Data: &at.QueryData{
ResultType: "vector",
@@ -594,7 +547,7 @@ func testSubqueryAggregation(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
})
}
func getRowsInsertedTotal(t *testing.T, sut at.PrometheusWriteQuerier) int {
func getRowsInsertedTotal(t *testing.T, sut at.WriteQuerier) int {
t.Helper()
selector := `vm_rows_inserted_total{type="graphite"}`

View File

@@ -1,154 +0,0 @@
package tests
import (
"fmt"
"os"
"path"
"testing"
"time"
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
)
// TestSingleVlagentRemoteWrite performs tests for remote write data ingestion
// by vlagent application
func TestSingleVlagentRemoteWrite(t *testing.T) {
os.RemoveAll(t.Name())
tc := at.NewTestCase(t)
defer tc.Stop()
// test data ingestion into
const instance = "vlsingle"
const r1Port = "50425"
sutFlags := []string{
"-httpListenAddr=127.0.0.1:" + r1Port,
"-storageDataPath=" + tc.Dir() + "/" + instance,
"-retentionPeriod=100y",
}
sut := tc.MustStartVlsingle(instance, sutFlags)
remoteWriteURL := fmt.Sprintf("http://%s/internal/insert", sut.HTTPAddr())
vlagent := tc.MustStartDefaultVlagent([]string{remoteWriteURL})
vlagent.JSONLineWrite(t, []string{
`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
}, at.QueryOptsLogs{})
sut.ForceFlush(t)
got := sut.LogsQLQuery(t, "ingest jsonline", at.QueryOptsLogs{})
wantLogLines := []string{
`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
}
assertLogsQLResponseEqual(t, got, &at.LogsQLQueryResponse{LogLines: wantLogLines})
// stop log storage and check data buffering works correctly
tc.StopApp(instance)
// ingest some data vlagent must hold it in memory
vlagent.JSONLineWrite(t, []string{
`{"_msg":"ingest jsonline2","_time": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
`{"_msg":"ingest jsonline2","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
}, at.QueryOptsLogs{})
vlagent.WaitQueueEmptyAfter(t, func() {
// start storage and check if buffered data correctly ingested
sut = tc.MustStartVlsingle(instance, sutFlags)
})
sut.ForceFlush(t)
got = sut.LogsQLQuery(t, "ingest jsonline2", at.QueryOptsLogs{})
wantLogLines = []string{
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
}
assertLogsQLResponseEqual(t, got, &at.LogsQLQueryResponse{LogLines: wantLogLines})
}
func TestSingleVlagentRemoteWriteReplication(t *testing.T) {
os.RemoveAll(t.Name())
tc := at.NewTestCase(t)
defer tc.Stop()
const (
instanceReplica0 = "vlsingle-0"
vlsinglePortR0 = "53541"
instanceReplica1 = "vlsingle-1"
vlsinglePortR1 = "53124"
vlagentInstance = "vlagent"
)
sutFlagsR0 := []string{
"-httpListenAddr=127.0.0.1:" + vlsinglePortR0,
"-storageDataPath=" + path.Join(tc.Dir(), instanceReplica0),
"-retentionPeriod=100y",
}
sutFlagsR1 := []string{
"-httpListenAddr=127.0.0.1:" + vlsinglePortR1,
"-storageDataPath=" + path.Join(tc.Dir(), instanceReplica1),
"-retentionPeriod=100y",
}
sutR0 := tc.MustStartVlsingle(instanceReplica0, sutFlagsR0)
sutR1 := tc.MustStartVlsingle(instanceReplica1, sutFlagsR1)
vlagentRemoteWriteURLs := []string{
fmt.Sprintf("http://%s/internal/insert", sutR0.HTTPAddr()),
fmt.Sprintf("http://%s/internal/insert", sutR1.HTTPAddr()),
}
vlagentFlags := []string{
"-remoteWrite.tmpDataPath=" + fmt.Sprintf("%s/%s-%d", os.TempDir(), vlagentInstance, time.Now().UnixNano()),
}
vlagent := tc.MustStartVlagent(vlagentInstance, vlagentRemoteWriteURLs, vlagentFlags)
// ingest data and check if it properly replicated to the vlsingles
vlagent.JSONLineWrite(t, []string{
`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
}, at.QueryOptsLogs{})
wantLogLines := []string{
`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
}
sutR0.ForceFlush(t)
gotR0 := sutR0.LogsQLQuery(t, "ingest jsonline", at.QueryOptsLogs{})
assertLogsQLResponseEqual(t, gotR0, &at.LogsQLQueryResponse{LogLines: wantLogLines})
sutR1.ForceFlush(t)
gotR1 := sutR1.LogsQLQuery(t, "ingest jsonline", at.QueryOptsLogs{})
assertLogsQLResponseEqual(t, gotR1, &at.LogsQLQueryResponse{LogLines: wantLogLines})
// stop log storage and check data buffering works correctly at vlagent
tc.StopApp(instanceReplica0)
// ingest some data vlagent must hold it in memory
vlagent.JSONLineWrite(t, []string{
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
}, at.QueryOptsLogs{})
// check alive storage received data
wantLogLines = []string{
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
`{"_msg":"ingest jsonline2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
}
sutR1.ForceFlush(t)
gotR1 = sutR1.LogsQLQuery(t, "ingest jsonline2", at.QueryOptsLogs{})
assertLogsQLResponseEqual(t, gotR1, &at.LogsQLQueryResponse{LogLines: wantLogLines})
// stop vmagent, it must buffer data on-disk
tc.StopApp(vlagentInstance)
vlagent = tc.MustStartVlagent(vlagentInstance, vlagentRemoteWriteURLs, vlagentFlags)
vlagent.WaitQueueEmptyAfter(t, func() {
// start storage and check if buffered data correctly ingested
sutR0 = tc.MustStartVlsingle(instanceReplica0, sutFlagsR0)
})
sutR0.ForceFlush(t)
gotR0 = sutR0.LogsQLQuery(t, "ingest jsonline2", at.QueryOptsLogs{})
assertLogsQLResponseEqual(t, gotR0, &at.LogsQLQueryResponse{LogLines: wantLogLines})
}

View File

@@ -45,12 +45,12 @@ func TestSingleVMAgentReloadConfigs(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: `unexpected metrics stored on vmagent remote write`,
Got: func() any {
return vmsingle.PrometheusAPIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
return vmsingle.APIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
Start: "2022-05-10T00:00:00Z",
End: "2022-05-10T23:59:59Z",
}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Want: &at.APIV1SeriesResponse{
Status: "success",
Data: []map[string]string{{"__name__": "foo_bar", "label1": "value1"}},
},
@@ -76,12 +76,12 @@ func TestSingleVMAgentReloadConfigs(t *testing.T) {
tc.Assert(&at.AssertOptions{
Msg: `unexpected metrics stored on vmagent remote write`,
Got: func() any {
return vmsingle.PrometheusAPIV1Series(t, `{__name__="bar_foo"}`, at.QueryOpts{
return vmsingle.APIV1Series(t, `{__name__="bar_foo"}`, at.QueryOpts{
Start: "2022-05-10T00:00:00Z",
End: "2022-05-10T23:59:59Z",
}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Want: &at.APIV1SeriesResponse{
Status: "success",
Data: []map[string]string{{"__name__": "bar_foo", "label1": "value2"}},
},
@@ -122,12 +122,12 @@ func testSingleVMAgentRemoteWrite(t *testing.T, forcePromProto bool) {
tc.Assert(&at.AssertOptions{
Msg: `unexpected metrics stored on vmagent remote write`,
Got: func() any {
return vmsingle.PrometheusAPIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
return vmsingle.APIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
Start: "2022-05-10T00:00:00Z",
End: "2022-05-10T23:59:59Z",
}).Sort()
},
Want: &at.PrometheusAPIV1SeriesResponse{
Want: &at.APIV1SeriesResponse{
Status: "success",
Data: []map[string]string{{"__name__": "foo_bar"}},
},

View File

@@ -77,19 +77,19 @@ func TestClusterTenantsToTenantsVmctlNativeProtocol(t *testing.T) {
testVmctlNativeProtocol(tc, clusterSrc, clusterDst, flags)
}
func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.PrometheusWriteQuerier, dstSut apptest.PrometheusWriteQuerier, vmctlFlags []string) {
func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.WriteQuerier, dstSut apptest.WriteQuerier, vmctlFlags []string) {
t := tc.T()
t.Helper()
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
// test for empty data request in the source
got := srcSut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
got := srcSut.APIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
Step: "5m",
Time: "2025-05-30T12:45:00Z",
})
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
@@ -116,14 +116,14 @@ func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.PrometheusWrit
dataSet[i] = fmt.Sprintf("%s %d %s", metricsName, i, ingestTimestamp)
}
wantResponse := apptest.PrometheusAPIV1QueryResponse{
wantResponse := apptest.APIV1QueryResponse{
Status: "success",
Data: &expectedQueryData,
}
wantResponse.Sort()
srcSut.PrometheusAPIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{})
srcSut.APIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{})
srcSut.ForceFlush(t)
tc.MustStartVmctl("vmctl", vmctlFlags)
@@ -134,7 +134,7 @@ func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.PrometheusWrit
Retries: 300,
Msg: `unexpected metrics stored on vmsingle via the native protocol`,
Got: func() any {
exported := dstSut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
exported := dstSut.APIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
Start: "2025-05-30T16:39:36Z",
End: "2025-05-30T16:39:37Z",
})
@@ -143,7 +143,7 @@ func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.PrometheusWrit
},
Want: wantResponse.Data.Result,
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}

View File

@@ -55,19 +55,19 @@ func TestClusterVmctlPrometheusProtocol(t *testing.T) {
testPrometheusProtocol(tc, cluster, vmctlFlags)
}
func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, vmctlFlags []string) {
func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.WriteQuerier, vmctlFlags []string) {
t := tc.T()
t.Helper()
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
// test for empty data request
got := sut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
got := sut.APIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
Step: "5m",
Time: "2025-06-02T17:14:00Z",
})
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
@@ -88,7 +88,7 @@ func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
t.Fatalf("cannot read expected series response file: %s", err)
}
var wantResponse apptest.PrometheusAPIV1QueryResponse
var wantResponse apptest.APIV1QueryResponse
if err := json.Unmarshal(bytes, &wantResponse); err != nil {
t.Fatalf("cannot unmarshal expected series response file: %s", err)
}
@@ -99,7 +99,7 @@ func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
Retries: 300,
Msg: `unexpected metrics stored on vmsingle via the prometheus protocol`,
Got: func() any {
expected := sut.PrometheusAPIV1Export(t, `{__name__="vm_log_messages_total", location=~"VictoriaMetrics/lib/ingestserver/opentsdb/server.go:(48|59)"}`, apptest.QueryOpts{
expected := sut.APIV1Export(t, `{__name__="vm_log_messages_total", location=~"VictoriaMetrics/lib/ingestserver/opentsdb/server.go:(48|59)"}`, apptest.QueryOpts{
Start: "2025-06-02T00:00:00Z",
End: "2025-06-02T23:59:59Z",
})
@@ -108,7 +108,7 @@ func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
},
Want: wantResponse.Data.Result,
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}

View File

@@ -75,7 +75,7 @@ func TestClusterVmctlRemoteReadProtocol(t *testing.T) {
testRemoteReadProtocol(tc, clusterDst, newRemoteReadServer, vmctlFlags)
}
func testRemoteReadProtocol(tc *at.TestCase, sut at.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
func testRemoteReadProtocol(tc *at.TestCase, sut at.WriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
t := tc.T()
t.Helper()
@@ -84,14 +84,14 @@ func testRemoteReadProtocol(tc *at.TestCase, sut at.PrometheusWriteQuerier, newR
expectedResult := transformSeriesToQueryResult(rrs.storage.store)
cmpOpt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
cmpOpt := cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType")
// test for empty data request
got := sut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, at.QueryOpts{
got := sut.APIV1Query(t, `{__name__=~".*"}`, at.QueryOpts{
Step: "5m",
Time: "2025-06-02T17:14:00Z",
})
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
want := at.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
@@ -106,7 +106,7 @@ func testRemoteReadProtocol(tc *at.TestCase, sut at.PrometheusWriteQuerier, newR
Retries: 300,
Msg: `unexpected metrics stored on vmsingle via the prometheus protocol`,
Got: func() any {
expected := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, at.QueryOpts{
expected := sut.APIV1Export(t, `{__name__=~".*"}`, at.QueryOpts{
Start: "2025-06-11T15:31:10Z",
End: "2025-06-11T15:32:20Z",
})
@@ -115,7 +115,7 @@ func testRemoteReadProtocol(tc *at.TestCase, sut at.PrometheusWriteQuerier, newR
},
Want: expectedResult,
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}

View File

@@ -1,159 +0,0 @@
package apptest
import (
"fmt"
"net/http"
"os"
"regexp"
"strings"
"testing"
"time"
)
// Vlagent holds the state of a vlagent app and provides vlagent-specific functions
type Vlagent struct {
*app
*ServesMetrics
remoteStoragesCount int
httpListenAddr string
}
// StartVlagent starts an instance of vlagent with the given flags.
// It also sets the default flags and populates the app instance state with runtime
// values extracted from the application log (such as httpListenAddr)
func StartVlagent(instance string, remoteWriteURLs []string, flags []string, cli *Client) (*Vlagent, error) {
extractREs := []*regexp.Regexp{
httpListenAddrRE,
}
app, stderrExtracts, err := startApp(instance, "../../bin/vlagent", flags, &appOptions{
defaultFlags: map[string]string{
"-httpListenAddr": "127.0.0.1:0",
"-remoteWrite.url": strings.Join(remoteWriteURLs, ","),
"-remoteWrite.tmpDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
"-remoteWrite.flushInterval": "10ms",
"-remoteWrite.showURL": "true",
},
extractREs: extractREs,
})
if err != nil {
return nil, err
}
return &Vlagent{
app: app,
remoteStoragesCount: len(remoteWriteURLs),
ServesMetrics: &ServesMetrics{
metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
cli: cli,
},
httpListenAddr: stderrExtracts[0],
}, nil
}
// JSONLineWrite is a test helper function that inserts a
// collection of records in json line format by sending a HTTP
// POST request to /insert/jsonline vlagent endpoint.
//
// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api
func (app *Vlagent) JSONLineWrite(t *testing.T, records []string, opts QueryOptsLogs) {
t.Helper()
data := []byte(strings.Join(records, "\n"))
url := fmt.Sprintf("http://%s/insert/jsonline", app.httpListenAddr)
uv := opts.asURLValues()
uvs := uv.Encode()
if len(uvs) > 0 {
url += "?" + uvs
}
app.sendBlocking(t, len(records), func() {
_, statusCode := app.cli.Post(t, url, "text/plain", data)
if statusCode != http.StatusOK {
t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
}
})
}
// WaitQueueEmptyAfter checks that persistent queue is empty
// after execution of provided callback
func (app *Vlagent) WaitQueueEmptyAfter(t *testing.T, cb func()) {
t.Helper()
const (
retries = 70
period = 100 * time.Millisecond
)
// vlagent_remotewrite_blocks_sent_total
// take in account data replication
blocksSent := app.remoteWriteBlocksSent(t)
cb()
for range retries {
if app.remoteWriteBlocksSent(t) > blocksSent && app.persistentQueueSize(t) == 0 {
return
}
time.Sleep(period)
}
t.Fatalf("timed out while waiting for inserted logs to be flushed to remote storage")
}
// sendBlocking sends the data to remote write url by executing `send` function and
// waits until the data is actually sent.
//
// vlagent does not send the data immediately. It first puts the data into a
// buffer. Then a background goroutine takes the data from the buffer sends it
// to the vmstorage. This happens every 1s by default.
//
// Waiting is implemented a retrieving the value of `vlagent_remotewrite_block_size_rows_sum`
// metric and checking whether it is equal or greater than the wanted value.
// If it is, then the data has been sent to remote storage.
//
// Unreliable if the records are inserted concurrently.
func (app *Vlagent) sendBlocking(t *testing.T, numRecordsToSend int, send func()) {
t.Helper()
send()
const (
retries = 50
period = 100 * time.Millisecond
)
// take in account data replication
wantRowsSentCount := app.remoteWriteRowsPushed(t) + numRecordsToSend*app.remoteStoragesCount
for range retries {
if app.remoteWriteRowsPushed(t) >= wantRowsSentCount {
return
}
time.Sleep(period)
}
t.Fatalf("timed out while waiting for inserted rows to be sent to remote storage")
}
func (app *Vlagent) remoteWriteBlocksSent(t *testing.T) int {
total := 0.0
for _, v := range app.GetMetricsByPrefix(t, "vlagent_remotewrite_blocks_sent_total") {
total += v
}
return int(total)
}
func (app *Vlagent) remoteWriteRowsPushed(t *testing.T) int {
total := 0.0
// vlagent_remotewrite_blocks_sent_total
for _, v := range app.GetMetricsByPrefix(t, "vlagent_remotewrite_block_size_rows_sum") {
total += v
}
return int(total)
}
func (app *Vlagent) persistentQueueSize(t *testing.T) int {
total := 0.0
for _, v := range app.GetMetricsByPrefix(t, "vlagent_remotewrite_pending_data_bytes") {
total += v
}
for _, v := range app.GetMetricsByPrefix(t, "vlagent_remotewrite_pending_inmemory_blocks") {
total += v
}
return int(total)
}

View File

@@ -127,12 +127,12 @@ func (app *Vminsert) GraphiteWrite(t *testing.T, records []string, _ QueryOpts)
app.cli.Write(t, app.graphiteListenAddr, records)
}
// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
// APIV1ImportCSV is a test helper function that inserts a collection
// of records in CSV format for the given tenant by sending an HTTP POST
// request to prometheus/api/v1/import/csv vminsert endpoint.
//
// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
func (app *Vminsert) APIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
t.Helper()
url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/csv", app.httpListenAddr, opts.getTenant())
@@ -150,12 +150,12 @@ func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, op
})
}
// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
// APIV1ImportNative is a test helper function that inserts a collection
// of records in Native format for the given tenant by sending an HTTP POST
// request to prometheus/api/v1/import/native vminsert endpoint.
//
// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
func (app *Vminsert) APIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
t.Helper()
url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/native", app.httpListenAddr, opts.getTenant())
@@ -195,10 +195,10 @@ func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOp
})
}
// PrometheusAPIV1Write is a test helper function that inserts a
// APIV1Write is a test helper function that inserts a
// collection of records in Prometheus remote-write format by sending a HTTP
// POST request to /prometheus/api/v1/write vminsert endpoint.
func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts) {
func (app *Vminsert) APIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts) {
t.Helper()
url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.getTenant())
@@ -212,13 +212,13 @@ func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries,
})
}
// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
// APIV1ImportPrometheus is a test helper function that inserts a
// collection of records in Prometheus text exposition format for the given
// tenant by sending a HTTP POST request to
// /prometheus/api/v1/import/prometheus vminsert endpoint.
//
// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
func (app *Vminsert) APIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
t.Helper()
url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, opts.getTenant())

Some files were not shown because too many files have changed in this diff Show More