mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-05-17 16:59:40 +03:00
Compare commits
2 Commits
roaring-bi
...
sort-order
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d4208f2cde | ||
|
|
cce1b50df5 |
3
.github/workflows/build.yml
vendored
3
.github/workflows/build.yml
vendored
@@ -71,8 +71,7 @@ jobs:
|
||||
go.sum
|
||||
Makefile
|
||||
app/**/Makefile
|
||||
go-version-file: 'go.mod'
|
||||
- run: go version
|
||||
go-version: stable
|
||||
|
||||
- name: Build victoria-metrics for ${{ matrix.os }}-${{ matrix.arch }}
|
||||
run: make victoria-metrics-${{ matrix.os }}-${{ matrix.arch }}
|
||||
|
||||
6
.github/workflows/check-licenses.yml
vendored
6
.github/workflows/check-licenses.yml
vendored
@@ -21,11 +21,9 @@ jobs:
|
||||
id: go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: stable
|
||||
cache: false
|
||||
|
||||
- run: go version
|
||||
|
||||
- name: Cache Go artifacts
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
@@ -34,7 +32,7 @@ jobs:
|
||||
~/go/pkg/mod
|
||||
~/go/bin
|
||||
key: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
|
||||
restore-keys: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-
|
||||
restore-keys: go-artifacts-${{ runner.os }}-check-licenses-
|
||||
|
||||
- name: Check License
|
||||
run: make check-licenses
|
||||
|
||||
5
.github/workflows/codeql-analysis-go.yml
vendored
5
.github/workflows/codeql-analysis-go.yml
vendored
@@ -36,8 +36,7 @@ jobs:
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
cache: false
|
||||
go-version-file: 'go.mod'
|
||||
- run: go version
|
||||
go-version: stable
|
||||
|
||||
- name: Cache Go artifacts
|
||||
uses: actions/cache@v4
|
||||
@@ -47,7 +46,7 @@ jobs:
|
||||
~/go/bin
|
||||
~/go/pkg/mod
|
||||
key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
|
||||
restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
|
||||
restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
|
||||
23
.github/workflows/test.yml
vendored
23
.github/workflows/test.yml
vendored
@@ -42,9 +42,8 @@ jobs:
|
||||
go.sum
|
||||
Makefile
|
||||
app/**/Makefile
|
||||
go-version-file: 'go.mod'
|
||||
go-version: stable
|
||||
|
||||
- run: go version
|
||||
|
||||
- name: Cache golangci-lint
|
||||
uses: actions/cache@v4
|
||||
@@ -52,7 +51,7 @@ jobs:
|
||||
path: |
|
||||
~/.cache/golangci-lint
|
||||
~/go/bin
|
||||
key: golangci-lint-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('.golangci.yml') }}
|
||||
key: golangci-lint-${{ runner.os }}-${{ hashFiles('.golangci.yml') }}
|
||||
|
||||
- name: Run check-all
|
||||
run: |
|
||||
@@ -82,20 +81,19 @@ jobs:
|
||||
go.sum
|
||||
Makefile
|
||||
app/**/Makefile
|
||||
go-version-file: 'go.mod'
|
||||
- run: go version
|
||||
go-version: stable
|
||||
|
||||
- name: Run tests
|
||||
run: make ${{ matrix.scenario}}
|
||||
run: GOGC=10 make ${{ matrix.scenario}}
|
||||
|
||||
- name: Publish coverage
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
files: ./coverage.txt
|
||||
|
||||
apptest:
|
||||
name: apptest
|
||||
runs-on: apptest
|
||||
integration:
|
||||
name: integration
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Code checkout
|
||||
@@ -109,8 +107,7 @@ jobs:
|
||||
go.sum
|
||||
Makefile
|
||||
app/**/Makefile
|
||||
go-version-file: 'go.mod'
|
||||
- run: go version
|
||||
go-version: stable
|
||||
|
||||
- name: Run app tests
|
||||
run: make apptest
|
||||
- name: Run integration tests
|
||||
run: make integration-test
|
||||
|
||||
31
Makefile
31
Makefile
@@ -17,7 +17,7 @@ EXTRA_GO_BUILD_TAGS ?=
|
||||
GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
|
||||
TAR_OWNERSHIP ?= --owner=1000 --group=1000
|
||||
|
||||
GOLANGCI_LINT_VERSION := 2.9.0
|
||||
GOLANGCI_LINT_VERSION := 2.7.2
|
||||
|
||||
.PHONY: $(MAKECMDGOALS)
|
||||
|
||||
@@ -443,7 +443,7 @@ fmt:
|
||||
gofmt -l -w -s ./apptest
|
||||
|
||||
vet:
|
||||
go vet -tags 'synctest' ./lib/...
|
||||
GOEXPERIMENT=synctest go vet ./lib/...
|
||||
go vet ./app/...
|
||||
go vet ./apptest/...
|
||||
|
||||
@@ -452,25 +452,28 @@ check-all: fmt vet golangci-lint govulncheck
|
||||
clean-checkers: remove-golangci-lint remove-govulncheck
|
||||
|
||||
test:
|
||||
go test -tags 'synctest' ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest go test ./lib/... ./app/...
|
||||
|
||||
test-race:
|
||||
go test -tags 'synctest' -race ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
|
||||
|
||||
test-pure:
|
||||
CGO_ENABLED=0 go test -tags 'synctest' ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
|
||||
|
||||
test-full:
|
||||
go test -tags 'synctest' -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
|
||||
test-full-386:
|
||||
GOARCH=386 go test -tags 'synctest' -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
|
||||
integration-test:
|
||||
$(MAKE) apptest
|
||||
|
||||
apptest:
|
||||
$(MAKE) victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
|
||||
go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
|
||||
|
||||
apptest-legacy: victoria-metrics vmbackup vmrestore
|
||||
integration-test-legacy: victoria-metrics vmbackup vmrestore
|
||||
OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
|
||||
ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
|
||||
VERSION=v1.132.0; \
|
||||
@@ -487,17 +490,17 @@ apptest-legacy: victoria-metrics vmbackup vmrestore
|
||||
go test ./apptest/tests -run="^TestLegacySingle.*"
|
||||
|
||||
benchmark:
|
||||
go test -run=NO_TESTS -bench=. ./lib/...
|
||||
go test -run=NO_TESTS -bench=. ./app/...
|
||||
GOEXPERIMENT=synctest go test -bench=. ./lib/...
|
||||
go test -bench=. ./app/...
|
||||
|
||||
benchmark-pure:
|
||||
CGO_ENABLED=0 go test -run=NO_TESTS -bench=. ./lib/...
|
||||
CGO_ENABLED=0 go test -run=NO_TESTS -bench=. ./app/...
|
||||
GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
|
||||
CGO_ENABLED=0 go test -bench=. ./app/...
|
||||
|
||||
vendor-update:
|
||||
go get -u ./lib/...
|
||||
go get -u ./app/...
|
||||
go mod tidy -compat=1.26
|
||||
go mod tidy -compat=1.24
|
||||
go mod vendor
|
||||
|
||||
app-local:
|
||||
@@ -521,7 +524,7 @@ install-qtc:
|
||||
|
||||
|
||||
golangci-lint: install-golangci-lint
|
||||
golangci-lint run --build-tags 'synctest'
|
||||
GOEXPERIMENT=synctest golangci-lint run
|
||||
|
||||
install-golangci-lint:
|
||||
which golangci-lint && (golangci-lint --version | grep -q $(GOLANGCI_LINT_VERSION)) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v$(GOLANGCI_LINT_VERSION)
|
||||
|
||||
19
README.md
19
README.md
@@ -16,21 +16,16 @@
|
||||
<img src="docs/victoriametrics/logo.webp" width="300" alt="VictoriaMetrics logo">
|
||||
</picture>
|
||||
|
||||
VictoriaMetrics is a fast, cost-effective, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
|
||||
VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
|
||||
|
||||
Here are some resources and information about VictoriaMetrics:
|
||||
|
||||
- **Case studies**: [Grammarly, Roblox, Wix, Spotify,...](https://docs.victoriametrics.com/victoriametrics/casestudies/).
|
||||
- **Available**: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), Docker images on [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics).
|
||||
- **Deployment types**: [Single-node version](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [Cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) under [Apache License 2.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE).
|
||||
- **Getting started:** Read [key concepts](https://docs.victoriametrics.com/victoriametrics/keyconcepts/) and follow the
|
||||
[quick start guide](https://docs.victoriametrics.com/victoriametrics/quick-start/).
|
||||
- **Community**: [Slack](https://slack.victoriametrics.com/) (join via [Slack Inviter](https://slack.victoriametrics.com/)), [X (Twitter)](https://x.com/VictoriaMetrics), [YouTube](https://www.youtube.com/@VictoriaMetrics). See full list [here](https://docs.victoriametrics.com/victoriametrics/#community-and-contributions).
|
||||
- **Changelog**: Project evolves fast - check the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/), and [How to upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
|
||||
- **Enterprise support:** [Contact us](mailto:info@victoriametrics.com) for commercial support with additional [enterprise features](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
- **Enterprise releases:** Enterprise and [long-term support releases (LTS)](https://docs.victoriametrics.com/victoriametrics/lts-releases/) are publicly available and can be evaluated for free
|
||||
using a [free trial license](https://victoriametrics.com/products/enterprise/trial/).
|
||||
- **Security:** we achieved [security certifications](https://victoriametrics.com/security/) for Database Software Development and Software-Based Monitoring Services.
|
||||
- Documentation: [docs.victoriametrics.com](https://docs.victoriametrics.com)
|
||||
- Case studies: [Grammarly, Roblox, Wix,...](https://docs.victoriametrics.com/victoriametrics/casestudies/).
|
||||
- Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), docker images [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
|
||||
- Deployment types: [Single-node version](https://docs.victoriametrics.com/), [Cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/)
|
||||
- Changelog: [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/), and [How to upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics)
|
||||
- Community: [Slack](https://slack.victoriametrics.com/), [X (Twitter)](https://x.com/VictoriaMetrics), [LinkedIn](https://www.linkedin.com/company/victoriametrics/), [YouTube](https://www.youtube.com/@VictoriaMetrics)
|
||||
|
||||
Yes, we open-source both the single-node VictoriaMetrics and the cluster version.
|
||||
|
||||
|
||||
@@ -33,13 +33,13 @@ func PopulateTimeTpl(b []byte, tGlobal time.Time) []byte {
|
||||
}
|
||||
switch strings.TrimSpace(parts[0]) {
|
||||
case `TIME_S`:
|
||||
return fmt.Appendf(nil, "%d", t.Unix())
|
||||
return []byte(fmt.Sprintf("%d", t.Unix()))
|
||||
case `TIME_MSZ`:
|
||||
return fmt.Appendf(nil, "%d", t.Unix()*1e3)
|
||||
return []byte(fmt.Sprintf("%d", t.Unix()*1e3))
|
||||
case `TIME_MS`:
|
||||
return fmt.Appendf(nil, "%d", timeToMillis(t))
|
||||
return []byte(fmt.Sprintf("%d", timeToMillis(t)))
|
||||
case `TIME_NS`:
|
||||
return fmt.Appendf(nil, "%d", t.UnixNano())
|
||||
return []byte(fmt.Sprintf("%d", t.UnixNano()))
|
||||
default:
|
||||
log.Fatalf("unknown time pattern %s in %s", parts[0], repl)
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestCalculateRetryDuration(t *testing.T) {
|
||||
f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
for range n {
|
||||
for i := 0; i < n; i++ {
|
||||
retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
|
||||
}
|
||||
|
||||
|
||||
@@ -51,9 +51,9 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
|
||||
|
||||
func newTestWriteRequest(seriesCount, labelsCount int) *prompb.WriteRequest {
|
||||
var wr prompb.WriteRequest
|
||||
for i := range seriesCount {
|
||||
for i := 0; i < seriesCount; i++ {
|
||||
var labels []prompb.Label
|
||||
for j := range labelsCount {
|
||||
for j := 0; j < labelsCount; j++ {
|
||||
labels = append(labels, prompb.Label{
|
||||
Name: fmt.Sprintf("label_%d_%d", i, j),
|
||||
Value: fmt.Sprintf("value_%d_%d", i, j),
|
||||
|
||||
@@ -20,7 +20,8 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to all -remoteWrite.url.")
|
||||
unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
|
||||
"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
|
||||
relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
|
||||
"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
|
||||
"The path can point either to local file or to http url. "+
|
||||
@@ -38,7 +39,7 @@ var (
|
||||
labelsGlobal []prompb.Label
|
||||
|
||||
remoteWriteRelabelConfigData atomic.Pointer[[]byte]
|
||||
remoteWriteURLRelabelConfigData atomic.Pointer[[]any]
|
||||
remoteWriteURLRelabelConfigData atomic.Pointer[[]interface{}]
|
||||
|
||||
relabelConfigReloads *metrics.Counter
|
||||
relabelConfigReloadErrors *metrics.Counter
|
||||
@@ -90,8 +91,8 @@ func WriteURLRelabelConfigData(w io.Writer) {
|
||||
return
|
||||
}
|
||||
type urlRelabelCfg struct {
|
||||
Url string `yaml:"url"`
|
||||
RelabelConfig any `yaml:"relabel_config"`
|
||||
Url string `yaml:"url"`
|
||||
RelabelConfig interface{} `yaml:"relabel_config"`
|
||||
}
|
||||
var cs []urlRelabelCfg
|
||||
for i, url := range *remoteWriteURLs {
|
||||
@@ -144,7 +145,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
|
||||
len(*relabelConfigPaths), (len(*remoteWriteURLs)))
|
||||
}
|
||||
|
||||
var urlRelabelCfgs []any
|
||||
var urlRelabelCfgs []interface{}
|
||||
rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
|
||||
for i, path := range *relabelConfigPaths {
|
||||
if len(path) == 0 {
|
||||
@@ -157,7 +158,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
|
||||
}
|
||||
rcs.perURL[i] = prc
|
||||
|
||||
var parsedCfg any
|
||||
var parsedCfg interface{}
|
||||
_ = yaml.Unmarshal(rawCfg, &parsedCfg)
|
||||
urlRelabelCfgs = append(urlRelabelCfgs, parsedCfg)
|
||||
}
|
||||
|
||||
@@ -1080,7 +1080,7 @@ func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries)
|
||||
}()
|
||||
|
||||
if len(labelsGlobal) > 0 {
|
||||
// Make a copy of tss before adding extra labels to prevent
|
||||
// Make a copy of tss before adding extra labels in order to prevent
|
||||
// from affecting time series for other remoteWrite.url configs.
|
||||
rctx = getRelabelCtx()
|
||||
v = tssPool.Get().(*[]prompb.TimeSeries)
|
||||
|
||||
@@ -28,12 +28,12 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
|
||||
itemsCount := 1_000 * bucketsCount
|
||||
m := make([]int, bucketsCount)
|
||||
var labels []prompb.Label
|
||||
for i := range itemsCount {
|
||||
for i := 0; i < itemsCount; i++ {
|
||||
labels = append(labels[:0], prompb.Label{
|
||||
Name: "__name__",
|
||||
Value: fmt.Sprintf("some_name_%d", i),
|
||||
})
|
||||
for j := range 10 {
|
||||
for j := 0; j < 10; j++ {
|
||||
labels = append(labels, prompb.Label{
|
||||
Name: fmt.Sprintf("label_%d", j),
|
||||
Value: fmt.Sprintf("value_%d_%d", i, j),
|
||||
@@ -248,7 +248,7 @@ func TestShardAmountRemoteWriteCtx(t *testing.T) {
|
||||
seriesCount := 100000
|
||||
// build 1000000 series
|
||||
tssBlock := make([]prompb.TimeSeries, 0, seriesCount)
|
||||
for i := range seriesCount {
|
||||
for i := 0; i < seriesCount; i++ {
|
||||
tssBlock = append(tssBlock, prompb.TimeSeries{
|
||||
Labels: []prompb.Label{
|
||||
{
|
||||
@@ -269,7 +269,7 @@ func TestShardAmountRemoteWriteCtx(t *testing.T) {
|
||||
// build active time series set
|
||||
nodes := make([]string, 0, remoteWriteCount)
|
||||
activeTimeSeriesByNodes := make([]map[string]struct{}, remoteWriteCount)
|
||||
for i := range remoteWriteCount {
|
||||
for i := 0; i < remoteWriteCount; i++ {
|
||||
nodes = append(nodes, fmt.Sprintf("node%d", i))
|
||||
activeTimeSeriesByNodes[i] = make(map[string]struct{})
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ func TestParseInputValue_Success(t *testing.T) {
|
||||
if len(outputExpected) != len(output) {
|
||||
t.Fatalf("unexpected output length; got %d; want %d", len(outputExpected), len(output))
|
||||
}
|
||||
for i := range outputExpected {
|
||||
for i := 0; i < len(outputExpected); i++ {
|
||||
if outputExpected[i].Omitted != output[i].Omitted {
|
||||
t.Fatalf("unexpected Omitted field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
@@ -350,7 +348,9 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
|
||||
for k := range alertEvalTimesMap {
|
||||
alertEvalTimes = append(alertEvalTimes, k)
|
||||
}
|
||||
slices.Sort(alertEvalTimes)
|
||||
sort.Slice(alertEvalTimes, func(i, j int) bool {
|
||||
return alertEvalTimes[i] < alertEvalTimes[j]
|
||||
})
|
||||
|
||||
// sort group eval order according to the given "group_eval_order".
|
||||
sort.Slice(testGroups, func(i, j int) bool {
|
||||
@@ -361,8 +361,12 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
|
||||
var groups []*rule.Group
|
||||
for _, group := range testGroups {
|
||||
mergedExternalLabels := make(map[string]string)
|
||||
maps.Copy(mergedExternalLabels, tg.ExternalLabels)
|
||||
maps.Copy(mergedExternalLabels, externalLabels)
|
||||
for k, v := range tg.ExternalLabels {
|
||||
mergedExternalLabels[k] = v
|
||||
}
|
||||
for k, v := range externalLabels {
|
||||
mergedExternalLabels[k] = v
|
||||
}
|
||||
ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
|
||||
ng.Init()
|
||||
groups = append(groups, ng)
|
||||
|
||||
@@ -2,7 +2,6 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaLogs/lib/logstorage"
|
||||
@@ -81,8 +80,12 @@ func (t *Type) ValidateExpr(expr string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain labels from LogsQL expr: %q, err: %w", expr, err)
|
||||
}
|
||||
if slices.Contains(labels, "_time") {
|
||||
return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
|
||||
for i := range labels {
|
||||
// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
|
||||
// making the result meaningless and may lead to cardinality issues.
|
||||
if labels[i] == "_time" {
|
||||
return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
|
||||
}
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown datasource type=%q", t.Name)
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -92,7 +91,9 @@ func (c *Client) Clone() *Client {
|
||||
ns.extraHeaders = make([]keyValue, len(c.extraHeaders))
|
||||
copy(ns.extraHeaders, c.extraHeaders)
|
||||
}
|
||||
maps.Copy(ns.extraParams, c.extraParams)
|
||||
for k, v := range c.extraParams {
|
||||
ns.extraParams[k] = v
|
||||
}
|
||||
|
||||
return ns
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ type promResponse struct {
|
||||
// Stats supported by VictoriaMetrics since v1.90
|
||||
Stats struct {
|
||||
SeriesFetched *string `json:"seriesFetched,omitempty"`
|
||||
} `json:"stats"`
|
||||
} `json:"stats,omitempty"`
|
||||
// IsPartial supported by VictoriaMetrics
|
||||
IsPartial *bool `json:"isPartial,omitempty"`
|
||||
}
|
||||
|
||||
@@ -134,7 +134,7 @@ func (ls Labels) String() string {
|
||||
func LabelCompare(a, b Labels) int {
|
||||
l := min(len(b), len(a))
|
||||
|
||||
for i := range l {
|
||||
for i := 0; i < l; i++ {
|
||||
if a[i].Name != b[i].Name {
|
||||
if a[i].Name < b[i].Name {
|
||||
return -1
|
||||
|
||||
@@ -13,7 +13,7 @@ func BenchmarkPromInstantUnmarshal(b *testing.B) {
|
||||
|
||||
// BenchmarkParsePrometheusResponse/Instant_std+fastjson-10 1760 668959 ns/op 280147 B/op 5781 allocs/op
|
||||
b.Run("Instant std+fastjson", func(b *testing.B) {
|
||||
for range b.N {
|
||||
for i := 0; i < b.N; i++ {
|
||||
var pi promInstant
|
||||
err = pi.Unmarshal(data)
|
||||
if err != nil {
|
||||
|
||||
@@ -69,7 +69,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
|
||||
for n := range workers {
|
||||
wg.Go(func() {
|
||||
r := rand.New(rand.NewSource(int64(n)))
|
||||
for range iterations {
|
||||
for i := 0; i < iterations; i++ {
|
||||
rnd := r.Intn(len(paths))
|
||||
cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
|
||||
if err != nil { // update can fail and this is expected
|
||||
@@ -259,7 +259,7 @@ func compareGroups(t *testing.T, a, b *rule.Group) {
|
||||
for i, r := range a.Rules {
|
||||
got, want := r, b.Rules[i]
|
||||
if a.CreateID() != b.CreateID() {
|
||||
t.Fatalf("expected to have rule %d; got %d", want.ID(), got.ID())
|
||||
t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
|
||||
}
|
||||
if err := rule.CompareRules(t, want, got); err != nil {
|
||||
t.Fatalf("comparison error: %s", err)
|
||||
|
||||
@@ -216,7 +216,7 @@ consul_sd_configs:
|
||||
for n := range workers {
|
||||
wg.Go(func() {
|
||||
r := rand.New(rand.NewSource(int64(n)))
|
||||
for range iterations {
|
||||
for i := 0; i < iterations; i++ {
|
||||
rnd := r.Intn(len(paths))
|
||||
_ = cw.reload(paths[rnd]) // update can fail and this is expected
|
||||
_ = cw.notifiers()
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
@@ -290,7 +291,7 @@ func GetTargets() map[TargetType][]Target {
|
||||
}
|
||||
|
||||
// Send sends alerts to all active notifiers
|
||||
func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) chan error {
|
||||
func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) *vmalertutil.ErrGroup {
|
||||
alertsToSend := make([]Alert, 0, len(alerts))
|
||||
lblss := make([][]prompb.Label, 0, len(alerts))
|
||||
// apply global relabel config first without modifying original alerts in alerts
|
||||
@@ -303,18 +304,17 @@ func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string
|
||||
lblss = append(lblss, lbls)
|
||||
}
|
||||
|
||||
errGr := new(vmalertutil.ErrGroup)
|
||||
wg := sync.WaitGroup{}
|
||||
activeNotifiers := getActiveNotifiers()
|
||||
errCh := make(chan error, len(activeNotifiers))
|
||||
defer close(errCh)
|
||||
for i := range activeNotifiers {
|
||||
nt := activeNotifiers[i]
|
||||
wg.Go(func() {
|
||||
if err := nt.Send(ctx, alertsToSend, lblss, notifierHeaders); err != nil {
|
||||
errCh <- fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err)
|
||||
errGr.Add(fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err))
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
return errCh
|
||||
return errGr
|
||||
}
|
||||
|
||||
@@ -205,9 +205,7 @@ alert_relabel_configs:
|
||||
},
|
||||
}
|
||||
errG := Send(context.Background(), firingAlerts, nil)
|
||||
for err := range errG {
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error when sending alerts: %s", err)
|
||||
}
|
||||
if errG.Err() != nil {
|
||||
t.Fatalf("unexpected error when sending alerts: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,7 +113,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
|
||||
input: make(chan prompb.TimeSeries, cfg.MaxQueueSize),
|
||||
}
|
||||
|
||||
for range cc {
|
||||
for i := 0; i < cc; i++ {
|
||||
c.run(ctx)
|
||||
}
|
||||
return c, nil
|
||||
@@ -238,10 +238,8 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
|
||||
defer func() {
|
||||
sendDuration.Add(time.Since(timeStart).Seconds())
|
||||
}()
|
||||
|
||||
attempts := 0
|
||||
L:
|
||||
for {
|
||||
for attempts := 0; ; attempts++ {
|
||||
err := c.send(ctx, b)
|
||||
if err != nil && (errors.Is(err, io.EOF) || netutil.IsTrivialNetworkError(err)) {
|
||||
// Something in the middle between client and destination might be closing
|
||||
@@ -283,7 +281,6 @@ L:
|
||||
time.Sleep(retryInterval)
|
||||
retryInterval *= 2
|
||||
|
||||
attempts++
|
||||
}
|
||||
|
||||
rwErrors.Inc()
|
||||
|
||||
@@ -44,7 +44,7 @@ func TestClient_Push(t *testing.T) {
|
||||
|
||||
r := rand.New(rand.NewSource(1))
|
||||
const rowsN = int(1e4)
|
||||
for range rowsN {
|
||||
for i := 0; i < rowsN; i++ {
|
||||
s := prompb.TimeSeries{
|
||||
Samples: []prompb.Sample{{
|
||||
Value: r.Float64(),
|
||||
@@ -102,7 +102,7 @@ func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
|
||||
}
|
||||
|
||||
// push time series to the client.
|
||||
for range pushCnt {
|
||||
for i := 0; i < pushCnt; i++ {
|
||||
if err = rwClient.Push(prompb.TimeSeries{}); err != nil {
|
||||
t.Fatalf("cannot time series to the client: %s", err)
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestDebugClient_Push(t *testing.T) {
|
||||
|
||||
const rowsN = 100
|
||||
var sent int
|
||||
for i := range rowsN {
|
||||
for i := 0; i < rowsN; i++ {
|
||||
s := prompb.TimeSeries{
|
||||
Samples: []prompb.Sample{{
|
||||
Value: float64(i),
|
||||
|
||||
@@ -818,9 +818,7 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
|
||||
expr := fmt.Sprintf("default_rollup(%s{%s%s}[%ds])",
|
||||
alertForStateMetricName, nameStr, labelsFilter, int(lookback.Seconds()))
|
||||
|
||||
// query ALERTS_FOR_STATE at `ts-1s` instead `ts` to avoid retrieving data written in the current run,
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10335
|
||||
res, _, err := q.Query(ctx, expr, ts.Add(-1*time.Second))
|
||||
res, _, err := q.Query(ctx, expr, ts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute restore query %q: %w ", expr, err)
|
||||
}
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
//go:build synctest
|
||||
|
||||
package rule
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
|
||||
)
|
||||
|
||||
// TestAlertingRule_ActiveAtPreservedInAnnotations ensures that the fix for
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9543 is preserved
|
||||
// while allowing query templates in labels (https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9783)
|
||||
func TestAlertingRule_ActiveAtPreservedInAnnotations(t *testing.T) {
|
||||
// wrap into synctest because of time manipulations
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
fq := &datasource.FakeQuerier{}
|
||||
|
||||
ar := &AlertingRule{
|
||||
Name: "TestActiveAtPreservation",
|
||||
Labels: map[string]string{
|
||||
"test_query_in_label": `{{ "static_value" }}`,
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"description": "Alert active since {{ $activeAt }}",
|
||||
},
|
||||
alerts: make(map[uint64]*notifier.Alert),
|
||||
q: fq,
|
||||
state: &ruleState{
|
||||
entries: make([]StateEntry, 10),
|
||||
},
|
||||
}
|
||||
|
||||
// Mock query result - return empty result to make suppress_for_mass_alert = false
|
||||
// (no need to add anything to fq for empty result)
|
||||
|
||||
// Add a metric that should trigger the alert
|
||||
fq.Add(metricWithValueAndLabels(t, 1, "instance", "server1"))
|
||||
|
||||
// First execution - creates new alert
|
||||
ts1 := time.Now()
|
||||
_, err := ar.exec(context.TODO(), ts1, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on first exec: %s", err)
|
||||
}
|
||||
|
||||
if len(ar.alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
|
||||
}
|
||||
|
||||
firstAlert := ar.GetAlerts()[0]
|
||||
// Verify first execution: activeAt should be ts1 and annotation should reflect it
|
||||
if !firstAlert.ActiveAt.Equal(ts1) {
|
||||
t.Fatalf("expected activeAt to be %v, got %v", ts1, firstAlert.ActiveAt)
|
||||
}
|
||||
|
||||
// Extract time from annotation (format will be like "Alert active since 2025-09-30 08:55:13.638551611 -0400 EDT m=+0.002928464")
|
||||
expectedTimeStr := ts1.Format("2006-01-02 15:04:05")
|
||||
if !strings.Contains(firstAlert.Annotations["description"], expectedTimeStr) {
|
||||
t.Fatalf("first exec annotation should contain time %s, got: %s", expectedTimeStr, firstAlert.Annotations["description"])
|
||||
}
|
||||
|
||||
// Second execution - should preserve activeAt in annotation
|
||||
|
||||
// Ensure different timestamp with different seconds
|
||||
// sleep is non-blocking thanks to synctest
|
||||
time.Sleep(2 * time.Second)
|
||||
ts2 := time.Now()
|
||||
_, err = ar.exec(context.TODO(), ts2, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on second exec: %s", err)
|
||||
}
|
||||
|
||||
// Get the alert again (should be the same alert)
|
||||
if len(ar.alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
|
||||
}
|
||||
secondAlert := ar.GetAlerts()[0]
|
||||
|
||||
// Critical test: activeAt should still be ts1, not ts2
|
||||
if !secondAlert.ActiveAt.Equal(ts1) {
|
||||
t.Fatalf("activeAt should be preserved as %v, but got %v", ts1, secondAlert.ActiveAt)
|
||||
}
|
||||
|
||||
// Critical test: annotation should still contain ts1 time, not ts2
|
||||
if !strings.Contains(secondAlert.Annotations["description"], expectedTimeStr) {
|
||||
t.Fatalf("second exec annotation should still contain original time %s, got: %s", expectedTimeStr, secondAlert.Annotations["description"])
|
||||
}
|
||||
|
||||
// Additional verification: annotation should NOT contain ts2 time
|
||||
ts2TimeStr := ts2.Format("2006-01-02 15:04:05")
|
||||
if strings.Contains(secondAlert.Annotations["description"], ts2TimeStr) {
|
||||
t.Fatalf("annotation should NOT contain new eval time %s, got: %s", ts2TimeStr, secondAlert.Annotations["description"])
|
||||
}
|
||||
|
||||
// Verify query template in labels still works (this would fail if query templates were broken)
|
||||
if firstAlert.Labels["test_query_in_label"] != "static_value" {
|
||||
t.Fatalf("expected test_query_in_label=static_value, got %s", firstAlert.Labels["test_query_in_label"])
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
@@ -1478,3 +1479,95 @@ func TestAlertingRule_QueryTemplateInLabels(t *testing.T) {
|
||||
t.Fatalf("expected 'suppress_for_mass_alert' label to be 'true' or 'false', got '%s'", suppressLabel)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAlertingRule_ActiveAtPreservedInAnnotations ensures that the fix for
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9543 is preserved
|
||||
// while allowing query templates in labels (https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9783)
|
||||
func TestAlertingRule_ActiveAtPreservedInAnnotations(t *testing.T) {
|
||||
// wrap into synctest because of time manipulations
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
fq := &datasource.FakeQuerier{}
|
||||
|
||||
ar := &AlertingRule{
|
||||
Name: "TestActiveAtPreservation",
|
||||
Labels: map[string]string{
|
||||
"test_query_in_label": `{{ "static_value" }}`,
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"description": "Alert active since {{ $activeAt }}",
|
||||
},
|
||||
alerts: make(map[uint64]*notifier.Alert),
|
||||
q: fq,
|
||||
state: &ruleState{
|
||||
entries: make([]StateEntry, 10),
|
||||
},
|
||||
}
|
||||
|
||||
// Mock query result - return empty result to make suppress_for_mass_alert = false
|
||||
// (no need to add anything to fq for empty result)
|
||||
|
||||
// Add a metric that should trigger the alert
|
||||
fq.Add(metricWithValueAndLabels(t, 1, "instance", "server1"))
|
||||
|
||||
// First execution - creates new alert
|
||||
ts1 := time.Now()
|
||||
_, err := ar.exec(context.TODO(), ts1, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on first exec: %s", err)
|
||||
}
|
||||
|
||||
if len(ar.alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
|
||||
}
|
||||
|
||||
firstAlert := ar.GetAlerts()[0]
|
||||
// Verify first execution: activeAt should be ts1 and annotation should reflect it
|
||||
if !firstAlert.ActiveAt.Equal(ts1) {
|
||||
t.Fatalf("expected activeAt to be %v, got %v", ts1, firstAlert.ActiveAt)
|
||||
}
|
||||
|
||||
// Extract time from annotation (format will be like "Alert active since 2025-09-30 08:55:13.638551611 -0400 EDT m=+0.002928464")
|
||||
expectedTimeStr := ts1.Format("2006-01-02 15:04:05")
|
||||
if !strings.Contains(firstAlert.Annotations["description"], expectedTimeStr) {
|
||||
t.Fatalf("first exec annotation should contain time %s, got: %s", expectedTimeStr, firstAlert.Annotations["description"])
|
||||
}
|
||||
|
||||
// Second execution - should preserve activeAt in annotation
|
||||
|
||||
// Ensure different timestamp with different seconds
|
||||
// sleep is non-blocking thanks to synctest
|
||||
time.Sleep(2 * time.Second)
|
||||
ts2 := time.Now()
|
||||
_, err = ar.exec(context.TODO(), ts2, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on second exec: %s", err)
|
||||
}
|
||||
|
||||
// Get the alert again (should be the same alert)
|
||||
if len(ar.alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
|
||||
}
|
||||
secondAlert := ar.GetAlerts()[0]
|
||||
|
||||
// Critical test: activeAt should still be ts1, not ts2
|
||||
if !secondAlert.ActiveAt.Equal(ts1) {
|
||||
t.Fatalf("activeAt should be preserved as %v, but got %v", ts1, secondAlert.ActiveAt)
|
||||
}
|
||||
|
||||
// Critical test: annotation should still contain ts1 time, not ts2
|
||||
if !strings.Contains(secondAlert.Annotations["description"], expectedTimeStr) {
|
||||
t.Fatalf("second exec annotation should still contain original time %s, got: %s", expectedTimeStr, secondAlert.Annotations["description"])
|
||||
}
|
||||
|
||||
// Additional verification: annotation should NOT contain ts2 time
|
||||
ts2TimeStr := ts2.Format("2006-01-02 15:04:05")
|
||||
if strings.Contains(secondAlert.Annotations["description"], ts2TimeStr) {
|
||||
t.Fatalf("annotation should NOT contain new eval time %s, got: %s", ts2TimeStr, secondAlert.Annotations["description"])
|
||||
}
|
||||
|
||||
// Verify query template in labels still works (this would fail if query templates were broken)
|
||||
if firstAlert.Labels["test_query_in_label"] != "static_value" {
|
||||
t.Fatalf("expected test_query_in_label=static_value, got %s", firstAlert.Labels["test_query_in_label"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"maps"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -19,7 +18,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
)
|
||||
@@ -31,8 +29,8 @@ var (
|
||||
"0 means no limit.")
|
||||
ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
|
||||
"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param.")
|
||||
resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier.")
|
||||
maxResolveDuration = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
|
||||
resendDelay = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier.")
|
||||
maxResolveDuration = flag.Duration("rule.maxResolveDuration", 0, "Limits the maxiMum duration for automatic alert expiration, "+
|
||||
"which by default is 4 times evaluationInterval of the parent group")
|
||||
evalDelay = flag.Duration("rule.evalDelay", 30*time.Second, "Adjustment of the 'time' parameter for rule evaluation requests to compensate intentional data delay from the datasource. "+
|
||||
"Normally, should be equal to '-search.latencyOffset' (cmd-line flag configured for VictoriaMetrics single-node or vmselect). "+
|
||||
@@ -98,7 +96,9 @@ type groupMetrics struct {
|
||||
// set2 has priority over set1.
|
||||
func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
|
||||
r := map[string]string{}
|
||||
maps.Copy(r, set1)
|
||||
for k, v := range set1 {
|
||||
r[k] = v
|
||||
}
|
||||
for k, v := range set2 {
|
||||
if prevV, ok := r[k]; ok {
|
||||
logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
|
||||
@@ -374,7 +374,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
|
||||
|
||||
g.infof("started")
|
||||
|
||||
eval := func(ctx context.Context, ts time.Time) time.Time {
|
||||
eval := func(ctx context.Context, ts time.Time) {
|
||||
g.metrics.iterationTotal.Inc()
|
||||
|
||||
start := time.Now()
|
||||
@@ -382,7 +382,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
|
||||
if len(g.Rules) < 1 {
|
||||
g.metrics.iterationDuration.UpdateDuration(start)
|
||||
g.LastEvaluation = start
|
||||
return ts
|
||||
return
|
||||
}
|
||||
|
||||
resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
|
||||
@@ -396,7 +396,6 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
|
||||
}
|
||||
g.metrics.iterationDuration.UpdateDuration(start)
|
||||
g.LastEvaluation = start
|
||||
return ts
|
||||
}
|
||||
|
||||
evalCtx, cancel := context.WithCancel(ctx)
|
||||
@@ -405,7 +404,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
|
||||
g.mu.Unlock()
|
||||
defer g.evalCancel()
|
||||
|
||||
realEvalTS := eval(evalCtx, evalTS)
|
||||
eval(evalCtx, evalTS)
|
||||
|
||||
t := time.NewTicker(g.Interval)
|
||||
defer t.Stop()
|
||||
@@ -413,7 +412,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
|
||||
// restore the rules state after the first evaluation
|
||||
// so only active alerts can be restored.
|
||||
if rr != nil {
|
||||
err := g.restore(ctx, rr, realEvalTS, *remoteReadLookBack)
|
||||
err := g.restore(ctx, rr, evalTS, *remoteReadLookBack)
|
||||
if err != nil {
|
||||
logger.Errorf("error while restoring ruleState for group %q: %s", g.Name, err)
|
||||
}
|
||||
@@ -494,8 +493,11 @@ func (g *Group) delayBeforeStart(ts time.Time, maxDelay time.Duration) time.Dura
|
||||
}
|
||||
|
||||
// otherwise, return a random duration between [0..min(interval, maxDelay)] based on group ID
|
||||
// artificially limit interval, so groups with big intervals could start sooner.
|
||||
interval := min(g.Interval, maxDelay)
|
||||
interval := g.Interval
|
||||
if interval > maxDelay {
|
||||
// artificially limit interval, so groups with big intervals could start sooner.
|
||||
interval = maxDelay
|
||||
}
|
||||
var randSleep time.Duration
|
||||
randSleep = time.Duration(float64(interval) * (float64(g.GetID()) / (1 << 64)))
|
||||
sleepOffset := time.Duration(ts.UnixNano() % interval.Nanoseconds())
|
||||
@@ -753,7 +755,6 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
|
||||
return fmt.Errorf("rule %q: failed to execute: %w", r, err)
|
||||
}
|
||||
|
||||
var errG vmalertutil.ErrGroup
|
||||
if e.Rw != nil {
|
||||
pushToRW := func(tss []prompb.TimeSeries) error {
|
||||
var lastErr error
|
||||
@@ -765,26 +766,20 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
|
||||
return lastErr
|
||||
}
|
||||
if err := pushToRW(tss); err != nil {
|
||||
errG.Add(err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ar, ok := r.(*AlertingRule)
|
||||
if !ok {
|
||||
return errG.Err()
|
||||
return nil
|
||||
}
|
||||
|
||||
alerts := ar.alertsToSend(resolveDuration, *resendDelay)
|
||||
if len(alerts) < 1 {
|
||||
return errG.Err()
|
||||
return nil
|
||||
}
|
||||
|
||||
notifierErr := notifier.Send(ctx, alerts, e.notifierHeaders)
|
||||
for err := range notifierErr {
|
||||
if err != nil {
|
||||
errG.Add(fmt.Errorf("rule %q: notifier failure: %w", r, err))
|
||||
}
|
||||
}
|
||||
|
||||
return errG.Err()
|
||||
errGr := notifier.Send(ctx, alerts, e.notifierHeaders)
|
||||
return errGr.Err()
|
||||
}
|
||||
|
||||
@@ -405,8 +405,7 @@ func TestGroupStart(t *testing.T) {
|
||||
|
||||
var cur uint64
|
||||
prev := g.metrics.iterationTotal.Get()
|
||||
i := 0
|
||||
for {
|
||||
for i := 0; ; i++ {
|
||||
if i > 40 {
|
||||
t.Fatalf("group wasn't able to perform %d evaluations during %d eval intervals", n, i)
|
||||
}
|
||||
@@ -415,7 +414,6 @@ func TestGroupStart(t *testing.T) {
|
||||
return
|
||||
}
|
||||
time.Sleep(interval)
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ func (s *ruleState) add(e StateEntry) {
|
||||
func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int) (int, error) {
|
||||
var err error
|
||||
var tss []prompb.TimeSeries
|
||||
for i := range replayRuleRetryAttempts {
|
||||
for i := 0; i < replayRuleRetryAttempts; i++ {
|
||||
tss, err = r.execRange(context.Background(), start, end)
|
||||
if err == nil {
|
||||
break
|
||||
|
||||
@@ -40,7 +40,7 @@ func TestRule_state(t *testing.T) {
|
||||
}
|
||||
|
||||
var last time.Time
|
||||
for range stateEntriesN * 2 {
|
||||
for i := 0; i < stateEntriesN*2; i++ {
|
||||
last = time.Now()
|
||||
r.state.add(StateEntry{At: last})
|
||||
}
|
||||
@@ -68,7 +68,7 @@ func TestRule_stateConcurrent(_ *testing.T) {
|
||||
var wg sync.WaitGroup
|
||||
for range workers {
|
||||
wg.Go(func() {
|
||||
for range iterations {
|
||||
for i := 0; i < iterations; i++ {
|
||||
r.state.add(StateEntry{At: time.Now()})
|
||||
r.state.getAll()
|
||||
r.state.getLast()
|
||||
|
||||
@@ -19,13 +19,13 @@ func CompareRules(t *testing.T, a, b Rule) error {
|
||||
case *AlertingRule:
|
||||
br, ok := b.(*AlertingRule)
|
||||
if !ok {
|
||||
return fmt.Errorf("rule %d supposed to be of type AlertingRule", b.ID())
|
||||
return fmt.Errorf("rule %q supposed to be of type AlertingRule", b.ID())
|
||||
}
|
||||
return compareAlertingRules(t, v, br)
|
||||
case *RecordingRule:
|
||||
br, ok := b.(*RecordingRule)
|
||||
if !ok {
|
||||
return fmt.Errorf("rule %d supposed to be of type RecordingRule", b.ID())
|
||||
return fmt.Errorf("rule %q supposed to be of type RecordingRule", b.ID())
|
||||
}
|
||||
return compareRecordingRules(t, v, br)
|
||||
default:
|
||||
|
||||
@@ -45,7 +45,7 @@ func (eg *ErrGroup) Error() string {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "errors(%d): \n", len(eg.errs))
|
||||
fmt.Fprintf(&b, "errors(%d): ", len(eg.errs))
|
||||
for i, err := range eg.errs {
|
||||
b.WriteString(err.Error())
|
||||
if i != len(eg.errs)-1 {
|
||||
|
||||
@@ -30,8 +30,8 @@ func TestErrGroup(t *testing.T) {
|
||||
}
|
||||
|
||||
f(nil, "")
|
||||
f([]error{errors.New("timeout")}, "errors(1): \ntimeout")
|
||||
f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): \ntimeout\ndeadline")
|
||||
f([]error{errors.New("timeout")}, "errors(1): timeout")
|
||||
f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): timeout\ndeadline")
|
||||
}
|
||||
|
||||
// TestErrGroupConcurrent supposed to test concurrent
|
||||
@@ -42,7 +42,7 @@ func TestErrGroupConcurrent(_ *testing.T) {
|
||||
|
||||
const writersN = 4
|
||||
payload := make(chan error, writersN)
|
||||
for range writersN {
|
||||
for i := 0; i < writersN; i++ {
|
||||
go func() {
|
||||
for err := range payload {
|
||||
eg.Add(err)
|
||||
@@ -51,7 +51,7 @@ func TestErrGroupConcurrent(_ *testing.T) {
|
||||
}
|
||||
|
||||
const iterations = 500
|
||||
for i := range iterations {
|
||||
for i := 0; i < iterations; i++ {
|
||||
payload <- fmt.Errorf("error %d", i)
|
||||
if i%10 == 0 {
|
||||
_ = eg.Err()
|
||||
|
||||
@@ -65,11 +65,10 @@ type AuthConfig struct {
|
||||
type UserInfo struct {
|
||||
Name string `yaml:"name,omitempty"`
|
||||
|
||||
BearerToken string `yaml:"bearer_token,omitempty"`
|
||||
JWT *JWTConfig `yaml:"jwt,omitempty"`
|
||||
AuthToken string `yaml:"auth_token,omitempty"`
|
||||
Username string `yaml:"username,omitempty"`
|
||||
Password string `yaml:"password,omitempty"`
|
||||
BearerToken string `yaml:"bearer_token,omitempty"`
|
||||
AuthToken string `yaml:"auth_token,omitempty"`
|
||||
Username string `yaml:"username,omitempty"`
|
||||
Password string `yaml:"password,omitempty"`
|
||||
|
||||
URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
|
||||
DiscoverBackendIPs *bool `yaml:"discover_backend_ips,omitempty"`
|
||||
@@ -589,7 +588,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
|
||||
|
||||
// Slow path - select other backend urls.
|
||||
n := atomicCounter.Add(1) - 1
|
||||
for i := range uint32(len(bus)) {
|
||||
for i := uint32(0); i < uint32(len(bus)); i++ {
|
||||
idx := (n + i) % uint32(len(bus))
|
||||
bu := bus[idx]
|
||||
if bu.isBroken() {
|
||||
@@ -800,9 +799,6 @@ var (
|
||||
// authUsers contains the currently loaded auth users
|
||||
authUsers atomic.Pointer[map[string]*UserInfo]
|
||||
|
||||
// jwt authentication cache
|
||||
jwtAuthCache atomic.Pointer[jwtCache]
|
||||
|
||||
authConfigWG sync.WaitGroup
|
||||
stopCh chan struct{}
|
||||
)
|
||||
@@ -842,14 +838,6 @@ func reloadAuthConfigData(data []byte) (bool, error) {
|
||||
return false, fmt.Errorf("failed to parse auth config: %w", err)
|
||||
}
|
||||
|
||||
jui, err := parseJWTUsers(ac)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse JWT users from auth config: %w", err)
|
||||
}
|
||||
jwtc := &jwtCache{
|
||||
users: jui,
|
||||
}
|
||||
|
||||
m, err := parseAuthConfigUsers(ac)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse users from auth config: %w", err)
|
||||
@@ -869,7 +857,6 @@ func reloadAuthConfigData(data []byte) (bool, error) {
|
||||
authConfig.Store(ac)
|
||||
authConfigData.Store(&data)
|
||||
authUsers.Store(&m)
|
||||
jwtAuthCache.Store(jwtc)
|
||||
|
||||
return true, nil
|
||||
}
|
||||
@@ -894,9 +881,6 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
|
||||
if ui.BearerToken != "" {
|
||||
return nil, fmt.Errorf("field bearer_token can't be specified for unauthorized_user section")
|
||||
}
|
||||
if ui.JWT != nil {
|
||||
return nil, fmt.Errorf("field jwt can't be specified for unauthorized_user section")
|
||||
}
|
||||
if ui.AuthToken != "" {
|
||||
return nil, fmt.Errorf("field auth_token can't be specified for unauthorized_user section")
|
||||
}
|
||||
@@ -943,17 +927,10 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
|
||||
}
|
||||
for i := range uis {
|
||||
ui := &uis[i]
|
||||
// users with jwt tokens are parsed by parseJWTUsers function.
|
||||
// the function also checks that users with jwt tokens do not have auth tokens, bearer tokens, usernames and passwords.
|
||||
if ui.JWT != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
ats, err := getAuthTokens(ui.AuthToken, ui.BearerToken, ui.Username, ui.Password)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, at := range ats {
|
||||
if uiOld := byAuthToken[at]; uiOld != nil {
|
||||
return nil, fmt.Errorf("duplicate auth token=%q found for username=%q, name=%q; the previous one is set for username=%q, name=%q",
|
||||
|
||||
@@ -378,7 +378,7 @@ users:
|
||||
RetryStatusCodes: []int{500, 501},
|
||||
LoadBalancingPolicy: "first_available",
|
||||
MergeQueryArgs: []string{"foo", "bar"},
|
||||
DropSrcPathPrefixParts: new(1),
|
||||
DropSrcPathPrefixParts: intp(1),
|
||||
DiscoverBackendIPs: &discoverBackendIPsTrue,
|
||||
},
|
||||
}, nil)
|
||||
@@ -621,22 +621,6 @@ unauthorized_user:
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// skip user info with jwt, it is parsed by parseJWTUsers
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
url_prefix: http://aaa:343/bbb
|
||||
- jwt: {skip_verify: true}
|
||||
url_prefix: http://aaa:343/bbb
|
||||
`, map[string]*UserInfo{
|
||||
getHTTPAuthBasicToken("foo", "bar"): {
|
||||
Username: "foo",
|
||||
Password: "bar",
|
||||
URLPrefix: mustParseURL("http://aaa:343/bbb"),
|
||||
},
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
|
||||
@@ -847,7 +831,7 @@ func TestBrokenBackend(t *testing.T) {
|
||||
bus[1].setBroken()
|
||||
|
||||
// broken backend should never return while there are healthy backends
|
||||
for range int(1e3) {
|
||||
for i := 0; i < 1e3; i++ {
|
||||
b := up.getBackendURL()
|
||||
if b.isBroken() {
|
||||
t.Fatalf("unexpected broken backend %q", b.url)
|
||||
@@ -979,6 +963,10 @@ func mustParseURLs(us []string) *URLPrefix {
|
||||
return up
|
||||
}
|
||||
|
||||
func intp(n int) *int {
|
||||
return &n
|
||||
}
|
||||
|
||||
func mustNewRegex(s string) *Regex {
|
||||
var re Regex
|
||||
if err := yaml.Unmarshal([]byte(s), &re); err != nil {
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/jwt"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
)
|
||||
|
||||
type jwtCache struct {
|
||||
// users contain UserInfo`s from AuthConfig with JWTConfig set
|
||||
users []*UserInfo
|
||||
}
|
||||
|
||||
type JWTConfig struct {
|
||||
PublicKeys []string `yaml:"public_keys,omitempty"`
|
||||
PublicKeyFiles []string `yaml:"public_key_files,omitempty"`
|
||||
SkipVerify bool `yaml:"skip_verify,omitempty"`
|
||||
|
||||
verifierPool *jwt.VerifierPool
|
||||
}
|
||||
|
||||
func parseJWTUsers(ac *AuthConfig) ([]*UserInfo, error) {
|
||||
jui := make([]*UserInfo, 0, len(ac.Users))
|
||||
for _, ui := range ac.Users {
|
||||
jwtToken := ui.JWT
|
||||
if jwtToken == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if ui.AuthToken != "" || ui.BearerToken != "" || ui.Username != "" || ui.Password != "" {
|
||||
return nil, fmt.Errorf("auth_token, bearer_token, username and password cannot be specified if jwt is set")
|
||||
}
|
||||
if len(jwtToken.PublicKeys) == 0 && len(jwtToken.PublicKeyFiles) == 0 && !jwtToken.SkipVerify {
|
||||
return nil, fmt.Errorf("jwt must contain at least a single public key, public_key_files or have skip_verify=true")
|
||||
}
|
||||
|
||||
if len(jwtToken.PublicKeys) > 0 || len(jwtToken.PublicKeyFiles) > 0 {
|
||||
keys := make([]any, 0, len(jwtToken.PublicKeys)+len(jwtToken.PublicKeyFiles))
|
||||
|
||||
for i := range jwtToken.PublicKeys {
|
||||
k, err := jwt.ParseKey([]byte(jwtToken.PublicKeys[i]))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
for _, filePath := range jwtToken.PublicKeyFiles {
|
||||
keyData, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read public key from file %q: %w", filePath, err)
|
||||
}
|
||||
k, err := jwt.ParseKey(keyData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse public key from file %q: %w", filePath, err)
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
vp, err := jwt.NewVerifierPool(keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
jwtToken.verifierPool = vp
|
||||
}
|
||||
|
||||
if err := ui.initURLs(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
metricLabels, err := ui.getMetricLabels()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
|
||||
}
|
||||
ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
|
||||
ui.requestErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_errors_total` + metricLabels)
|
||||
ui.backendRequests = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_requests_total` + metricLabels)
|
||||
ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
|
||||
ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
|
||||
mcr := ui.getMaxConcurrentRequests()
|
||||
ui.concurrencyLimitCh = make(chan struct{}, mcr)
|
||||
ui.concurrencyLimitReached = ac.ms.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
|
||||
_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
|
||||
return float64(cap(ui.concurrencyLimitCh))
|
||||
})
|
||||
_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
|
||||
return float64(len(ui.concurrencyLimitCh))
|
||||
})
|
||||
|
||||
rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
|
||||
}
|
||||
ui.rt = rt
|
||||
|
||||
jui = append(jui, &ui)
|
||||
}
|
||||
|
||||
// the limitation will be lifted once claim based matching will be implemented
|
||||
if len(jui) > 1 {
|
||||
return nil, fmt.Errorf("multiple users with JWT tokens are not supported; found %d users", len(jui))
|
||||
}
|
||||
|
||||
return jui, nil
|
||||
}
|
||||
|
||||
func getUserInfoByJWTToken(ats []string) *UserInfo {
|
||||
js := *jwtAuthCache.Load()
|
||||
if len(js.users) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, at := range ats {
|
||||
if strings.Count(at, ".") != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
at, _ = strings.CutPrefix(at, `http_auth:`)
|
||||
|
||||
tkn, err := jwt.NewToken(at, true)
|
||||
if err != nil {
|
||||
if *logInvalidAuthTokens {
|
||||
logger.Infof("cannot parse jwt token: %s", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if tkn.IsExpired(time.Now()) {
|
||||
if *logInvalidAuthTokens {
|
||||
logger.Infof("jwt token is expired")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
for _, ui := range js.users {
|
||||
if ui.JWT.SkipVerify {
|
||||
return ui
|
||||
}
|
||||
|
||||
if err := ui.JWT.verifierPool.Verify(tkn); err != nil {
|
||||
if *logInvalidAuthTokens {
|
||||
logger.Infof("cannot verify jwt token: %s", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
return ui
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,304 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestJWTParseAuthConfigFailure(t *testing.T) {
|
||||
validRSAPublicKey := `-----BEGIN PUBLIC KEY-----
|
||||
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAiX7oPWKOWRQsGFEWvwZO
|
||||
mL2PYsdYUsu9nr0qtPCjxQHUJgLfT3rdKlvKpPFYv7ZmKnqTncg36Wz9uiYmWJ7e
|
||||
IB5Z+fko8kVIMzarCqVvpAJDzYF/pUii68xvuYoK3L9TIOAeyCXv+prwnr2IH+Mw
|
||||
9AONzWbRrYoO74XyTE9vMU5qmI/L1VPk+PR8lqPOSptLvzsfoaIk2ED4yK2nRB+6
|
||||
st+k4nccPqbErqHc8aiXnXfugfnr6b+NPFYUzKsDqkymGOokVijrI8B3jNw6c6Do
|
||||
zphk+D3wgLsXYHfMcZbXIMqffqm/aB8Qg88OpFOkQ3rd2p6R9+hacnZkfkn3Phiw
|
||||
yQIDAQAB
|
||||
-----END PUBLIC KEY-----
|
||||
`
|
||||
// ECDSA with the P-521 curve
|
||||
validECDSAPublicKey := `-----BEGIN PUBLIC KEY-----
|
||||
MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQAU9RmtkCRuYTKCyvLlDn5DtBZOHSe
|
||||
QTa5j9q/oQVpCKqcXVFrH5dgh0GL+P/ZhkeuowPzCZqntGf0+7wPt9OxSJcADVJm
|
||||
dv92m540MXss8zdHf5qtE0gsu2Ved0R7Z8a8QwGZ/1mYZ+kFGGbdQTlSvRqDySTq
|
||||
XOtclIk1uhc03oL9nOQ=
|
||||
-----END PUBLIC KEY-----
|
||||
`
|
||||
|
||||
f := func(s string, expErr string) {
|
||||
t.Helper()
|
||||
ac, err := parseAuthConfig([]byte(s))
|
||||
if err != nil {
|
||||
if expErr != err.Error() {
|
||||
t.Fatalf("unexpected error; got %q; want %q", err.Error(), expErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
users, err := parseJWTUsers(ac)
|
||||
if err != nil {
|
||||
if expErr != err.Error() {
|
||||
t.Fatalf("unexpected error; got %q; want %q", err.Error(), expErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("expecting non-nil error; got %v", users)
|
||||
}
|
||||
|
||||
// unauthorized_user cannot be used with jwt
|
||||
f(`
|
||||
unauthorized_user:
|
||||
jwt: {skip_verify: true}
|
||||
url_prefix: http://foo.bar
|
||||
`, `field jwt can't be specified for unauthorized_user section`)
|
||||
|
||||
// username and jwt in a single config
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
jwt: {skip_verify: true}
|
||||
url_prefix: http://foo.bar
|
||||
`, `auth_token, bearer_token, username and password cannot be specified if jwt is set`)
|
||||
// bearer_token and jwt in a single config
|
||||
f(`
|
||||
users:
|
||||
- bearer_token: foo
|
||||
jwt: {skip_verify: true}
|
||||
url_prefix: http://foo.bar
|
||||
`, `auth_token, bearer_token, username and password cannot be specified if jwt is set`)
|
||||
// bearer_token and jwt in a single config
|
||||
f(`
|
||||
users:
|
||||
- auth_token: "Foo token"
|
||||
jwt: {skip_verify: true}
|
||||
url_prefix: http://foo.bar
|
||||
`, `auth_token, bearer_token, username and password cannot be specified if jwt is set`)
|
||||
|
||||
// jwt public_keys or skip_verify must be set, part 1
|
||||
f(`
|
||||
users:
|
||||
- jwt: {}
|
||||
url_prefix: http://foo.bar
|
||||
`, `jwt must contain at least a single public key, public_key_files or have skip_verify=true`)
|
||||
|
||||
// jwt public_keys or skip_verify must be set, part 2
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_keys: null}
|
||||
url_prefix: http://foo.bar
|
||||
`, `jwt must contain at least a single public key, public_key_files or have skip_verify=true`)
|
||||
|
||||
// jwt public_keys or skip_verify must be set, part 3
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_keys: []}
|
||||
url_prefix: http://foo.bar
|
||||
`, `jwt must contain at least a single public key, public_key_files or have skip_verify=true`)
|
||||
|
||||
// jwt public_keys, public_key_files or skip_verify must be set
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_key_files: []}
|
||||
url_prefix: http://foo.bar
|
||||
`, `jwt must contain at least a single public key, public_key_files or have skip_verify=true`)
|
||||
|
||||
// invalid public key, part 1
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_keys: [""]}
|
||||
url_prefix: http://foo.bar
|
||||
`, `failed to parse key "": failed to decode PEM block containing public key`)
|
||||
|
||||
// invalid public key, part 2
|
||||
f(`
|
||||
users:
|
||||
- jwt: {public_keys: ["invalid"]}
|
||||
url_prefix: http://foo.bar
|
||||
`, `failed to parse key "invalid": failed to decode PEM block containing public key`)
|
||||
|
||||
// invalid public key, part 2
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
- %q
|
||||
- "invalid"
|
||||
url_prefix: http://foo.bar
|
||||
`, validRSAPublicKey, validECDSAPublicKey), `failed to parse key "invalid": failed to decode PEM block containing public key`)
|
||||
|
||||
// several jwt users
|
||||
// invalid public key, part 2
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validRSAPublicKey, validECDSAPublicKey), `multiple users with JWT tokens are not supported; found 2 users`)
|
||||
|
||||
// public key file doesn't exist
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
public_key_files:
|
||||
- /path/to/nonexistent/file.pem
|
||||
url_prefix: http://foo.bar
|
||||
`, "cannot read public key from file \"/path/to/nonexistent/file.pem\": open /path/to/nonexistent/file.pem: no such file or directory")
|
||||
|
||||
// public key file invalid
|
||||
// auth with key from file
|
||||
publicKeyFile := filepath.Join(t.TempDir(), "a_public_key.pem")
|
||||
if err := os.WriteFile(publicKeyFile, []byte(`invalidPEM`), 0o644); err != nil {
|
||||
t.Fatalf("failed to write public key file: %s", err)
|
||||
}
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
public_key_files:
|
||||
- `+publicKeyFile+`
|
||||
url_prefix: http://foo.bar
|
||||
`, "cannot parse public key from file \""+publicKeyFile+"\": failed to parse key \"invalidPEM\": failed to decode PEM block containing public key")
|
||||
}
|
||||
|
||||
func TestJWTParseAuthConfigSuccess(t *testing.T) {
|
||||
validRSAPublicKey := `-----BEGIN PUBLIC KEY-----
|
||||
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAiX7oPWKOWRQsGFEWvwZO
|
||||
mL2PYsdYUsu9nr0qtPCjxQHUJgLfT3rdKlvKpPFYv7ZmKnqTncg36Wz9uiYmWJ7e
|
||||
IB5Z+fko8kVIMzarCqVvpAJDzYF/pUii68xvuYoK3L9TIOAeyCXv+prwnr2IH+Mw
|
||||
9AONzWbRrYoO74XyTE9vMU5qmI/L1VPk+PR8lqPOSptLvzsfoaIk2ED4yK2nRB+6
|
||||
st+k4nccPqbErqHc8aiXnXfugfnr6b+NPFYUzKsDqkymGOokVijrI8B3jNw6c6Do
|
||||
zphk+D3wgLsXYHfMcZbXIMqffqm/aB8Qg88OpFOkQ3rd2p6R9+hacnZkfkn3Phiw
|
||||
yQIDAQAB
|
||||
-----END PUBLIC KEY-----
|
||||
`
|
||||
// ECDSA with the P-521 curve
|
||||
validECDSAPublicKey := `-----BEGIN PUBLIC KEY-----
|
||||
MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQAU9RmtkCRuYTKCyvLlDn5DtBZOHSe
|
||||
QTa5j9q/oQVpCKqcXVFrH5dgh0GL+P/ZhkeuowPzCZqntGf0+7wPt9OxSJcADVJm
|
||||
dv92m540MXss8zdHf5qtE0gsu2Ved0R7Z8a8QwGZ/1mYZ+kFGGbdQTlSvRqDySTq
|
||||
XOtclIk1uhc03oL9nOQ=
|
||||
-----END PUBLIC KEY-----
|
||||
`
|
||||
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
ac, err := parseAuthConfig([]byte(s))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
jui, err := parseJWTUsers(ac)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
for _, ui := range jui {
|
||||
if ui.JWT == nil {
|
||||
t.Fatalf("unexpected nil JWTConfig")
|
||||
}
|
||||
|
||||
if ui.JWT.SkipVerify {
|
||||
if ui.JWT.verifierPool != nil {
|
||||
t.Fatalf("unexpected non-nil verifier pool for skip_verify=true")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if ui.JWT.verifierPool == nil {
|
||||
t.Fatalf("unexpected nil verifier pool for non-empty public keys")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validRSAPublicKey))
|
||||
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validECDSAPublicKey))
|
||||
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validRSAPublicKey, validECDSAPublicKey))
|
||||
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
url_prefix: http://foo.bar
|
||||
`)
|
||||
|
||||
// combined with other auth methods
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
url_prefix: http://foo.bar
|
||||
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
url_prefix: http://foo.bar
|
||||
|
||||
- bearer_token: foo
|
||||
url_prefix: http://foo.bar
|
||||
`)
|
||||
|
||||
rsaKeyFile := filepath.Join(t.TempDir(), "rsa_public_key.pem")
|
||||
if err := os.WriteFile(rsaKeyFile, []byte(validRSAPublicKey), 0o644); err != nil {
|
||||
t.Fatalf("failed to write RSA key file: %s", err)
|
||||
}
|
||||
ecdsaKeyFile := filepath.Join(t.TempDir(), "ecdsa_public_key.pem")
|
||||
if err := os.WriteFile(ecdsaKeyFile, []byte(validECDSAPublicKey), 0o644); err != nil {
|
||||
t.Fatalf("failed to write ECDSA key file: %s", err)
|
||||
}
|
||||
|
||||
// Test single public key file
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_key_files:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, rsaKeyFile))
|
||||
|
||||
// Test multiple public key files
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_key_files:
|
||||
- %q
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, rsaKeyFile, ecdsaKeyFile))
|
||||
|
||||
// Test combined inline keys and files
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
public_key_files:
|
||||
- %q
|
||||
url_prefix: http://foo.bar
|
||||
`, validECDSAPublicKey, rsaKeyFile))
|
||||
}
|
||||
@@ -181,32 +181,29 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
if ui := getUserInfoByAuthTokens(ats); ui != nil {
|
||||
processUserRequest(w, r, ui)
|
||||
return true
|
||||
}
|
||||
if ui := getUserInfoByJWTToken(ats); ui != nil {
|
||||
processUserRequest(w, r, ui)
|
||||
return true
|
||||
}
|
||||
|
||||
uu := authConfig.Load().UnauthorizedUser
|
||||
if uu != nil {
|
||||
processUserRequest(w, r, uu)
|
||||
return true
|
||||
}
|
||||
|
||||
invalidAuthTokenRequests.Inc()
|
||||
if *logInvalidAuthTokens {
|
||||
err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
Err: err,
|
||||
StatusCode: http.StatusUnauthorized,
|
||||
ui := getUserInfoByAuthTokens(ats)
|
||||
if ui == nil {
|
||||
uu := authConfig.Load().UnauthorizedUser
|
||||
if uu != nil {
|
||||
processUserRequest(w, r, uu)
|
||||
return true
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
} else {
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||
|
||||
invalidAuthTokenRequests.Inc()
|
||||
if *logInvalidAuthTokens {
|
||||
err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
Err: err,
|
||||
StatusCode: http.StatusUnauthorized,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
} else {
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
processUserRequest(w, r, ui)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -371,7 +368,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
}
|
||||
|
||||
maxAttempts := up.getBackendsCount()
|
||||
for range maxAttempts {
|
||||
for i := 0; i < maxAttempts; i++ {
|
||||
bu := up.getBackendURL()
|
||||
if bu == nil {
|
||||
break
|
||||
@@ -404,7 +401,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
ui.backendErrors.Inc()
|
||||
}
|
||||
err := &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), ui.name()),
|
||||
Err: fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
|
||||
StatusCode: http.StatusBadGateway,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
|
||||
@@ -3,20 +3,11 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -429,7 +420,7 @@ unauthorized_user:
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
|
||||
all the 2 backends for the user "" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// all the backend_urls are unavailable for authorized user
|
||||
@@ -447,7 +438,7 @@ users:
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
all the 2 backends for the user "some-user" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
|
||||
all the 2 backends for the user "some-user" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// zero discovered backend IPs
|
||||
@@ -469,7 +460,7 @@ unauthorized_user:
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
all the 0 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
|
||||
all the 0 backends for the user "" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
netutil.Resolver = origResolver
|
||||
|
||||
@@ -486,7 +477,7 @@ unauthorized_user:
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
|
||||
all the 2 backends for the user "" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
if n := retries.Load(); n != 2 {
|
||||
t.Fatalf("unexpected number of retries; got %d; want 2", n)
|
||||
@@ -515,218 +506,6 @@ requested_url={BACKEND}/path2/foo/?de=fg`
|
||||
}
|
||||
}
|
||||
|
||||
func TestJWTRequestHandler(t *testing.T) {
|
||||
// Generate RSA key pair for testing
|
||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot generate RSA key: %s", err)
|
||||
}
|
||||
|
||||
// Generate public key PEM
|
||||
publicKeyBytes, err := x509.MarshalPKIXPublicKey(&privateKey.PublicKey)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal public key: %s", err)
|
||||
}
|
||||
publicKeyPEM := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "PUBLIC KEY",
|
||||
Bytes: publicKeyBytes,
|
||||
})
|
||||
|
||||
genToken := func(t *testing.T, body map[string]any, valid bool) string {
|
||||
t.Helper()
|
||||
|
||||
headerJSON, err := json.Marshal(map[string]any{
|
||||
"alg": "RS256",
|
||||
"typ": "JWT",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal header: %s", err)
|
||||
}
|
||||
headerB64 := base64.RawURLEncoding.EncodeToString(headerJSON)
|
||||
|
||||
bodyJSON, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal body: %s", err)
|
||||
}
|
||||
bodyB64 := base64.RawURLEncoding.EncodeToString(bodyJSON)
|
||||
|
||||
payload := headerB64 + "." + bodyB64
|
||||
|
||||
var signatureB64 string
|
||||
if valid {
|
||||
// Create real RSA signature
|
||||
hash := crypto.SHA256
|
||||
h := hash.New()
|
||||
h.Write([]byte(payload))
|
||||
digest := h.Sum(nil)
|
||||
|
||||
signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, hash, digest)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot sign token: %s", err)
|
||||
}
|
||||
signatureB64 = base64.RawURLEncoding.EncodeToString(signature)
|
||||
} else {
|
||||
signatureB64 = base64.RawURLEncoding.EncodeToString([]byte("invalid_signature"))
|
||||
}
|
||||
|
||||
return payload + "." + signatureB64
|
||||
}
|
||||
genToken(t, nil, false)
|
||||
|
||||
f := func(cfgStr string, r *http.Request, responseExpected string) {
|
||||
t.Helper()
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := w.Write([]byte(r.RequestURI + "\n")); err != nil {
|
||||
panic(fmt.Errorf("cannot write response: %w", err))
|
||||
}
|
||||
if v := r.Header.Get(`extra_label`); v != "" {
|
||||
if _, err := w.Write([]byte(`extra_label=` + v + "\n")); err != nil {
|
||||
panic(fmt.Errorf("cannot write response: %w", err))
|
||||
}
|
||||
}
|
||||
if v := r.Header.Get(`extra_filters`); v != "" {
|
||||
if _, err := w.Write([]byte(`extra_filters=` + v + "\n")); err != nil {
|
||||
panic(fmt.Errorf("cannot write response: %w", err))
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
|
||||
responseExpected = strings.ReplaceAll(responseExpected, "{BACKEND}", ts.URL)
|
||||
|
||||
cfgOrigP := authConfigData.Load()
|
||||
if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
|
||||
t.Fatalf("cannot load config data: %s", err)
|
||||
}
|
||||
defer func() {
|
||||
cfgOrig := []byte("unauthorized_user:\n url_prefix: http://foo/bar")
|
||||
if cfgOrigP != nil {
|
||||
cfgOrig = *cfgOrigP
|
||||
}
|
||||
_, err := reloadAuthConfigData(cfgOrig)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot load the original config: %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
w := &fakeResponseWriter{}
|
||||
if !requestHandlerWithInternalRoutes(w, r) {
|
||||
t.Fatalf("unexpected false is returned from requestHandler")
|
||||
}
|
||||
|
||||
response := w.getResponse()
|
||||
response = strings.ReplaceAll(response, "\r\n", "\n")
|
||||
response = strings.TrimSpace(response)
|
||||
responseExpected = strings.TrimSpace(responseExpected)
|
||||
if response != responseExpected {
|
||||
t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, responseExpected)
|
||||
}
|
||||
}
|
||||
|
||||
simpleCfgStr := fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
url_prefix: {BACKEND}/foo`, string(publicKeyPEM))
|
||||
noVMAccessClaimToken := genToken(t, nil, true)
|
||||
defaultVMAccessClaimToken := genToken(t, map[string]any{
|
||||
"exp": time.Now().Add(10 * time.Minute).Unix(),
|
||||
"vm_access": map[string]any{},
|
||||
}, true)
|
||||
expiredToken := genToken(t, map[string]any{
|
||||
"exp": 10,
|
||||
"vm_access": map[string]any{},
|
||||
}, true)
|
||||
invalidSignatureToken := genToken(t, map[string]any{
|
||||
"exp": time.Now().Add(10 * time.Minute).Unix(),
|
||||
"vm_access": map[string]any{},
|
||||
}, false)
|
||||
|
||||
// missing authorization
|
||||
request := httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
responseExpected := `
|
||||
statusCode=401
|
||||
Www-Authenticate: Basic realm="Restricted"
|
||||
missing 'Authorization' request header`
|
||||
f(simpleCfgStr, request, responseExpected)
|
||||
|
||||
// token without vm_access claim
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Unauthorized`
|
||||
f(simpleCfgStr, request, responseExpected)
|
||||
|
||||
// expired token
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+expiredToken)
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Unauthorized`
|
||||
f(simpleCfgStr, request, responseExpected)
|
||||
|
||||
// invalid signature token
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+invalidSignatureToken)
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Unauthorized`
|
||||
f(simpleCfgStr, request, responseExpected)
|
||||
|
||||
// invalid signature token and skip verify
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+invalidSignatureToken)
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
/foo/abc`
|
||||
f(`
|
||||
users:
|
||||
- jwt:
|
||||
skip_verify: true
|
||||
url_prefix: {BACKEND}/foo`, request, responseExpected)
|
||||
|
||||
// token with default valid vm_access claim
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+defaultVMAccessClaimToken)
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
/foo/abc`
|
||||
f(simpleCfgStr, request, responseExpected)
|
||||
|
||||
// jwt token used but no matching user with JWT token in config
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+defaultVMAccessClaimToken)
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Unauthorized`
|
||||
f(`
|
||||
users:
|
||||
- password: a-password
|
||||
username: a-user
|
||||
url_prefix: {BACKEND}/foo`, request, responseExpected)
|
||||
|
||||
// auth with key from file
|
||||
publicKeyFile := filepath.Join(t.TempDir(), "a_public_key.pem")
|
||||
if err := os.WriteFile(publicKeyFile, []byte(publicKeyPEM), 0o644); err != nil {
|
||||
t.Fatalf("failed to write public key file: %s", err)
|
||||
}
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+defaultVMAccessClaimToken)
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
/foo/abc`
|
||||
f(fmt.Sprintf(`
|
||||
users:
|
||||
- jwt:
|
||||
public_key_files:
|
||||
- %q
|
||||
url_prefix: {BACKEND}/foo`, string(publicKeyFile)), request, responseExpected)
|
||||
}
|
||||
|
||||
type fakeResponseWriter struct {
|
||||
h http.Header
|
||||
|
||||
@@ -1053,7 +832,7 @@ func TestBufferedBody_RetrySuccess(t *testing.T) {
|
||||
if !canRetry {
|
||||
t.Fatalf("canRetry() must return true before reading anything")
|
||||
}
|
||||
for i := range 5 {
|
||||
for i := 0; i < 5; i++ {
|
||||
data, err := io.ReadAll(rb)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error when reading all the data at iteration %d: %s", i, err)
|
||||
@@ -1111,7 +890,7 @@ func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
|
||||
if !canRetry {
|
||||
t.Fatalf("canRetry must return true")
|
||||
}
|
||||
for i := range len(s) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
buf := make([]byte, i)
|
||||
n, err := io.ReadFull(rb, buf)
|
||||
if err != nil {
|
||||
|
||||
@@ -174,7 +174,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
},
|
||||
RetryStatusCodes: []int{503, 501},
|
||||
LoadBalancingPolicy: "first_available",
|
||||
DropSrcPathPrefixParts: new(2),
|
||||
DropSrcPathPrefixParts: intp(2),
|
||||
}, "/a/b/c", "http://foo.bar/c", `bb: aaa`, `x: y`, []int{503, 501}, "first_available", 2)
|
||||
f(&UserInfo{
|
||||
URLPrefix: mustParseURL("http://foo.bar/federate"),
|
||||
@@ -219,13 +219,13 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
},
|
||||
RetryStatusCodes: []int{503, 500, 501},
|
||||
LoadBalancingPolicy: "first_available",
|
||||
DropSrcPathPrefixParts: new(1),
|
||||
DropSrcPathPrefixParts: intp(1),
|
||||
},
|
||||
{
|
||||
SrcPaths: getRegexs([]string{"/api/v1/write"}),
|
||||
URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
|
||||
RetryStatusCodes: []int{},
|
||||
DropSrcPathPrefixParts: new(0),
|
||||
DropSrcPathPrefixParts: intp(0),
|
||||
},
|
||||
{
|
||||
SrcPaths: getRegexs([]string{"/metrics"}),
|
||||
@@ -242,7 +242,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
},
|
||||
},
|
||||
RetryStatusCodes: []int{502},
|
||||
DropSrcPathPrefixParts: new(2),
|
||||
DropSrcPathPrefixParts: intp(2),
|
||||
}
|
||||
f(ui, "http://host42/vmsingle/api/v1/query?query=up&db=foo", "http://vmselect/0/prometheus/api/v1/query?db=foo&query=up",
|
||||
"xx: aa\nyy: asdf", "qwe: rty", []int{503, 500, 501}, "first_available", 1)
|
||||
@@ -259,7 +259,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
SrcPaths: getRegexs([]string{"/api/v1/write"}),
|
||||
URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
|
||||
RetryStatusCodes: []int{},
|
||||
DropSrcPathPrefixParts: new(0),
|
||||
DropSrcPathPrefixParts: intp(0),
|
||||
},
|
||||
{
|
||||
SrcPaths: getRegexs([]string{"/metrics/a/b"}),
|
||||
@@ -275,7 +275,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
|
||||
},
|
||||
},
|
||||
RetryStatusCodes: []int{502},
|
||||
DropSrcPathPrefixParts: new(2),
|
||||
DropSrcPathPrefixParts: intp(2),
|
||||
}
|
||||
f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0)
|
||||
f(ui, "https://foo-host/metrics/a/b", "http://metrics-server/b", "", "", []int{502}, "least_loaded", 2)
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
)
|
||||
|
||||
@@ -47,7 +45,7 @@ func New(retries int, factor float64, minDuration time.Duration) (*Backoff, erro
|
||||
// Retry process retries until all attempts are completed
|
||||
func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
|
||||
var attempt uint64
|
||||
for i := range b.retries {
|
||||
for i := 0; i < b.retries; i++ {
|
||||
err := cb()
|
||||
if err == nil {
|
||||
return attempt, nil
|
||||
@@ -57,7 +55,6 @@ func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
|
||||
return attempt, err // fail fast if not recoverable
|
||||
}
|
||||
attempt++
|
||||
retriesTotal.Inc()
|
||||
backoff := float64(b.minDuration) * math.Pow(b.factor, float64(i))
|
||||
dur := time.Duration(backoff)
|
||||
logger.Errorf("got error: %s on attempt: %d; will retry in %v", err, attempt, dur)
|
||||
@@ -77,7 +74,3 @@ func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
|
||||
}
|
||||
return attempt, fmt.Errorf("execution failed after %d retry attempts", b.retries)
|
||||
}
|
||||
|
||||
var (
|
||||
retriesTotal = metrics.NewCounter(`vmctl_backoff_retries_total`)
|
||||
)
|
||||
|
||||
@@ -14,12 +14,6 @@ const (
|
||||
globalSilent = "s"
|
||||
globalVerbose = "verbose"
|
||||
globalDisableProgressBar = "disable-progress-bar"
|
||||
|
||||
globalPushMetricsURL = "pushmetrics.url"
|
||||
globalPushMetricsInterval = "pushmetrics.interval"
|
||||
globalPushExtraLabels = "pushmetrics.extraLabel"
|
||||
globalPushHeaders = "pushmetrics.header"
|
||||
globalPushDisableCompression = "pushmetrics.disableCompression"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -39,29 +33,6 @@ var (
|
||||
Value: false,
|
||||
Usage: "Whether to disable progress bar during the import.",
|
||||
},
|
||||
&cli.StringSliceFlag{
|
||||
Name: globalPushMetricsURL,
|
||||
Usage: "Optional URL to push metrics. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics",
|
||||
},
|
||||
&cli.DurationFlag{
|
||||
Name: globalPushMetricsInterval,
|
||||
Value: 10 * time.Second,
|
||||
Usage: "Interval for pushing metrics to every -pushmetrics.url",
|
||||
},
|
||||
&cli.StringSliceFlag{
|
||||
Name: globalPushExtraLabels,
|
||||
Usage: "Extra labels to add to pushed metrics. In case of collision, label value defined by flag will have priority. " +
|
||||
"Flag can be set multiple times, to add few additional labels. " +
|
||||
"For example, -pushmetrics.extraLabel='instance=\"foo\"' adds instance=\"foo\" label to all the metrics pushed to every -pushmetrics.url",
|
||||
},
|
||||
&cli.StringSliceFlag{
|
||||
Name: globalPushHeaders,
|
||||
Usage: "Optional HTTP headers to add to pushed metrics. Flag can be set multiple times, to add few additional headers.",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: globalPushDisableCompression,
|
||||
Usage: "Whether to disable compression when pushing metrics.",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
|
||||
@@ -54,7 +52,6 @@ func (ip *influxProcessor) run(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
influxSeriesTotal.Add(len(series))
|
||||
bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing series"), len(series))
|
||||
if err := barpool.Start(); err != nil {
|
||||
return err
|
||||
@@ -70,11 +67,9 @@ func (ip *influxProcessor) run(ctx context.Context) error {
|
||||
wg.Go(func() {
|
||||
for s := range seriesCh {
|
||||
if err := ip.do(s); err != nil {
|
||||
influxErrorsTotal.Inc()
|
||||
errCh <- fmt.Errorf("request failed for %q.%q: %s", s.Measurement, s.Field, err)
|
||||
return
|
||||
}
|
||||
influxSeriesProcessed.Inc()
|
||||
bar.Increment()
|
||||
}
|
||||
})
|
||||
@@ -86,7 +81,6 @@ func (ip *influxProcessor) run(ctx context.Context) error {
|
||||
case infErr := <-errCh:
|
||||
return fmt.Errorf("influx error: %s", infErr)
|
||||
case vmErr := <-ip.im.Errors():
|
||||
influxErrorsTotal.Inc()
|
||||
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, ip.isVerbose))
|
||||
case seriesCh <- s:
|
||||
}
|
||||
@@ -99,7 +93,6 @@ func (ip *influxProcessor) run(ctx context.Context) error {
|
||||
// drain import errors channel
|
||||
for vmErr := range ip.im.Errors() {
|
||||
if vmErr.Err != nil {
|
||||
influxErrorsTotal.Inc()
|
||||
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, ip.isVerbose))
|
||||
}
|
||||
}
|
||||
@@ -174,9 +167,3 @@ func (ip *influxProcessor) do(s *influx.Series) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
influxSeriesTotal = metrics.NewCounter(`vmctl_influx_migration_series_total`)
|
||||
influxSeriesProcessed = metrics.NewCounter(`vmctl_influx_migration_series_processed`)
|
||||
influxErrorsTotal = metrics.NewCounter(`vmctl_influx_migration_errors_total`)
|
||||
)
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
)
|
||||
|
||||
@@ -47,16 +45,9 @@ func (l *Limiter) Register(dataLen int) {
|
||||
t := timerpool.Get(d)
|
||||
<-t.C
|
||||
timerpool.Put(t)
|
||||
limiterThrottleEventsTotal.Inc()
|
||||
}
|
||||
l.budget += limit
|
||||
l.deadline = time.Now().Add(time.Second)
|
||||
}
|
||||
l.budget -= int64(dataLen)
|
||||
limiterBytesProcessed.Add(dataLen)
|
||||
}
|
||||
|
||||
var (
|
||||
limiterBytesProcessed = metrics.NewCounter(`vmctl_limiter_bytes_processed_total`)
|
||||
limiterThrottleEventsTotal = metrics.NewCounter(`vmctl_limiter_throttle_events_total`)
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
@@ -20,9 +19,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
|
||||
@@ -44,20 +41,11 @@ func main() {
|
||||
ctx, cancelCtx := context.WithCancel(context.Background())
|
||||
start := time.Now()
|
||||
beforeFn := func(c *cli.Context) error {
|
||||
flag.Parse()
|
||||
logger.Init()
|
||||
isSilent = c.Bool(globalSilent)
|
||||
if c.Bool(globalDisableProgressBar) {
|
||||
barpool.Disable(true)
|
||||
}
|
||||
netutil.EnableIPv6()
|
||||
pushmetrics.InitWith(&pushmetrics.Config{
|
||||
URLs: c.StringSlice(globalPushMetricsURL),
|
||||
Interval: c.Duration(globalPushMetricsInterval),
|
||||
ExtraLabels: c.StringSlice(globalPushExtraLabels),
|
||||
DisableCompression: c.Bool(globalPushDisableCompression),
|
||||
Headers: c.StringSlice(globalPushHeaders),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
app := &cli.App{
|
||||
@@ -463,7 +451,6 @@ func main() {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
log.Printf("Total time: %v", time.Since(start))
|
||||
pushmetrics.StopAndPush()
|
||||
}
|
||||
|
||||
func initConfigVM(c *cli.Context) (vm.Config, error) {
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
|
||||
)
|
||||
|
||||
@@ -38,15 +36,12 @@ type Response struct {
|
||||
|
||||
// Explore finds metric names by provided filter from api/v1/label/__name__/values
|
||||
func (c *Client) Explore(ctx context.Context, f Filter, tenantID string, start, end time.Time) ([]string, error) {
|
||||
startTime := time.Now()
|
||||
exploreRequestsTotal.Inc()
|
||||
url := fmt.Sprintf("%s/%s", c.Addr, nativeMetricNamesAddr)
|
||||
if tenantID != "" {
|
||||
url = fmt.Sprintf("%s/select/%s/prometheus/%s", c.Addr, tenantID, nativeMetricNamesAddr)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
exploreRequestsErrorsTotal.Inc()
|
||||
return nil, fmt.Errorf("cannot create request to %q: %s", url, err)
|
||||
}
|
||||
|
||||
@@ -58,53 +53,37 @@ func (c *Client) Explore(ctx context.Context, f Filter, tenantID string, start,
|
||||
|
||||
resp, err := c.do(req, http.StatusOK)
|
||||
if err != nil {
|
||||
exploreRequestsErrorsTotal.Inc()
|
||||
exploreDuration.UpdateDuration(startTime)
|
||||
return nil, fmt.Errorf("series request failed: %s", err)
|
||||
}
|
||||
|
||||
var response Response
|
||||
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
|
||||
exploreRequestsErrorsTotal.Inc()
|
||||
exploreDuration.UpdateDuration(startTime)
|
||||
return nil, fmt.Errorf("cannot decode series response: %s", err)
|
||||
}
|
||||
exploreDuration.UpdateDuration(startTime)
|
||||
return response.MetricNames, resp.Body.Close()
|
||||
}
|
||||
|
||||
// ImportPipe uses pipe reader in request to process data
|
||||
func (c *Client) ImportPipe(ctx context.Context, dstURL string, pr *io.PipeReader) error {
|
||||
startTime := time.Now()
|
||||
importRequestsTotal.Inc()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, dstURL, pr)
|
||||
if err != nil {
|
||||
importRequestsErrorsTotal.Inc()
|
||||
return fmt.Errorf("cannot create import request to %q: %s", c.Addr, err)
|
||||
}
|
||||
|
||||
importResp, err := c.do(req, http.StatusNoContent)
|
||||
if err != nil {
|
||||
importRequestsErrorsTotal.Inc()
|
||||
importDuration.UpdateDuration(startTime)
|
||||
return fmt.Errorf("import request failed: %s", err)
|
||||
}
|
||||
if err := importResp.Body.Close(); err != nil {
|
||||
importRequestsErrorsTotal.Inc()
|
||||
importDuration.UpdateDuration(startTime)
|
||||
return fmt.Errorf("cannot close import response body: %s", err)
|
||||
}
|
||||
importDuration.UpdateDuration(startTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExportPipe makes request by provided filter and return io.ReadCloser which can be used to get data
|
||||
func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadCloser, error) {
|
||||
startTime := time.Now()
|
||||
exportRequestsTotal.Inc()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
exportRequestsErrorsTotal.Inc()
|
||||
return nil, fmt.Errorf("cannot create request to %q: %s", c.Addr, err)
|
||||
}
|
||||
|
||||
@@ -123,11 +102,8 @@ func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadC
|
||||
|
||||
resp, err := c.do(req, http.StatusOK)
|
||||
if err != nil {
|
||||
exportRequestsErrorsTotal.Inc()
|
||||
exportDuration.UpdateDuration(startTime)
|
||||
return nil, fmt.Errorf("export request failed: %w", err)
|
||||
}
|
||||
exportDuration.UpdateDuration(startTime)
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
@@ -186,16 +162,3 @@ func (c *Client) do(req *http.Request, expSC int) (*http.Response, error) {
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
var (
|
||||
importRequestsTotal = metrics.NewCounter(`vmctl_vm_native_requests_total{type="import"}`)
|
||||
exportRequestsTotal = metrics.NewCounter(`vmctl_vm_native_requests_total{type="export"}`)
|
||||
exploreRequestsTotal = metrics.NewCounter(`vmctl_vm_native_requests_total{type="explore"}`)
|
||||
importRequestsErrorsTotal = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="import"}`)
|
||||
exportRequestsErrorsTotal = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="export"}`)
|
||||
exploreRequestsErrorsTotal = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="explore"}`)
|
||||
|
||||
importDuration = metrics.NewHistogram(`vmctl_vm_native_import_duration_seconds`)
|
||||
exportDuration = metrics.NewHistogram(`vmctl_vm_native_export_duration_seconds`)
|
||||
exploreDuration = metrics.NewHistogram(`vmctl_vm_native_explore_duration_seconds`)
|
||||
)
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
vmetrics "github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
|
||||
"github.com/cheggaaa/pb/v3"
|
||||
@@ -59,7 +57,6 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
|
||||
if !prompt(ctx, question) {
|
||||
return nil
|
||||
}
|
||||
|
||||
op.im.ResetStats()
|
||||
var startTime int64
|
||||
if op.oc.HardTS != 0 {
|
||||
@@ -87,7 +84,6 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
|
||||
seriesCh := make(chan queryObj, op.otsdbcc)
|
||||
errCh := make(chan error)
|
||||
// we're going to make serieslist * queryRanges queries, so we should represent that in the progress bar
|
||||
otsdbSeriesTotal.Add(len(serieslist) * queryRanges)
|
||||
bar := pb.StartNew(len(serieslist) * queryRanges)
|
||||
defer func(bar *pb.ProgressBar) {
|
||||
bar.Finish()
|
||||
@@ -97,11 +93,9 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
|
||||
wg.Go(func() {
|
||||
for s := range seriesCh {
|
||||
if err := op.do(s); err != nil {
|
||||
otsdbErrorsTotal.Inc()
|
||||
errCh <- fmt.Errorf("couldn't retrieve series for %s : %s", metric, err)
|
||||
return
|
||||
}
|
||||
otsdbSeriesProcessed.Inc()
|
||||
bar.Increment()
|
||||
}
|
||||
})
|
||||
@@ -121,7 +115,6 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
|
||||
case otsdbErr := <-errCh:
|
||||
return fmt.Errorf("opentsdb error: %s", otsdbErr)
|
||||
case vmErr := <-op.im.Errors():
|
||||
otsdbErrorsTotal.Inc()
|
||||
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, op.isVerbose))
|
||||
case seriesCh <- queryObj{
|
||||
Tr: tr, StartTime: startTime,
|
||||
@@ -146,7 +139,6 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
|
||||
op.im.Close()
|
||||
for vmErr := range op.im.Errors() {
|
||||
if vmErr.Err != nil {
|
||||
otsdbErrorsTotal.Inc()
|
||||
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, op.isVerbose))
|
||||
}
|
||||
}
|
||||
@@ -177,9 +169,3 @@ func (op *otsdbProcessor) do(s queryObj) error {
|
||||
}
|
||||
return op.im.Input(&ts)
|
||||
}
|
||||
|
||||
var (
|
||||
otsdbSeriesTotal = vmetrics.NewCounter(`vmctl_opentsdb_migration_series_total`)
|
||||
otsdbSeriesProcessed = vmetrics.NewCounter(`vmctl_opentsdb_migration_series_processed`)
|
||||
otsdbErrorsTotal = vmetrics.NewCounter(`vmctl_opentsdb_migration_errors_total`)
|
||||
)
|
||||
|
||||
@@ -109,7 +109,7 @@ func (c Client) FindMetrics(q string) ([]string, error) {
|
||||
return nil, fmt.Errorf("failed to send GET request to %q: %s", q, err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, fmt.Errorf("bad return from OpenTSDB: %d: %v", resp.StatusCode, resp)
|
||||
return nil, fmt.Errorf("bad return from OpenTSDB: %q: %v", resp.StatusCode, resp)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
@@ -133,7 +133,7 @@ func (c Client) FindSeries(metric string) ([]Meta, error) {
|
||||
return nil, fmt.Errorf("failed to set GET request to %q: %s", q, err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, fmt.Errorf("bad return from OpenTSDB: %d: %v", resp.StatusCode, resp)
|
||||
return nil, fmt.Errorf("bad return from OpenTSDB: %q: %v", resp.StatusCode, resp)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
|
||||
@@ -11,8 +11,6 @@ import (
|
||||
"github.com/prometheus/prometheus/tsdb"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
|
||||
@@ -115,7 +113,6 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
|
||||
}
|
||||
|
||||
func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
|
||||
promBlocksTotal.Add(len(blocks))
|
||||
bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
|
||||
if err := barpool.Start(); err != nil {
|
||||
return err
|
||||
@@ -131,11 +128,9 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
|
||||
wg.Go(func() {
|
||||
for br := range blockReadersCh {
|
||||
if err := pp.do(br); err != nil {
|
||||
promErrorsTotal.Inc()
|
||||
errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
|
||||
return
|
||||
}
|
||||
promBlocksProcessed.Inc()
|
||||
bar.Increment()
|
||||
}
|
||||
})
|
||||
@@ -148,7 +143,6 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
|
||||
return fmt.Errorf("prometheus error: %s", promErr)
|
||||
case vmErr := <-pp.im.Errors():
|
||||
close(blockReadersCh)
|
||||
promErrorsTotal.Inc()
|
||||
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
|
||||
case blockReadersCh <- br:
|
||||
}
|
||||
@@ -162,7 +156,6 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
|
||||
// drain import errors channel
|
||||
for vmErr := range pp.im.Errors() {
|
||||
if vmErr.Err != nil {
|
||||
promErrorsTotal.Inc()
|
||||
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
|
||||
}
|
||||
}
|
||||
@@ -172,9 +165,3 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
promBlocksTotal = metrics.NewCounter(`vmctl_prometheus_migration_blocks_total`)
|
||||
promBlocksProcessed = metrics.NewCounter(`vmctl_prometheus_migration_blocks_processed`)
|
||||
promErrorsTotal = metrics.NewCounter(`vmctl_prometheus_migration_errors_total`)
|
||||
)
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
|
||||
@@ -53,7 +51,6 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
remoteReadRangesTotal.Add(len(ranges))
|
||||
bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing ranges"), len(ranges))
|
||||
if err := barpool.Start(); err != nil {
|
||||
return err
|
||||
@@ -73,11 +70,9 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
|
||||
wg.Go(func() {
|
||||
for r := range rangeC {
|
||||
if err := rrp.do(ctx, r); err != nil {
|
||||
remoteReadErrorsTotal.Inc()
|
||||
errCh <- fmt.Errorf("request failed for: %s", err)
|
||||
return
|
||||
}
|
||||
remoteReadRangesProcessed.Inc()
|
||||
bar.Increment()
|
||||
}
|
||||
})
|
||||
@@ -88,7 +83,6 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
|
||||
case infErr := <-errCh:
|
||||
return fmt.Errorf("remote read error: %s", infErr)
|
||||
case vmErr := <-rrp.dst.Errors():
|
||||
remoteReadErrorsTotal.Inc()
|
||||
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, rrp.isVerbose))
|
||||
case rangeC <- &remoteread.Filter{
|
||||
StartTimestampMs: r[0].UnixMilli(),
|
||||
@@ -104,7 +98,6 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
|
||||
// drain import errors channel
|
||||
for vmErr := range rrp.dst.Errors() {
|
||||
if vmErr.Err != nil {
|
||||
remoteReadErrorsTotal.Inc()
|
||||
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, rrp.isVerbose))
|
||||
}
|
||||
}
|
||||
@@ -125,9 +118,3 @@ func (rrp *remoteReadProcessor) do(ctx context.Context, filter *remoteread.Filte
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
var (
|
||||
remoteReadRangesTotal = metrics.NewCounter(`vmctl_remote_read_migration_ranges_total`)
|
||||
remoteReadRangesProcessed = metrics.NewCounter(`vmctl_remote_read_migration_ranges_processed`)
|
||||
remoteReadErrorsTotal = metrics.NewCounter(`vmctl_remote_read_migration_errors_total`)
|
||||
)
|
||||
|
||||
@@ -76,11 +76,11 @@ func (ts *TimeSeries) write(w io.Writer) (int, error) {
|
||||
|
||||
pointsCount := len(timestampsBatch)
|
||||
cw.printf(`},"timestamps":[`)
|
||||
for i := range pointsCount - 1 {
|
||||
for i := 0; i < pointsCount-1; i++ {
|
||||
cw.printf(`%d,`, timestampsBatch[i])
|
||||
}
|
||||
cw.printf(`%d],"values":[`, timestampsBatch[pointsCount-1])
|
||||
for i := range pointsCount - 1 {
|
||||
for i := 0; i < pointsCount-1; i++ {
|
||||
cw.printf(`%v,`, valuesBatch[i])
|
||||
}
|
||||
cw.printf("%v]}\n", valuesBatch[pointsCount-1])
|
||||
|
||||
@@ -12,8 +12,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
|
||||
@@ -82,12 +80,6 @@ type Importer struct {
|
||||
|
||||
s *stats
|
||||
backoff *backoff.Backoff
|
||||
|
||||
importRequestsTotal *metrics.Counter
|
||||
importRequestsErrorsTotal *metrics.Counter
|
||||
importSamplesTotal *metrics.Counter
|
||||
importBytesTotal *metrics.Counter
|
||||
importDuration *metrics.Histogram
|
||||
}
|
||||
|
||||
// ResetStats resets im stats.
|
||||
@@ -155,12 +147,6 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
|
||||
input: make(chan *TimeSeries, cfg.Concurrency*4),
|
||||
errors: make(chan *ImportError, cfg.Concurrency),
|
||||
backoff: cfg.Backoff,
|
||||
|
||||
importRequestsTotal: metrics.GetOrCreateCounter(`vmctl_importer_requests_total`),
|
||||
importRequestsErrorsTotal: metrics.GetOrCreateCounter(`vmctl_importer_request_errors_total`),
|
||||
importSamplesTotal: metrics.GetOrCreateCounter(`vmctl_importer_samples_total`),
|
||||
importBytesTotal: metrics.GetOrCreateCounter(`vmctl_importer_bytes_total`),
|
||||
importDuration: metrics.GetOrCreateHistogram(`vmctl_importer_request_duration_seconds`),
|
||||
}
|
||||
if err := im.Ping(); err != nil {
|
||||
return nil, fmt.Errorf("ping to %q failed: %s", addr, err)
|
||||
@@ -325,13 +311,9 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
im.importRequestsTotal.Inc()
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
req, err := http.NewRequest(http.MethodPost, im.importPath, pr)
|
||||
if err != nil {
|
||||
im.importRequestsErrorsTotal.Inc()
|
||||
return fmt.Errorf("cannot create request to %q: %s", im.addr, err)
|
||||
}
|
||||
if im.user != "" {
|
||||
@@ -351,7 +333,6 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
|
||||
if im.compress {
|
||||
zw, err := gzip.NewWriterLevel(w, 1)
|
||||
if err != nil {
|
||||
im.importRequestsErrorsTotal.Inc()
|
||||
return fmt.Errorf("unexpected error when creating gzip writer: %s", err)
|
||||
}
|
||||
w = zw
|
||||
@@ -363,39 +344,29 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
|
||||
for _, ts := range tsBatch {
|
||||
n, err := ts.write(bw)
|
||||
if err != nil {
|
||||
im.importRequestsErrorsTotal.Inc()
|
||||
return fmt.Errorf("write err: %w", err)
|
||||
}
|
||||
totalBytes += n
|
||||
totalSamples += len(ts.Values)
|
||||
}
|
||||
if err := bw.Flush(); err != nil {
|
||||
im.importRequestsErrorsTotal.Inc()
|
||||
return err
|
||||
}
|
||||
if closer, ok := w.(io.Closer); ok {
|
||||
err := closer.Close()
|
||||
if err != nil {
|
||||
im.importRequestsErrorsTotal.Inc()
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := pw.Close(); err != nil {
|
||||
im.importRequestsErrorsTotal.Inc()
|
||||
return err
|
||||
}
|
||||
|
||||
requestErr := <-errCh
|
||||
if requestErr != nil {
|
||||
im.importRequestsErrorsTotal.Inc()
|
||||
im.importDuration.UpdateDuration(startTime)
|
||||
return fmt.Errorf("import request error for %q: %w", im.addr, requestErr)
|
||||
}
|
||||
|
||||
im.importSamplesTotal.Add(totalSamples)
|
||||
im.importBytesTotal.Add(totalBytes)
|
||||
im.importDuration.UpdateDuration(startTime)
|
||||
|
||||
im.s.Lock()
|
||||
im.s.bytes += uint64(totalBytes)
|
||||
im.s.samples += uint64(totalSamples)
|
||||
|
||||
@@ -9,8 +9,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
|
||||
@@ -84,19 +82,13 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
|
||||
if !prompt(ctx, question) {
|
||||
return nil
|
||||
}
|
||||
migrationTenantsTotal.Set(uint64(len(tenants)))
|
||||
}
|
||||
|
||||
for _, tenantID := range tenants {
|
||||
err := p.runBackfilling(ctx, tenantID, ranges)
|
||||
if err != nil {
|
||||
migrationErrorsTotal.Inc()
|
||||
return fmt.Errorf("migration failed: %s", err)
|
||||
}
|
||||
|
||||
if p.interCluster {
|
||||
migrationTenantsProcessed.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("Import finished!")
|
||||
@@ -164,7 +156,6 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU
|
||||
p.s.bytes += uint64(written)
|
||||
p.s.requests++
|
||||
p.s.Unlock()
|
||||
migrationBytesTransferredTotal.AddInt64(written)
|
||||
|
||||
if err := pw.Close(); err != nil {
|
||||
return err
|
||||
@@ -208,7 +199,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
|
||||
|
||||
var foundSeriesMsg string
|
||||
var requestsToMake int
|
||||
var metricsMap = map[string][][]time.Time{
|
||||
var metrics = map[string][][]time.Time{
|
||||
"": ranges,
|
||||
}
|
||||
|
||||
@@ -220,11 +211,11 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
|
||||
|
||||
if !p.disablePerMetricRequests {
|
||||
format = fmt.Sprintf(nativeWithBackoffTpl, barPrefix)
|
||||
metricsMap, err = p.explore(ctx, p.src, tenantID, ranges)
|
||||
metrics, err = p.explore(ctx, p.src, tenantID, ranges)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to explore metric names: %s", err)
|
||||
}
|
||||
if len(metricsMap) == 0 {
|
||||
if len(metrics) == 0 {
|
||||
errMsg := "no metrics found"
|
||||
if tenantID != "" {
|
||||
errMsg = fmt.Sprintf("%s for tenant id: %s", errMsg, tenantID)
|
||||
@@ -232,14 +223,10 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
|
||||
log.Println(errMsg)
|
||||
return nil
|
||||
}
|
||||
for _, m := range metricsMap {
|
||||
for _, m := range metrics {
|
||||
requestsToMake += len(m)
|
||||
}
|
||||
foundSeriesMsg = fmt.Sprintf("Found %d unique metric names to import. Total import/export requests to make %d", len(metricsMap), requestsToMake)
|
||||
|
||||
migrationMetricsTotal.Add(len(metricsMap))
|
||||
} else {
|
||||
requestsToMake = len(ranges)
|
||||
foundSeriesMsg = fmt.Sprintf("Found %d unique metric names to import. Total import/export requests to make %d", len(metrics), requestsToMake)
|
||||
}
|
||||
|
||||
if !p.interCluster {
|
||||
@@ -253,7 +240,6 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
|
||||
log.Print(foundSeriesMsg)
|
||||
}
|
||||
|
||||
migrationRequestsPlanned.Add(requestsToMake)
|
||||
bar := barpool.NewSingleProgress(format, requestsToMake)
|
||||
bar.Start()
|
||||
defer bar.Finish()
|
||||
@@ -262,7 +248,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
|
||||
errCh := make(chan error, p.cc)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for range p.cc {
|
||||
for i := 0; i < p.cc; i++ {
|
||||
wg.Go(func() {
|
||||
for f := range filterCh {
|
||||
if !p.disablePerMetricRequests {
|
||||
@@ -277,13 +263,12 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
|
||||
return
|
||||
}
|
||||
}
|
||||
migrationRequestsCompleted.Inc()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// any error breaks the import
|
||||
for mName, mRanges := range metricsMap {
|
||||
for mName, mRanges := range metrics {
|
||||
match, err := buildMatchWithFilter(p.filter.Match, mName)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to build filter %q for metric name %q: %s", p.filter.Match, mName, err)
|
||||
@@ -303,9 +288,6 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
|
||||
}:
|
||||
}
|
||||
}
|
||||
if !p.disablePerMetricRequests {
|
||||
migrationMetricsProcessed.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
close(filterCh)
|
||||
@@ -414,18 +396,3 @@ func buildMatchWithFilter(filter string, metricName string) (string, error) {
|
||||
match := "{" + strings.Join(filters, " or ") + "}"
|
||||
return match, nil
|
||||
}
|
||||
|
||||
var (
|
||||
migrationMetricsTotal = metrics.NewCounter(`vmctl_vm_native_migration_metrics_total`)
|
||||
migrationMetricsProcessed = metrics.NewCounter(`vmctl_vm_native_migration_metrics_processed`)
|
||||
|
||||
migrationRequestsPlanned = metrics.NewCounter(`vmctl_vm_native_migration_requests_planned`)
|
||||
migrationRequestsCompleted = metrics.NewCounter(`vmctl_vm_native_migration_requests_completed`)
|
||||
|
||||
migrationErrorsTotal = metrics.NewCounter(`vmctl_vm_native_migration_errors_total`)
|
||||
|
||||
migrationTenantsTotal = metrics.NewCounter(`vmctl_vm_native_migration_tenants_total`)
|
||||
migrationTenantsProcessed = metrics.NewCounter(`vmctl_vm_native_migration_tenants_processed`)
|
||||
|
||||
migrationBytesTransferredTotal = metrics.NewCounter(`vmctl_vm_native_migration_bytes_transferred_total`)
|
||||
)
|
||||
|
||||
@@ -182,7 +182,6 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
|
||||
mm.Type = mmpb.Type
|
||||
mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
|
||||
}
|
||||
ctx.mms = mms
|
||||
|
||||
err := vmstorage.AddMetadataRows(mms)
|
||||
if err != nil {
|
||||
@@ -207,7 +206,6 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
|
||||
mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
|
||||
mm.Type = mmpb.Type
|
||||
}
|
||||
ctx.mms = mms
|
||||
|
||||
err := vmstorage.AddMetadataRows(mms)
|
||||
if err != nil {
|
||||
|
||||
@@ -142,7 +142,7 @@ type aggrStatePercentile struct {
|
||||
|
||||
func newAggrStatePercentile(pointsLen int, n float64) aggrState {
|
||||
hs := make([]*histogram.Fast, pointsLen)
|
||||
for i := range pointsLen {
|
||||
for i := 0; i < pointsLen; i++ {
|
||||
hs[i] = histogram.NewFast()
|
||||
}
|
||||
return &aggrStatePercentile{
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
@@ -50,7 +49,7 @@ func (ec *evalConfig) newTimestamps(step int64) []int64 {
|
||||
pointsLen := ec.pointsLen(step)
|
||||
timestamps := make([]int64, pointsLen)
|
||||
ts := ec.startTime
|
||||
for i := range pointsLen {
|
||||
for i := 0; i < pointsLen; i++ {
|
||||
timestamps[i] = ts
|
||||
ts += step
|
||||
}
|
||||
@@ -197,17 +196,12 @@ func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr g
|
||||
pathExpression: safePathExpression(expr),
|
||||
}
|
||||
s.summarize(aggrAvg, ec.startTime, ec.endTime, ec.storageStep, 0)
|
||||
|
||||
// A negative or zero duration will cause timer.C to return immediately
|
||||
remainingTimeout := ec.deadline.Deadline() - fasttime.UnixTimestamp()
|
||||
t := timerpool.Get(time.Duration(remainingTimeout) * time.Second)
|
||||
t := timerpool.Get(30 * time.Second)
|
||||
defer timerpool.Put(t)
|
||||
|
||||
select {
|
||||
case seriesCh <- s:
|
||||
case <-t.C:
|
||||
logger.Errorf("reached timeout when processing the %s (full query: %s), it can be due to the amount of storageNodes configured in vmselect is more than vmselect’s available CPU count "+
|
||||
"or vmselect is heavy loaded. Consider adding resources or increasing `-search.maxQueryDuration` or `timeout` parameter in the query.",
|
||||
logger.Errorf("resource leak when processing the %s (full query: %s); please report this error to VictoriaMetrics developers",
|
||||
expr.AppendString(nil), ec.originalQuery)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -25,7 +25,7 @@ func naturalLess(a, b string) bool {
|
||||
}
|
||||
|
||||
func getNonNumPrefix(s string) (prefix string, tail string) {
|
||||
for i := range len(s) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
ch := s[i]
|
||||
if ch >= '0' && ch <= '9' {
|
||||
return s[:i], s[i:]
|
||||
|
||||
@@ -82,7 +82,7 @@ func RenderHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
|
||||
if s := r.FormValue("maxDataPoints"); len(s) > 0 {
|
||||
n, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse maxDataPoints=%d: %w", maxDataPoints, err)
|
||||
return fmt.Errorf("cannot parse maxDataPoints=%q: %w", maxDataPoints, err)
|
||||
}
|
||||
if n <= 0 {
|
||||
return fmt.Errorf("maxDataPoints must be greater than 0; got %f", n)
|
||||
@@ -209,7 +209,7 @@ func parseInterval(s string) (int64, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
prefix := s
|
||||
var suffix string
|
||||
for i := range len(s) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
ch := s[i]
|
||||
if ch != '-' && ch != '+' && ch != '.' && (ch < '0' || ch > '9') {
|
||||
prefix = s[:i]
|
||||
|
||||
@@ -1228,7 +1228,7 @@ func transformDelay(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, er
|
||||
stepsLocal = len(values)
|
||||
}
|
||||
copy(values[stepsLocal:], values[:len(values)-stepsLocal])
|
||||
for i := range stepsLocal {
|
||||
for i := 0; i < stepsLocal; i++ {
|
||||
values[i] = nan
|
||||
}
|
||||
}
|
||||
@@ -1740,7 +1740,7 @@ func transformGroup(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, er
|
||||
|
||||
func groupSeriesLists(ec *evalConfig, args []*graphiteql.ArgExpr, expr graphiteql.Expr) (nextSeriesFunc, error) {
|
||||
var nextSeriess []nextSeriesFunc
|
||||
for i := range args {
|
||||
for i := 0; i < len(args); i++ {
|
||||
nextSeries, err := evalSeriesList(ec, args, "seriesList", i)
|
||||
if err != nil {
|
||||
for _, f := range nextSeriess {
|
||||
@@ -3233,7 +3233,7 @@ func transformSeriesByTag(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFu
|
||||
return nil, fmt.Errorf("at least one tagExpression must be passed to seriesByTag")
|
||||
}
|
||||
var tagExpressions []string
|
||||
for i := range args {
|
||||
for i := 0; i < len(args); i++ {
|
||||
te, err := getString(args, "tagExpressions", i)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -3633,7 +3633,7 @@ var graphiteToGolangRe = regexp.MustCompile(`\\(\d+)`)
|
||||
|
||||
func getNodes(args []*graphiteql.ArgExpr) ([]graphiteql.Expr, error) {
|
||||
var nodes []graphiteql.Expr
|
||||
for i := range args {
|
||||
for i := 0; i < len(args); i++ {
|
||||
expr := args[i].Expr
|
||||
switch expr.(type) {
|
||||
case *graphiteql.NumberExpr, *graphiteql.StringExpr:
|
||||
@@ -4052,7 +4052,7 @@ func formatPathsFromSeriesExpressions(seriesExpressions []string, sortPaths bool
|
||||
|
||||
func newNaNSeries(ec *evalConfig, step int64) *series {
|
||||
values := make([]float64, ec.pointsLen(step))
|
||||
for i := range values {
|
||||
for i := 0; i < len(values); i++ {
|
||||
values[i] = nan
|
||||
}
|
||||
return &series{
|
||||
@@ -5244,7 +5244,7 @@ func transformLinearRegression(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSer
|
||||
|
||||
func linearRegressionForSeries(ec *evalConfig, fe *graphiteql.FuncExpr, ss, sourceSeries []*series) (nextSeriesFunc, error) {
|
||||
var resp []*series
|
||||
for i := range ss {
|
||||
for i := 0; i < len(ss); i++ {
|
||||
source := sourceSeries[i]
|
||||
s := ss[i]
|
||||
s.Tags["linearRegressions"] = fmt.Sprintf("%d, %d", ec.startTime/1e3, ec.endTime/1e3)
|
||||
@@ -5258,7 +5258,7 @@ func linearRegressionForSeries(ec *evalConfig, fe *graphiteql.FuncExpr, ss, sour
|
||||
continue
|
||||
}
|
||||
values := s.Values
|
||||
for j := range values {
|
||||
for j := 0; j < len(values); j++ {
|
||||
values[j] = offset + (float64(int(s.Timestamps[0])+j*int(s.step)))*factor
|
||||
}
|
||||
resp = append(resp, s)
|
||||
@@ -5370,7 +5370,7 @@ func holtWinterConfidenceBands(ec *evalConfig, fe *graphiteql.FuncExpr, args []*
|
||||
valuesLen := len(forecastValues)
|
||||
upperBand := make([]float64, 0, valuesLen)
|
||||
lowerBand := make([]float64, 0, valuesLen)
|
||||
for i := range valuesLen {
|
||||
for i := 0; i < valuesLen; i++ {
|
||||
forecastItem := forecastValues[i]
|
||||
deviationItem := deviationValues[i]
|
||||
if math.IsNaN(forecastItem) || math.IsNaN(deviationItem) {
|
||||
@@ -5464,7 +5464,7 @@ func transformHoltWintersAberration(ec *evalConfig, fe *graphiteql.FuncExpr) (ne
|
||||
return nil, fmt.Errorf("bug, len mismatch for series: %d and upperBand values: %d or lowerBand values: %d", len(values), len(upperBand), len(lowerBand))
|
||||
}
|
||||
aberration := make([]float64, 0, len(values))
|
||||
for i := range values {
|
||||
for i := 0; i < len(values); i++ {
|
||||
v := values[i]
|
||||
upperValue := upperBand[i]
|
||||
lowerValue := lowerBand[i]
|
||||
|
||||
@@ -280,7 +280,7 @@ func isMetricExprChar(ch byte) bool {
|
||||
}
|
||||
|
||||
func appendEscapedIdent(dst []byte, s string) []byte {
|
||||
for i := range len(s) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
ch := s[i]
|
||||
if isIdentChar(ch) || isMetricExprChar(ch) {
|
||||
if i == 0 && !isFirstIdentChar(ch) {
|
||||
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -492,7 +490,10 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
|
||||
}
|
||||
|
||||
// Prepare worker channels.
|
||||
workers := max(min(len(upws), gomaxprocs), 1)
|
||||
workers := min(len(upws), gomaxprocs)
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
itemsPerWorker := (len(upws) + workers - 1) / workers
|
||||
workChs := make([]chan *unpackWork, workers)
|
||||
for i := range workChs {
|
||||
@@ -577,7 +578,6 @@ func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
|
||||
return
|
||||
}
|
||||
heap.Init(sbh)
|
||||
var dedupSamples int
|
||||
for {
|
||||
sbs := sbh.sbs
|
||||
top := sbs[0]
|
||||
@@ -593,7 +593,6 @@ func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
|
||||
if n := equalSamplesPrefix(top, sbNext); n > 0 && dedupInterval > 0 {
|
||||
// Skip n replicated samples at top if deduplication is enabled.
|
||||
top.NextIdx = topNextIdx + n
|
||||
dedupSamples += n
|
||||
} else {
|
||||
// Copy samples from top to dst with timestamps not exceeding tsNext.
|
||||
top.NextIdx = topNextIdx + binarySearchTimestamps(top.Timestamps[topNextIdx:], tsNext)
|
||||
@@ -608,8 +607,8 @@ func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
|
||||
}
|
||||
}
|
||||
timestamps, values := storage.DeduplicateSamples(dst.Timestamps, dst.Values, dedupInterval)
|
||||
dedupSamples += len(dst.Timestamps) - len(timestamps)
|
||||
dedupsDuringSelect.Add(dedupSamples)
|
||||
dedups := len(dst.Timestamps) - len(timestamps)
|
||||
dedupsDuringSelect.Add(dedups)
|
||||
dst.Timestamps = timestamps
|
||||
dst.Values = values
|
||||
}
|
||||
@@ -635,7 +634,7 @@ func equalTimestampsPrefix(a, b []int64) int {
|
||||
|
||||
func equalValuesPrefix(a, b []float64) int {
|
||||
for i, v := range a {
|
||||
if i >= len(b) || math.Float64bits(v) != math.Float64bits(b[i]) {
|
||||
if i >= len(b) || v != b[i] {
|
||||
return i
|
||||
}
|
||||
}
|
||||
@@ -830,7 +829,12 @@ func GraphiteTags(qt *querytracer.Tracer, filter string, limit int, deadline sea
|
||||
}
|
||||
|
||||
func hasString(a []string, s string) bool {
|
||||
return slices.Contains(a, s)
|
||||
for _, x := range a {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// LabelValues returns label values matching the given labelName and sq until the given deadline.
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
package netstorage
|
||||
|
||||
import (
|
||||
"math"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
||||
)
|
||||
|
||||
func TestMergeSortBlocks(t *testing.T) {
|
||||
@@ -197,111 +194,3 @@ func TestMergeSortBlocks(t *testing.T) {
|
||||
Values: []float64{7, 24, 26},
|
||||
})
|
||||
}
|
||||
|
||||
func TestEqualSamplesPrefix(t *testing.T) {
|
||||
f := func(a, b *sortBlock, expected int) {
|
||||
t.Helper()
|
||||
|
||||
actual := equalSamplesPrefix(a, b)
|
||||
if actual != expected {
|
||||
t.Fatalf("unexpected result: got %d, want %d", actual, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// Empty blocks
|
||||
f(&sortBlock{}, &sortBlock{}, 0)
|
||||
|
||||
// Identical blocks
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, 6, 7, 8},
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, 6, 7, 8},
|
||||
}, 4)
|
||||
|
||||
// Non-zero NextIdx
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, 6, 7, 8},
|
||||
NextIdx: 2,
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{10, 20, 3, 4},
|
||||
Values: []float64{50, 60, 7, 8},
|
||||
NextIdx: 2,
|
||||
}, 2)
|
||||
|
||||
// Non-zero NextIdx with mismatch
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, 6, 7, 8},
|
||||
NextIdx: 1,
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{10, 2, 3, 4},
|
||||
Values: []float64{50, 6, 7, 80},
|
||||
NextIdx: 1,
|
||||
}, 2)
|
||||
|
||||
// Different lengths
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, 6, 7, 8},
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{1, 2, 3},
|
||||
Values: []float64{5, 6, 7},
|
||||
}, 3)
|
||||
|
||||
// Timestamps diverge
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, 6, 7, 8},
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{1, 2, 30, 4},
|
||||
Values: []float64{5, 6, 7, 8},
|
||||
}, 2)
|
||||
|
||||
// Values diverge
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, 6, 7, 8},
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, 60, 7, 8},
|
||||
}, 1)
|
||||
|
||||
// Zero matches
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, 6, 7, 8},
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{5, 6, 7, 8},
|
||||
Values: []float64{1, 2, 3, 4},
|
||||
}, 0)
|
||||
|
||||
// Compare staleness markers, matching
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, decimal.StaleNaN, 7, 8},
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{5, decimal.StaleNaN, 7, 8},
|
||||
}, 4)
|
||||
|
||||
// Special float values: +Inf, -Inf, 0, -0
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{math.Inf(1), math.Inf(-1), math.Copysign(0, +1), math.Copysign(0, -1)},
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{1, 2, 3, 4},
|
||||
Values: []float64{math.Inf(1), math.Inf(-1), math.Copysign(0, +1), math.Copysign(0, -1)},
|
||||
}, 4)
|
||||
|
||||
// Positive zero vs negative zero (bitwise different)
|
||||
f(&sortBlock{
|
||||
Timestamps: []int64{1, 2},
|
||||
Values: []float64{5, math.Copysign(0, +1)},
|
||||
}, &sortBlock{
|
||||
Timestamps: []int64{1, 2},
|
||||
Values: []float64{5, math.Copysign(0, -1)},
|
||||
}, 1)
|
||||
}
|
||||
|
||||
@@ -10,14 +10,14 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
|
||||
b.Run(fmt.Sprintf("replicationFactor-%d", replicationFactor), func(b *testing.B) {
|
||||
const samplesPerBlock = 8192
|
||||
var blocks []*sortBlock
|
||||
for j := range 10 {
|
||||
for j := 0; j < 10; j++ {
|
||||
timestamps := make([]int64, samplesPerBlock)
|
||||
values := make([]float64, samplesPerBlock)
|
||||
for i := range timestamps {
|
||||
timestamps[i] = int64(j*samplesPerBlock + i)
|
||||
values[i] = float64(j*samplesPerBlock + i)
|
||||
}
|
||||
for range replicationFactor {
|
||||
for i := 0; i < replicationFactor; i++ {
|
||||
blocks = append(blocks, &sortBlock{
|
||||
Timestamps: timestamps,
|
||||
Values: values,
|
||||
@@ -30,7 +30,7 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
|
||||
b.Run("overlapped-blocks-bestcase", func(b *testing.B) {
|
||||
const samplesPerBlock = 8192
|
||||
var blocks []*sortBlock
|
||||
for j := range 10 {
|
||||
for j := 0; j < 10; j++ {
|
||||
timestamps := make([]int64, samplesPerBlock)
|
||||
values := make([]float64, samplesPerBlock)
|
||||
for i := range timestamps {
|
||||
@@ -45,7 +45,7 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
|
||||
for j := 1; j < len(blocks); j++ {
|
||||
prev := blocks[j-1].Timestamps
|
||||
curr := blocks[j].Timestamps
|
||||
for i := range samplesPerBlock / 2 {
|
||||
for i := 0; i < samplesPerBlock/2; i++ {
|
||||
prev[i+samplesPerBlock/2], curr[i] = curr[i], prev[i+samplesPerBlock/2]
|
||||
}
|
||||
}
|
||||
@@ -54,7 +54,7 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
|
||||
b.Run("overlapped-blocks-worstcase", func(b *testing.B) {
|
||||
const samplesPerBlock = 8192
|
||||
var blocks []*sortBlock
|
||||
for j := range 5 {
|
||||
for j := 0; j < 5; j++ {
|
||||
timestamps := make([]int64, samplesPerBlock)
|
||||
values := make([]float64, samplesPerBlock)
|
||||
for i := range timestamps {
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"math"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -1005,7 +1004,14 @@ func removeEmptyValuesAndTimeseries(tss []netstorage.Result) []netstorage.Result
|
||||
dst := tss[:0]
|
||||
for i := range tss {
|
||||
ts := &tss[i]
|
||||
if !slices.ContainsFunc(ts.Values, math.IsNaN) {
|
||||
hasNaNs := false
|
||||
for _, v := range ts.Values {
|
||||
if math.IsNaN(v) {
|
||||
hasNaNs = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasNaNs {
|
||||
// Fast path: nothing to remove.
|
||||
if len(ts.Values) > 0 {
|
||||
dst = append(dst, *ts)
|
||||
|
||||
@@ -742,7 +742,7 @@ func getRangeTopKTimeseries(tss []*timeseries, modifier *metricsql.ModifierExpr,
|
||||
|
||||
func reverseSeries(tss []*timeseries) {
|
||||
j := len(tss)
|
||||
for i := range len(tss) / 2 {
|
||||
for i := 0; i < len(tss)/2; i++ {
|
||||
j--
|
||||
tss[i], tss[j] = tss[j], tss[i]
|
||||
}
|
||||
@@ -983,7 +983,7 @@ func getPerPointIQRBounds(tss []*timeseries) ([]float64, []float64) {
|
||||
var qs []float64
|
||||
lower := make([]float64, pointsLen)
|
||||
upper := make([]float64, pointsLen)
|
||||
for i := range pointsLen {
|
||||
for i := 0; i < pointsLen; i++ {
|
||||
values = values[:0]
|
||||
for _, ts := range tss {
|
||||
v := ts.Values[i]
|
||||
|
||||
@@ -53,7 +53,7 @@ func TestIncrementalAggr(t *testing.T) {
|
||||
Values: valuesExpected,
|
||||
}}
|
||||
// run the test multiple times to make sure there are no side effects on concurrency
|
||||
for i := range 10 {
|
||||
for i := 0; i < 10; i++ {
|
||||
iafc := newIncrementalAggrFuncContext(ae, callbacks)
|
||||
tssSrcCopy := copyTimeseries(tssSrc)
|
||||
if err := testIncrementalParallelAggr(iafc, tssSrcCopy, tssExpected); err != nil {
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -1714,7 +1713,6 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
|
||||
return nil, err
|
||||
}
|
||||
defer rml.Put(uint64(rollupMemorySize))
|
||||
qs.addMemoryUsage(rollupMemorySize)
|
||||
qt.Printf("the rollup evaluation needs an estimated %d bytes of RAM for %d series and %d points per series (summary %d points)",
|
||||
rollupMemorySize, timeseriesLen, pointsPerSeries, rollupPoints)
|
||||
|
||||
@@ -1936,7 +1934,14 @@ func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]flo
|
||||
return values, timestamps
|
||||
}
|
||||
// Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values.
|
||||
if !slices.ContainsFunc(values, decimal.IsStaleNaN) {
|
||||
hasStaleSamples := false
|
||||
for _, v := range values {
|
||||
if decimal.IsStaleNaN(v) {
|
||||
hasStaleSamples = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasStaleSamples {
|
||||
// Fast path: values have no Prometheus staleness marks.
|
||||
return values, timestamps
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly boo
|
||||
if querystats.Enabled() {
|
||||
startTime := time.Now()
|
||||
defer func() {
|
||||
querystats.RegisterQuery(q, ec.End-ec.Start, startTime, ec.QueryStats.memoryUsage())
|
||||
querystats.RegisterQuery(q, ec.End-ec.Start, startTime)
|
||||
ec.QueryStats.addExecutionTimeMsec(startTime)
|
||||
}()
|
||||
}
|
||||
@@ -313,7 +313,7 @@ func escapeDots(s string) string {
|
||||
return s
|
||||
}
|
||||
result := make([]byte, 0, len(s)+2*dotsCount)
|
||||
for i := range len(s) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '.' && (i == 0 || s[i-1] != '\\') && (i+1 == len(s) || i+1 < len(s) && s[i+1] != '*' && s[i+1] != '+' && s[i+1] != '{') {
|
||||
// Escape a dot if the following conditions are met:
|
||||
// - if it isn't escaped already, i.e. if there is no `\` char before the dot.
|
||||
|
||||
@@ -67,7 +67,7 @@ func TestExecSuccess(t *testing.T) {
|
||||
Deadline: searchutil.NewDeadline(time.Now(), time.Minute, ""),
|
||||
RoundDigits: 100,
|
||||
}
|
||||
for range 5 {
|
||||
for i := 0; i < 5; i++ {
|
||||
result, err := Exec(nil, ec, q, false)
|
||||
if err != nil {
|
||||
t.Fatalf(`unexpected error when executing %q: %s`, q, err)
|
||||
@@ -9827,7 +9827,7 @@ func TestExecError(t *testing.T) {
|
||||
Deadline: searchutil.NewDeadline(time.Now(), time.Minute, ""),
|
||||
RoundDigits: 100,
|
||||
}
|
||||
for range 4 {
|
||||
for i := 0; i < 4; i++ {
|
||||
rv, err := Exec(nil, ec, q, false)
|
||||
if err == nil {
|
||||
t.Fatalf(`expecting non-nil error on %q`, q)
|
||||
|
||||
@@ -55,7 +55,7 @@ type parseCache struct {
|
||||
|
||||
func newParseCache() *parseCache {
|
||||
pc := new(parseCache)
|
||||
for i := range parseBucketCount {
|
||||
for i := 0; i < parseBucketCount; i++ {
|
||||
pc.buckets[i] = newParseBucket()
|
||||
}
|
||||
return pc
|
||||
@@ -75,7 +75,7 @@ func (pc *parseCache) get(q string) *parseCacheValue {
|
||||
|
||||
func (pc *parseCache) requests() uint64 {
|
||||
var n uint64
|
||||
for i := range parseBucketCount {
|
||||
for i := 0; i < parseBucketCount; i++ {
|
||||
n += pc.buckets[i].requests.Load()
|
||||
}
|
||||
return n
|
||||
@@ -83,7 +83,7 @@ func (pc *parseCache) requests() uint64 {
|
||||
|
||||
func (pc *parseCache) misses() uint64 {
|
||||
var n uint64
|
||||
for i := range parseBucketCount {
|
||||
for i := 0; i < parseBucketCount; i++ {
|
||||
n += pc.buckets[i].misses.Load()
|
||||
}
|
||||
return n
|
||||
@@ -91,7 +91,7 @@ func (pc *parseCache) misses() uint64 {
|
||||
|
||||
func (pc *parseCache) len() uint64 {
|
||||
var n uint64
|
||||
for i := range parseBucketCount {
|
||||
for i := 0; i < parseBucketCount; i++ {
|
||||
n += pc.buckets[i].len()
|
||||
}
|
||||
return n
|
||||
|
||||
@@ -17,7 +17,7 @@ func testGetParseCacheValue(q string) *parseCacheValue {
|
||||
|
||||
func testGenerateQueries(items int) []string {
|
||||
queries := make([]string, items)
|
||||
for i := range items {
|
||||
for i := 0; i < items; i++ {
|
||||
queries[i] = fmt.Sprintf(`node_time_seconds{instance="node%d", job="job%d"}`, i, i)
|
||||
}
|
||||
return queries
|
||||
@@ -102,7 +102,7 @@ func TestParseCacheBucketOverflow(t *testing.T) {
|
||||
v := testGetParseCacheValue(queries[0])
|
||||
|
||||
// Fill bucket
|
||||
for i := range parseBucketMaxLen {
|
||||
for i := 0; i < parseBucketMaxLen; i++ {
|
||||
b.put(queries[i], v)
|
||||
}
|
||||
expectedLen = uint64(parseBucketMaxLen)
|
||||
|
||||
@@ -15,7 +15,7 @@ func BenchmarkCachePutNoOverFlow(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := range items {
|
||||
for i := 0; i < items; i++ {
|
||||
pc.put(queries[i], v)
|
||||
}
|
||||
}
|
||||
@@ -32,14 +32,14 @@ func BenchmarkCacheGetNoOverflow(b *testing.B) {
|
||||
queries := testGenerateQueries(items)
|
||||
v := testGetParseCacheValue(queries[0])
|
||||
|
||||
for i := range queries {
|
||||
for i := 0; i < len(queries); i++ {
|
||||
pc.put(queries[i], v)
|
||||
}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := range items {
|
||||
for i := 0; i < items; i++ {
|
||||
if v := pc.get(queries[i]); v == nil {
|
||||
b.Errorf("unexpected nil value obtained from cache for query: %s ", queries[i])
|
||||
}
|
||||
@@ -59,7 +59,7 @@ func BenchmarkCachePutGetNoOverflow(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := range items {
|
||||
for i := 0; i < items; i++ {
|
||||
pc.put(queries[i], v)
|
||||
if res := pc.get(queries[i]); res == nil {
|
||||
b.Errorf("unexpected nil value obtained from cache for query: %s ", queries[i])
|
||||
@@ -79,7 +79,7 @@ func BenchmarkCachePutOverflow(b *testing.B) {
|
||||
queries := testGenerateQueries(items)
|
||||
v := testGetParseCacheValue(queries[0])
|
||||
|
||||
for i := range parseCacheMaxLen {
|
||||
for i := 0; i < parseCacheMaxLen; i++ {
|
||||
c.put(queries[i], v)
|
||||
}
|
||||
|
||||
@@ -105,7 +105,7 @@ func BenchmarkCachePutGetOverflow(b *testing.B) {
|
||||
queries := testGenerateQueries(items)
|
||||
v := testGetParseCacheValue(queries[0])
|
||||
|
||||
for i := range parseCacheMaxLen {
|
||||
for i := 0; i < parseCacheMaxLen; i++ {
|
||||
c.put(queries[i], v)
|
||||
}
|
||||
|
||||
@@ -141,8 +141,8 @@ var testSimpleQueries = []string{
|
||||
|
||||
func BenchmarkParsePromQLWithCacheSimple(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for range b.N {
|
||||
for j := range testSimpleQueries {
|
||||
for i := 0; i < b.N; i++ {
|
||||
for j := 0; j < len(testSimpleQueries); j++ {
|
||||
_, err := parsePromQLWithCache(testSimpleQueries[j])
|
||||
if err != nil {
|
||||
b.Errorf("unexpected error: %s", err)
|
||||
@@ -155,7 +155,7 @@ func BenchmarkParsePromQLWithCacheSimpleParallel(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := range testSimpleQueries {
|
||||
for i := 0; i < len(testSimpleQueries); i++ {
|
||||
_, err := parsePromQLWithCache(testSimpleQueries[i])
|
||||
if err != nil {
|
||||
b.Errorf("unexpected error: %s", err)
|
||||
@@ -210,8 +210,8 @@ var testComplexQueries = []string{
|
||||
|
||||
func BenchmarkParsePromQLWithCacheComplex(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for range b.N {
|
||||
for j := range testComplexQueries {
|
||||
for i := 0; i < b.N; i++ {
|
||||
for j := 0; j < len(testComplexQueries); j++ {
|
||||
_, err := parsePromQLWithCache(testComplexQueries[j])
|
||||
if err != nil {
|
||||
b.Errorf("unexpected error: %s", err)
|
||||
@@ -224,7 +224,7 @@ func BenchmarkParsePromQLWithCacheComplexParallel(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for i := range testComplexQueries {
|
||||
for i := 0; i < len(testComplexQueries); i++ {
|
||||
_, err := parsePromQLWithCache(testComplexQueries[i])
|
||||
if err != nil {
|
||||
b.Errorf("unexpected error: %s", err)
|
||||
|
||||
@@ -13,8 +13,6 @@ type QueryStats struct {
|
||||
ExecutionDuration atomic.Pointer[time.Duration]
|
||||
// SeriesFetched contains the number of series fetched from storage or cache.
|
||||
SeriesFetched atomic.Int64
|
||||
// MemoryUsage contains the estimated memory consumption of the query
|
||||
MemoryUsage atomic.Int64
|
||||
|
||||
at *auth.Token
|
||||
|
||||
@@ -55,17 +53,3 @@ func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
|
||||
d := time.Since(startTime)
|
||||
qs.ExecutionDuration.Store(&d)
|
||||
}
|
||||
|
||||
func (qs *QueryStats) addMemoryUsage(memoryUsage int64) {
|
||||
if qs == nil {
|
||||
return
|
||||
}
|
||||
qs.MemoryUsage.Store(memoryUsage)
|
||||
}
|
||||
|
||||
func (qs *QueryStats) memoryUsage() int64 {
|
||||
if qs == nil {
|
||||
return 0
|
||||
}
|
||||
return qs.MemoryUsage.Load()
|
||||
}
|
||||
|
||||
@@ -739,7 +739,7 @@ func (mi *rollupResultCacheMetainfo) Unmarshal(src []byte) error {
|
||||
entriesLen := int(encoding.UnmarshalUint32(src))
|
||||
src = src[4:]
|
||||
mi.entries = slicesutil.SetLength(mi.entries, entriesLen)
|
||||
for i := range entriesLen {
|
||||
for i := 0; i < entriesLen; i++ {
|
||||
tail, err := mi.entries[i].Unmarshal(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot unmarshal entry #%d: %w", i, err)
|
||||
|
||||
@@ -11,14 +11,14 @@ import (
|
||||
|
||||
func TestRollupResultCacheInitStop(t *testing.T) {
|
||||
t.Run("inmemory", func(_ *testing.T) {
|
||||
for range 5 {
|
||||
for i := 0; i < 5; i++ {
|
||||
InitRollupResultCache("")
|
||||
StopRollupResultCache()
|
||||
}
|
||||
})
|
||||
t.Run("file-based", func(_ *testing.T) {
|
||||
cacheFilePath := "test-rollup-result-cache"
|
||||
for range 3 {
|
||||
for i := 0; i < 3; i++ {
|
||||
InitRollupResultCache(cacheFilePath)
|
||||
StopRollupResultCache()
|
||||
}
|
||||
@@ -241,12 +241,12 @@ func TestRollupResultCache(t *testing.T) {
|
||||
t.Run("big-timeseries", func(t *testing.T) {
|
||||
ResetRollupResultCache()
|
||||
var tss []*timeseries
|
||||
for i := range 1000 {
|
||||
for i := 0; i < 1000; i++ {
|
||||
ts := ×eries{
|
||||
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
||||
Values: []float64{1, 2, 3, 4, 5, 6},
|
||||
}
|
||||
ts.MetricName.MetricGroup = fmt.Appendf(nil, "metric %d", i)
|
||||
ts.MetricName.MetricGroup = []byte(fmt.Sprintf("metric %d", i))
|
||||
tss = append(tss, ts)
|
||||
}
|
||||
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
||||
|
||||
@@ -240,7 +240,7 @@ func testRollupFunc(t *testing.T, funcName string, args []any, vExpected float64
|
||||
if rollupFuncsRemoveCounterResets[funcName] {
|
||||
removeCounterResets(rfa.values, rfa.timestamps, 0)
|
||||
}
|
||||
for range 5 {
|
||||
for i := 0; i < 5; i++ {
|
||||
v := rf(&rfa)
|
||||
if math.IsNaN(vExpected) {
|
||||
if !math.IsNaN(v) {
|
||||
@@ -1493,7 +1493,7 @@ func TestRollupBigNumberOfValues(t *testing.T) {
|
||||
rc.Timestamps = rc.getTimestamps()
|
||||
srcValues := make([]float64, srcValuesCount)
|
||||
srcTimestamps := make([]int64, srcValuesCount)
|
||||
for i := range int(srcValuesCount) {
|
||||
for i := 0; i < srcValuesCount; i++ {
|
||||
srcValues[i] = float64(i)
|
||||
srcTimestamps[i] = int64(i / 2)
|
||||
}
|
||||
|
||||
@@ -451,7 +451,7 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
sort.Slice(leGroup, func(i, j int) bool {
|
||||
return leGroup[i].le < leGroup[j].le
|
||||
})
|
||||
for n := range pointsCount {
|
||||
for n := 0; n < pointsCount; n++ {
|
||||
prevValue := float64(0)
|
||||
for i := range leGroup {
|
||||
xx := &leGroup[i]
|
||||
@@ -1192,7 +1192,7 @@ func transformInterpolate(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
}
|
||||
prevValue := nan
|
||||
var nextValue float64
|
||||
for i := range values {
|
||||
for i := 0; i < len(values); i++ {
|
||||
if !math.IsNaN(values[i]) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package promql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -280,6 +281,87 @@ func timeseriesToPromMetrics(tss []*timeseries) string {
|
||||
return strings.Join(a, "\n")
|
||||
}
|
||||
|
||||
func TestTransformFuncSort(t *testing.T) {
|
||||
f := func(isDesc bool, metrics, expectedMetrics string) {
|
||||
t.Helper()
|
||||
|
||||
tss := promMetricsToTimeseries(metrics)
|
||||
|
||||
// Input tss order is not stable in VictoriaMetrics
|
||||
// Shuffle tss to reflect that
|
||||
// Commenting out the shuffle to make the test stable
|
||||
rand.Shuffle(len(tss), func(i, j int) {
|
||||
tss[i], tss[j] = tss[j], tss[i]
|
||||
})
|
||||
|
||||
sortFunc := newTransformFuncSort(isDesc)
|
||||
sorted, err := sortFunc(&transformFuncArg{
|
||||
args: [][]*timeseries{tss},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("sort failed: %s", err)
|
||||
}
|
||||
|
||||
result := timeseriesToPromMetrics(sorted)
|
||||
if result != expectedMetrics {
|
||||
t.Fatalf("unexpected sort result:\ngot\n%s\nwant\n%s", result, expectedMetrics)
|
||||
}
|
||||
}
|
||||
|
||||
// Test asc sort with different values
|
||||
f(
|
||||
false,
|
||||
`foo{label="a"} 3 123
|
||||
foo{label="b"} 2 123
|
||||
foo{label="c"} 1 123`,
|
||||
`foo{label="c"} 1 123
|
||||
foo{label="b"} 2 123
|
||||
foo{label="a"} 3 123`,
|
||||
)
|
||||
|
||||
// Test desc sort with different values
|
||||
f(
|
||||
true,
|
||||
`foo{label="a"} 3 123
|
||||
foo{label="b"} 2 123
|
||||
foo{label="c"} 1 123`,
|
||||
`foo{label="a"} 3 123
|
||||
foo{label="b"} 2 123
|
||||
foo{label="c"} 1 123`,
|
||||
)
|
||||
|
||||
// Test asc sort with mixed values
|
||||
f(
|
||||
false,
|
||||
`foo{label="a"} 1 123
|
||||
foo{label="b"} 1 123
|
||||
foo{label="c"} 2 123
|
||||
foo{label="d"} 2 123
|
||||
foo{label="e"} 3 123
|
||||
`,
|
||||
`foo{label="a"} 1 123
|
||||
foo{label="b"} 1 123
|
||||
foo{label="c"} 2 123
|
||||
foo{label="d"} 2 123
|
||||
foo{label="e"} 3 123`,
|
||||
)
|
||||
|
||||
// Test desc sort with mixed values
|
||||
f(
|
||||
true,
|
||||
`foo{label="a"} 1 123
|
||||
foo{label="b"} 1 123
|
||||
foo{label="c"} 2 123
|
||||
foo{label="d"} 2 123
|
||||
foo{label="e"} 3 123`,
|
||||
`foo{label="e"} 3 123
|
||||
foo{label="c"} 2 123
|
||||
foo{label="d"} 2 123
|
||||
foo{label="a"} 1 123
|
||||
foo{label="b"} 1 123`,
|
||||
)
|
||||
}
|
||||
|
||||
func TestGetNumPrefix(t *testing.T) {
|
||||
f := func(s, prefixExpected string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
|
||||
)
|
||||
@@ -16,8 +15,7 @@ import (
|
||||
var (
|
||||
lastQueriesCount = flag.Int("search.queryStats.lastQueriesCount", 20000, "Query stats for /api/v1/status/top_queries is tracked on this number of last queries. "+
|
||||
"Zero value disables query stats tracking")
|
||||
minQueryDuration = flag.Duration("search.queryStats.minQueryDuration", time.Millisecond, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats")
|
||||
minQueryMemoryUsage = flagutil.NewBytes("search.queryStats.minQueryMemoryUsage", 1024, "The minimum memory bytes consumption for queries to track in query stats at /api/v1/status/top_queries. Queries with lower memory bytes consumption are ignored in query stats")
|
||||
minQueryDuration = flag.Duration("search.queryStats.minQueryDuration", time.Millisecond, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats")
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -33,9 +31,9 @@ func Enabled() bool {
|
||||
// RegisterQuery registers the query on the given timeRangeMsecs, which has been started at startTime.
|
||||
//
|
||||
// RegisterQuery must be called when the query is finished.
|
||||
func RegisterQuery(query string, timeRangeMsecs int64, startTime time.Time, memoryUsage int64) {
|
||||
func RegisterQuery(query string, timeRangeMsecs int64, startTime time.Time) {
|
||||
initOnce.Do(initQueryStats)
|
||||
qsTracker.registerQuery(query, timeRangeMsecs, startTime, memoryUsage)
|
||||
qsTracker.registerQuery(query, timeRangeMsecs, startTime)
|
||||
}
|
||||
|
||||
// WriteJSONQueryStats writes query stats to given writer in json format.
|
||||
@@ -56,7 +54,6 @@ type queryStatRecord struct {
|
||||
timeRangeSecs int64
|
||||
registerTime time.Time
|
||||
duration time.Duration
|
||||
memoryUsage int64
|
||||
}
|
||||
|
||||
type queryStatKey struct {
|
||||
@@ -69,8 +66,8 @@ func initQueryStats() {
|
||||
if recordsCount <= 0 {
|
||||
recordsCount = 1
|
||||
} else {
|
||||
logger.Infof("enabled query stats tracking at `/api/v1/status/top_queries` with -search.queryStats.lastQueriesCount=%d, -search.queryStats.minQueryDuration=%s, -search.queryStats.minQueryMemoryUsage=%s",
|
||||
*lastQueriesCount, *minQueryDuration, minQueryMemoryUsage)
|
||||
logger.Infof("enabled query stats tracking at `/api/v1/status/top_queries` with -search.queryStats.lastQueriesCount=%d, -search.queryStats.minQueryDuration=%s",
|
||||
*lastQueriesCount, *minQueryDuration)
|
||||
}
|
||||
qsTracker = &queryStatsTracker{
|
||||
a: make([]queryStatRecord, recordsCount),
|
||||
@@ -81,7 +78,6 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, maxLife
|
||||
fmt.Fprintf(w, `{"topN":"%d","maxLifetime":"%s",`, topN, maxLifetime)
|
||||
fmt.Fprintf(w, `"search.queryStats.lastQueriesCount":%d,`, *lastQueriesCount)
|
||||
fmt.Fprintf(w, `"search.queryStats.minQueryDuration":"%s",`, *minQueryDuration)
|
||||
fmt.Fprintf(w, `"search.queryStats.minQueryMemoryUsage":"%s",`, minQueryMemoryUsage)
|
||||
fmt.Fprintf(w, `"topByCount":[`)
|
||||
topByCount := qst.getTopByCount(topN, maxLifetime)
|
||||
for i, r := range topByCount {
|
||||
@@ -106,28 +102,15 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, maxLife
|
||||
fmt.Fprintf(w, `,`)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, `],"topByAvgMemoryUsage":[`)
|
||||
topByAvgMemoryConsumption := qst.getTopByAvgMemoryUsage(topN, maxLifetime)
|
||||
for i, r := range topByAvgMemoryConsumption {
|
||||
fmt.Fprintf(w, `{"query":%s,"timeRangeSeconds":%d,"avgMemoryBytes":%d,"count":%d}`, stringsutil.JSONString(r.query), r.timeRangeSecs, r.memoryUsage, r.count)
|
||||
if i+1 < len(topByAvgMemoryConsumption) {
|
||||
fmt.Fprintf(w, `,`)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, `]}`)
|
||||
}
|
||||
|
||||
func (qst *queryStatsTracker) registerQuery(query string, timeRangeMsecs int64, startTime time.Time, memoryUsage int64) {
|
||||
func (qst *queryStatsTracker) registerQuery(query string, timeRangeMsecs int64, startTime time.Time) {
|
||||
registerTime := time.Now()
|
||||
duration := registerTime.Sub(startTime)
|
||||
if duration < *minQueryDuration {
|
||||
return
|
||||
}
|
||||
if memoryUsage < int64(minQueryMemoryUsage.IntN()) {
|
||||
return
|
||||
}
|
||||
|
||||
qst.mu.Lock()
|
||||
defer qst.mu.Unlock()
|
||||
@@ -143,7 +126,6 @@ func (qst *queryStatsTracker) registerQuery(query string, timeRangeMsecs int64,
|
||||
r.timeRangeSecs = timeRangeMsecs / 1000
|
||||
r.registerTime = registerTime
|
||||
r.duration = duration
|
||||
r.memoryUsage = memoryUsage
|
||||
}
|
||||
|
||||
func (r *queryStatRecord) matches(currentTime time.Time, maxLifetime time.Duration) bool {
|
||||
@@ -275,47 +257,3 @@ func (qst *queryStatsTracker) getTopBySumDuration(topN int, maxLifetime time.Dur
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
type queryStatByMemory struct {
|
||||
query string
|
||||
timeRangeSecs int64
|
||||
memoryUsage int64
|
||||
count int
|
||||
}
|
||||
|
||||
func (qst *queryStatsTracker) getTopByAvgMemoryUsage(topN int, maxLifetime time.Duration) []queryStatByMemory {
|
||||
currentTime := time.Now()
|
||||
qst.mu.Lock()
|
||||
type countSum struct {
|
||||
count int
|
||||
sum int64
|
||||
}
|
||||
m := make(map[queryStatKey]countSum)
|
||||
for _, r := range qst.a {
|
||||
if r.matches(currentTime, maxLifetime) {
|
||||
k := r.key()
|
||||
ks := m[k]
|
||||
ks.count++
|
||||
ks.sum += r.memoryUsage
|
||||
m[k] = ks
|
||||
}
|
||||
}
|
||||
qst.mu.Unlock()
|
||||
|
||||
var a []queryStatByMemory
|
||||
for k, ks := range m {
|
||||
a = append(a, queryStatByMemory{
|
||||
query: k.query,
|
||||
timeRangeSecs: k.timeRangeSecs,
|
||||
memoryUsage: ks.sum / int64(ks.count),
|
||||
count: ks.count,
|
||||
})
|
||||
}
|
||||
sort.Slice(a, func(i, j int) bool {
|
||||
return a[i].memoryUsage > a[j].memoryUsage
|
||||
})
|
||||
if len(a) > topN {
|
||||
a = a[:topN]
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ aliases:
|
||||
- /MetricsQL.html
|
||||
- /metricsql/index.html
|
||||
- /metricsql/
|
||||
- /MetricsQL/
|
||||
---
|
||||
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
|
||||
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
|
||||
209
app/vmselect/vmui/assets/index-B6lol36n.js
Normal file
209
app/vmselect/vmui/assets/index-B6lol36n.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/vmselect/vmui/assets/index-VQRcNK83.css
Normal file
1
app/vmselect/vmui/assets/index-VQRcNK83.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -37,10 +37,10 @@
|
||||
<meta property="og:title" content="UI for VictoriaMetrics">
|
||||
<meta property="og:url" content="https://victoriametrics.com/">
|
||||
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
|
||||
<script type="module" crossorigin src="./assets/index-C1hTBemk.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-BR6Q0Fin.js">
|
||||
<script type="module" crossorigin src="./assets/index-B6lol36n.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-EZef-S_8.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-D7CzMv1O.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-VQRcNK83.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
||||
@@ -62,7 +62,7 @@ var (
|
||||
"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . "+
|
||||
"See also -storage.maxHourlySeries")
|
||||
|
||||
minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 100e6, "The minimum free disk space at -storageDataPath after which the storage stops accepting new data")
|
||||
minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which the storage stops accepting new data")
|
||||
|
||||
cacheSizeStorageTSID = flagutil.NewBytes("storage.cacheSizeStorageTSID", 0, "Overrides max size for storage/tsid cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
@@ -655,7 +655,6 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/metricID"}`, idbm.MetricIDCacheSize)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSize)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSize)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheSize)
|
||||
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/tsid"}`, m.TSIDCacheSizeBytes)
|
||||
@@ -671,7 +670,6 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeBytes)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeBytes)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeBytes)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheSizeBytes)
|
||||
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/tsid"}`, m.TSIDCacheSizeMaxBytes)
|
||||
@@ -683,7 +681,6 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeMaxBytes)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeMaxBytes)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeMaxBytes)
|
||||
metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheSizeMaxBytes)
|
||||
|
||||
metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
|
||||
metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/tsid"}`, m.TSIDCacheRequests)
|
||||
@@ -695,7 +692,6 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
|
||||
metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheRequests)
|
||||
metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheRequests)
|
||||
metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheRequests)
|
||||
metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheRequests)
|
||||
|
||||
metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
|
||||
metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/tsid"}`, m.TSIDCacheMisses)
|
||||
@@ -707,7 +703,6 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
|
||||
metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheMisses)
|
||||
metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheMisses)
|
||||
metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheMisses)
|
||||
metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheMisses)
|
||||
|
||||
metrics.WriteCounterUint64(w, `vm_cache_resets_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheResets)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.0 AS build-web-stage
|
||||
FROM golang:1.25.6 AS build-web-stage
|
||||
COPY build /build
|
||||
|
||||
WORKDIR /build
|
||||
@@ -6,7 +6,7 @@ COPY web/ /build/
|
||||
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
|
||||
GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
|
||||
|
||||
FROM alpine:3.23.3
|
||||
FROM alpine:3.22.2
|
||||
USER root
|
||||
|
||||
COPY --from=build-web-stage /build/web-amd64 /app/web
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
copy-metricsql-docs:
|
||||
cp docs/victoriametrics/MetricsQL.md app/vmui/packages/vmui/src/assets/MetricsQL.md
|
||||
|
||||
vmui-package-base-image:
|
||||
docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
|
||||
|
||||
vmui-run-npm-command: vmui-package-base-image
|
||||
docker run --rm \
|
||||
--user $(shell id -u):$(shell id -g) \
|
||||
|
||||
567
app/vmui/packages/vmui/package-lock.json
generated
567
app/vmui/packages/vmui/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -24,10 +24,10 @@
|
||||
"dayjs": "^1.11.19",
|
||||
"lodash.debounce": "^4.0.8",
|
||||
"marked": "^17.0.1",
|
||||
"preact": "^10.28.3",
|
||||
"preact": "^10.28.2",
|
||||
"qs": "^6.14.1",
|
||||
"react-input-mask": "^2.0.4",
|
||||
"react-router-dom": "^7.13.0",
|
||||
"react-router-dom": "^7.12.0",
|
||||
"uplot": "^1.6.32",
|
||||
"vite": "^7.3.1",
|
||||
"web-vitals": "^5.1.0"
|
||||
@@ -35,29 +35,29 @@
|
||||
"devDependencies": {
|
||||
"@eslint/eslintrc": "^3.3.3",
|
||||
"@eslint/js": "^9.39.2",
|
||||
"@preact/preset-vite": "^2.10.3",
|
||||
"@preact/preset-vite": "^2.10.2",
|
||||
"@testing-library/jest-dom": "^6.9.1",
|
||||
"@testing-library/preact": "^3.2.4",
|
||||
"@types/lodash.debounce": "^4.0.9",
|
||||
"@types/node": "^25.2.0",
|
||||
"@types/node": "^25.0.8",
|
||||
"@types/qs": "^6.14.0",
|
||||
"@types/react": "^19.2.10",
|
||||
"@types/react": "^19.2.8",
|
||||
"@types/react-input-mask": "^3.0.6",
|
||||
"@types/react-router-dom": "^5.3.3",
|
||||
"@typescript-eslint/eslint-plugin": "^8.54.0",
|
||||
"@typescript-eslint/parser": "^8.54.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.53.0",
|
||||
"@typescript-eslint/parser": "^8.53.0",
|
||||
"cross-env": "^10.1.0",
|
||||
"eslint": "^9.39.2",
|
||||
"eslint-plugin-react": "^7.37.5",
|
||||
"eslint-plugin-unused-imports": "^4.3.0",
|
||||
"globals": "^17.3.0",
|
||||
"globals": "^17.0.0",
|
||||
"http-proxy-middleware": "^3.0.5",
|
||||
"jsdom": "^28.0.0",
|
||||
"jsdom": "^27.4.0",
|
||||
"postcss": "^8.5.6",
|
||||
"rollup-plugin-visualizer": "^6.0.5",
|
||||
"sass-embedded": "^1.97.3",
|
||||
"sass-embedded": "^1.97.2",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.0.18"
|
||||
"vitest": "^4.0.17"
|
||||
},
|
||||
"browserslist": {
|
||||
"production": [
|
||||
|
||||
@@ -12,7 +12,6 @@ aliases:
|
||||
- /MetricsQL.html
|
||||
- /metricsql/index.html
|
||||
- /metricsql/
|
||||
- /MetricsQL/
|
||||
---
|
||||
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
|
||||
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
|
||||
|
||||
@@ -6,8 +6,6 @@ import { QueryContextType } from "../../../types";
|
||||
import { AUTOCOMPLETE_LIMITS } from "../../../constants/queryAutocomplete";
|
||||
import { QueryEditorAutocompleteProps } from "./QueryEditor";
|
||||
import { getExprLastPart, getValueByContext, getContext } from "./autocompleteUtils";
|
||||
import { extractCurrentLabel, extractLabelMatchers, extractMetric, splitByCursor } from "./utils/parser";
|
||||
import { escapeLabelName } from "../../../utils/metric";
|
||||
|
||||
const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
value,
|
||||
@@ -22,39 +20,45 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
const metricsqlFunctions = useGetMetricsQL(includeFunctions);
|
||||
|
||||
const values = useMemo(() => {
|
||||
return splitByCursor(value, caretPosition);
|
||||
if (caretPosition[0] !== caretPosition[1]) return { beforeCursor: value, afterCursor: "" };
|
||||
const beforeCursor = value.substring(0, caretPosition[0]);
|
||||
const afterCursor = value.substring(caretPosition[1]);
|
||||
return { beforeCursor, afterCursor };
|
||||
}, [value, caretPosition]);
|
||||
|
||||
const exprLastPart = useMemo(() => {
|
||||
return getExprLastPart(values.beforeCursor);
|
||||
}, [values.beforeCursor]);
|
||||
const exprLastPart = useMemo(() => getExprLastPart(values.beforeCursor), [values]);
|
||||
|
||||
const metric = useMemo(() => {
|
||||
return extractMetric(exprLastPart);
|
||||
const regex1 = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
|
||||
const matchAlt = [...exprLastPart.matchAll(regex1)];
|
||||
if (matchAlt.length > 0 && matchAlt[0].groups && matchAlt[0].groups.metricName) {
|
||||
return matchAlt[0].groups.metricName;
|
||||
}
|
||||
|
||||
const regex2 = /^\s*\b(?<metricName>[^{}(),\s]+)(?={|$)/g;
|
||||
const match = [...exprLastPart.matchAll(regex2)];
|
||||
if (match.length > 0 && match[0].groups && match[0].groups.metricName) {
|
||||
return match[0].groups.metricName;
|
||||
}
|
||||
|
||||
return "";
|
||||
}, [exprLastPart]);
|
||||
|
||||
const label = useMemo(() => {
|
||||
return extractCurrentLabel(exprLastPart);
|
||||
const regexp = /[a-z_:-][\w\-.:/]*\b(?=\s*(=|!=|=~|!~))/g;
|
||||
const match = exprLastPart.match(regexp);
|
||||
return match ? match[match.length - 1] : "";
|
||||
}, [exprLastPart]);
|
||||
|
||||
const context = useMemo(() => {
|
||||
return getContext(values.beforeCursor, metric, label);
|
||||
}, [values.beforeCursor, metric, label]);
|
||||
const context = useMemo(() => getContext(values.beforeCursor, metric, label), [values, metric, label]);
|
||||
|
||||
const valueByContext = useMemo(() => {
|
||||
return getValueByContext(values.beforeCursor);
|
||||
}, [values.beforeCursor]);
|
||||
|
||||
const labelMatchers = useMemo(() => {
|
||||
return extractLabelMatchers(values.beforeCursor, label);
|
||||
}, [values.beforeCursor, label]);
|
||||
const valueByContext = useMemo(() => getValueByContext(values.beforeCursor), [values.beforeCursor]);
|
||||
|
||||
const { metrics, labels, labelValues, loading } = useFetchQueryOptions({
|
||||
valueByContext,
|
||||
metric,
|
||||
label,
|
||||
context,
|
||||
labelMatchers,
|
||||
});
|
||||
|
||||
const options = useMemo(() => {
|
||||
@@ -68,18 +72,18 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
}, [context, metrics, labels, labelValues, metricsqlFunctions]);
|
||||
}, [context, metrics, labels, labelValues]);
|
||||
|
||||
const handleSelect = useCallback((insert: string) => {
|
||||
// Find the start and end of valueByContext in the query string
|
||||
const beforeCursor = values.beforeCursor;
|
||||
const value = values.beforeCursor;
|
||||
let valueAfterCursor = values.afterCursor;
|
||||
const startIndexOfValueByContext = beforeCursor.lastIndexOf(valueByContext, caretPosition[0]);
|
||||
const startIndexOfValueByContext = value.lastIndexOf(valueByContext, caretPosition[0]);
|
||||
const endIndexOfValueByContext = startIndexOfValueByContext + valueByContext.length;
|
||||
|
||||
// Split the original string into parts: before, during, and after valueByContext
|
||||
const beforeValueByContext = beforeCursor.substring(0, startIndexOfValueByContext);
|
||||
const afterValueByContext = beforeCursor.substring(endIndexOfValueByContext);
|
||||
const beforeValueByContext = value.substring(0, startIndexOfValueByContext);
|
||||
const afterValueByContext = value.substring(endIndexOfValueByContext);
|
||||
|
||||
// Add quotes around the value if the context is labelValue
|
||||
if (context === QueryContextType.labelValue) {
|
||||
@@ -91,7 +95,6 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
}
|
||||
|
||||
if (context === QueryContextType.label) {
|
||||
insert = escapeLabelName(insert);
|
||||
valueAfterCursor = valueAfterCursor.replace(/^[^\s=!,{}()"|+\-/*^]*/, "");
|
||||
}
|
||||
|
||||
@@ -101,7 +104,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
// Assemble the new value with the inserted text
|
||||
const newVal = `${beforeValueByContext}${insert}${afterValueByContext}${valueAfterCursor}`;
|
||||
onSelect(newVal, beforeValueByContext.length + insert.length);
|
||||
}, [values.beforeCursor, values.afterCursor, valueByContext, caretPosition, context, onSelect]);
|
||||
}, [values]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!anchorEl.current) {
|
||||
@@ -139,7 +142,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
|
||||
span.remove();
|
||||
marker.remove();
|
||||
}, [anchorEl, caretPosition, hasHelperText, values.beforeCursor, values.afterCursor]);
|
||||
}, [anchorEl, caretPosition, hasHelperText]);
|
||||
|
||||
return (
|
||||
<>
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
splitByCursor,
|
||||
extractMetric,
|
||||
extractCurrentLabel,
|
||||
extractLabelMatchers,
|
||||
} from "./parser";
|
||||
|
||||
describe("splitByCursor", () => {
|
||||
it("splits by caret when selection is collapsed", () => {
|
||||
const res = splitByCursor("abcdef", [2, 2]);
|
||||
expect(res).toEqual({ beforeCursor: "ab", afterCursor: "cdef" });
|
||||
});
|
||||
|
||||
it("returns whole value as beforeCursor when selection is not collapsed", () => {
|
||||
const res = splitByCursor("abcdef", [1, 3]);
|
||||
expect(res).toEqual({ beforeCursor: "abcdef", afterCursor: "" });
|
||||
});
|
||||
|
||||
it("handles caret at 0", () => {
|
||||
const res = splitByCursor("abc", [0, 0]);
|
||||
expect(res).toEqual({ beforeCursor: "", afterCursor: "abc" });
|
||||
});
|
||||
|
||||
it("handles caret at end", () => {
|
||||
const res = splitByCursor("abc", [3, 3]);
|
||||
expect(res).toEqual({ beforeCursor: "abc", afterCursor: "" });
|
||||
});
|
||||
|
||||
it("treats reversed selection as non-collapsed (browser may return [end,start])", () => {
|
||||
const res = splitByCursor("abcdef", [4, 2]);
|
||||
expect(res).toEqual({ beforeCursor: "abcdef", afterCursor: "" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractMetric", () => {
|
||||
it("extracts metric from plain selector", () => {
|
||||
expect(extractMetric("kube_pod_info{job=\"x\"}")).toBe("kube_pod_info");
|
||||
});
|
||||
|
||||
it("extracts metric from plain expr with leading spaces", () => {
|
||||
expect(extractMetric(" http_requests_total")).toBe("http_requests_total");
|
||||
});
|
||||
|
||||
it("extracts metric from expr with braces right after metric", () => {
|
||||
expect(extractMetric("foo_bar{a=\"b\"}")).toBe("foo_bar");
|
||||
});
|
||||
|
||||
it("extracts metric before grouping modifiers (by/without/on/ignoring)", () => {
|
||||
expect(extractMetric("sum(kube_pod_info) by (pod)")).toBe("kube_pod_info");
|
||||
expect(extractMetric("sum(kube_pod_info) without (pod)")).toBe("kube_pod_info");
|
||||
expect(extractMetric("sum(kube_pod_info) on (pod)")).toBe("kube_pod_info");
|
||||
expect(extractMetric("sum(kube_pod_info) ignoring (pod)")).toBe("kube_pod_info");
|
||||
});
|
||||
|
||||
it("returns empty string when no metric found", () => {
|
||||
expect(extractMetric("{job=\"x\"}")).toBe("");
|
||||
expect(extractMetric("")).toBe("");
|
||||
expect(extractMetric("()")).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractCurrentLabel", () => {
|
||||
it("returns last label before operator", () => {
|
||||
expect(extractCurrentLabel("metric{job=\"foo\", instance=\"bar\"}")).toBe(
|
||||
"instance"
|
||||
);
|
||||
});
|
||||
|
||||
it("supports spaces around operator", () => {
|
||||
expect(extractCurrentLabel("metric{job=\"foo\", instance = \"bar\"}")).toBe(
|
||||
"instance"
|
||||
);
|
||||
});
|
||||
|
||||
it("supports regexp operators", () => {
|
||||
expect(extractCurrentLabel("metric{pod=~\"api-.*\",namespace=\"dev\"}")).toBe(
|
||||
"namespace"
|
||||
);
|
||||
});
|
||||
|
||||
it("supports label chars : - . /", () => {
|
||||
expect(extractCurrentLabel("m{foo-bar.baz/qux=\"1\"}")).toBe("foo-bar.baz/qux");
|
||||
});
|
||||
|
||||
it("returns empty string when no label pattern", () => {
|
||||
expect(extractCurrentLabel("metric{}").trim()).toBe("");
|
||||
expect(extractCurrentLabel("metric")).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractLabelMatchers", () => {
|
||||
it("returns all matchers (quoted only)", () => {
|
||||
const expr = "metric{job=\"foo\", instance=\"bar\"}";
|
||||
expect(extractLabelMatchers(expr)).toEqual(["job=\"foo\"", "instance=\"bar\""]);
|
||||
});
|
||||
|
||||
it("keeps original spacing", () => {
|
||||
const expr = "metric{ job = \"foo\" , instance = \"bar\" }";
|
||||
expect(extractLabelMatchers(expr)).toEqual(["job = \"foo\"", "instance = \"bar\""]);
|
||||
});
|
||||
|
||||
it("supports !=, =~, !~", () => {
|
||||
const expr = "m{env!=\"prod\",pod=~\"api-.*\",zone!~\"eu-.*\"}";
|
||||
expect(extractLabelMatchers(expr)).toEqual([
|
||||
"env!=\"prod\"",
|
||||
"pod=~\"api-.*\"",
|
||||
"zone!~\"eu-.*\"",
|
||||
]);
|
||||
});
|
||||
|
||||
it("excludes only the specified currentLabel matcher (exact label, not prefix)", () => {
|
||||
const expr = "m{job=\"foo\", instance=\"bar\", pod=~\"api-.*\"}";
|
||||
expect(extractLabelMatchers(expr, "instance")).toEqual([
|
||||
"job=\"foo\"",
|
||||
"pod=~\"api-.*\"",
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not exclude other labels that share a prefix with currentLabel", () => {
|
||||
const expr = "m{instance=\"bar\", insight=\"x\"}";
|
||||
expect(extractLabelMatchers(expr, "insight")).toEqual(["instance=\"bar\""]);
|
||||
});
|
||||
|
||||
it("excludes currentLabel matcher even with spaces around operator", () => {
|
||||
const expr = "m{job=\"foo\", instance = \"bar\"}";
|
||||
expect(extractLabelMatchers(expr, "instance")).toEqual(["job=\"foo\""]);
|
||||
});
|
||||
|
||||
it("returns [] when no matchers", () => {
|
||||
expect(extractLabelMatchers("m{}")).toEqual([]);
|
||||
expect(extractLabelMatchers("m")).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not include unclosed quotes", () => {
|
||||
const expr = "m{job=\"foo\", instance=\"ba";
|
||||
expect(extractLabelMatchers(expr)).toEqual(["job=\"foo\""]);
|
||||
});
|
||||
});
|
||||
@@ -1,52 +0,0 @@
|
||||
|
||||
export const splitByCursor = (
|
||||
value: string,
|
||||
caret: [number, number]
|
||||
) => {
|
||||
if (caret[0] !== caret[1]) {
|
||||
return { beforeCursor: value, afterCursor: "" };
|
||||
}
|
||||
|
||||
return {
|
||||
beforeCursor: value.substring(0, caret[0]),
|
||||
afterCursor: value.substring(caret[1]),
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
export const extractMetric = (expr: string): string => {
|
||||
const fnRegex = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
|
||||
const fnMatch = [...expr.matchAll(fnRegex)];
|
||||
|
||||
if (fnMatch[0]?.groups?.metricName) {
|
||||
return fnMatch[0].groups.metricName;
|
||||
}
|
||||
|
||||
const plainRegex = /^\s*\b(?<metricName>[^{}(),\s]+)(?={|$)/g;
|
||||
const match = [...expr.matchAll(plainRegex)];
|
||||
return match[0]?.groups?.metricName || "";
|
||||
};
|
||||
|
||||
export const extractCurrentLabel = (expr: string): string => {
|
||||
const regexp = /[a-z_:-][\w\-.:/]*\b(?=\s*(=|!=|=~|!~))/g;
|
||||
const match = expr.match(regexp);
|
||||
return match ? match[match.length - 1] : "";
|
||||
};
|
||||
|
||||
|
||||
export const extractLabelMatchers = (
|
||||
expr: string,
|
||||
currentLabel?: string
|
||||
): string[] => {
|
||||
const regexp = /([a-z_:-][\w\-.:/]*)\s*(?:=|!=|=~|!~)\s*"[^"]*"/g;
|
||||
|
||||
const matches = [...expr.matchAll(regexp)];
|
||||
// m[1] = label name
|
||||
// m[0] = full matcher string
|
||||
|
||||
if (!currentLabel) return matches.map(m => m[0]);
|
||||
|
||||
return matches
|
||||
.filter(m => m[1] !== currentLabel)
|
||||
.map(m => m[0]);
|
||||
};
|
||||
@@ -30,7 +30,6 @@ type FetchQueryArguments = {
|
||||
metric: string;
|
||||
label: string;
|
||||
context: QueryContextType
|
||||
labelMatchers: string[];
|
||||
}
|
||||
|
||||
const icons = {
|
||||
@@ -39,7 +38,7 @@ const icons = {
|
||||
[TypeData.labelValue]: <ValueIcon/>,
|
||||
};
|
||||
|
||||
export const useFetchQueryOptions = ({ valueByContext, metric, label, context, labelMatchers }: FetchQueryArguments) => {
|
||||
export const useFetchQueryOptions = ({ valueByContext, metric, label, context }: FetchQueryArguments) => {
|
||||
const { serverUrl } = useAppState();
|
||||
const { period: { start, end } } = useTimeState();
|
||||
const { autocompleteCache } = useQueryState();
|
||||
@@ -144,19 +143,17 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context, l
|
||||
setLabels([]);
|
||||
|
||||
const metricEscaped = escapeDoubleQuotes(metric);
|
||||
const matchMetric = metric ? `__name__="${metricEscaped}"` : "";
|
||||
const matchValue = [matchMetric, ...labelMatchers].filter(Boolean).join(",");
|
||||
|
||||
fetchData({
|
||||
value,
|
||||
urlSuffix: "labels",
|
||||
setter: setLabels,
|
||||
type: TypeData.label,
|
||||
params: getQueryParams({ "match[]": `{${matchValue}}` })
|
||||
params: getQueryParams(metric ? { "match[]": `{__name__="${metricEscaped}"}` } : undefined)
|
||||
});
|
||||
|
||||
return () => abortControllerRef.current?.abort();
|
||||
}, [serverUrl, value, context, metric, labelMatchers]);
|
||||
}, [serverUrl, value, context, metric]);
|
||||
|
||||
// fetch labelValues
|
||||
useEffect(() => {
|
||||
@@ -169,7 +166,7 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context, l
|
||||
const valueReEscaped = escapeDoubleQuotes(escapeRegexp(value));
|
||||
const matchMetric = metric ? `__name__="${metricEscaped}"` : "";
|
||||
const matchLabel = `${label}=~".*${valueReEscaped}.*"`;
|
||||
const matchValue = [matchMetric, ...labelMatchers, matchLabel].filter(Boolean).join(",");
|
||||
const matchValue = [matchMetric, matchLabel].filter(Boolean).join(",");
|
||||
|
||||
fetchData({
|
||||
value,
|
||||
@@ -180,7 +177,7 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context, l
|
||||
});
|
||||
|
||||
return () => abortControllerRef.current?.abort();
|
||||
}, [serverUrl, value, context, metric, label, labelMatchers]);
|
||||
}, [serverUrl, value, context, metric, label]);
|
||||
|
||||
return {
|
||||
metrics,
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { QueryUpdater } from "./types";
|
||||
import { escapeLabelName } from "../../utils/metric";
|
||||
|
||||
export const queryUpdater: QueryUpdater = {
|
||||
seriesCountByMetricName: ({ query }): string => {
|
||||
@@ -29,5 +28,5 @@ const getSeriesSelector = (label: string | null, value: string): string => {
|
||||
if (!label) {
|
||||
return "";
|
||||
}
|
||||
return "{" + escapeLabelName(label) + "=" + JSON.stringify(value) + "}";
|
||||
return "{" + label + "=" + JSON.stringify(value) + "}";
|
||||
};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user