mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-28 21:18:23 +03:00
Compare commits
5 Commits
v1.104.0
...
ark/implem
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9b10e6d26b | ||
|
|
042444f3ea | ||
|
|
21beb994ad | ||
|
|
4605a72684 | ||
|
|
08de878047 |
59
.github/workflows/test_package.yml
vendored
Normal file
59
.github/workflows/test_package.yml
vendored
Normal file
@@ -0,0 +1,59 @@
|
||||
name: Package tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- cluster
|
||||
- master
|
||||
paths:
|
||||
- '.github/workflows/test_package.yml'
|
||||
- '**/Dockerfile*'
|
||||
- '**/Makefile'
|
||||
pull_request:
|
||||
branches:
|
||||
- cluster
|
||||
- master
|
||||
paths:
|
||||
- '.github/workflows/test_package.yml'
|
||||
- '**/Dockerfile*'
|
||||
- '**/Makefile'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
|
||||
jobs:
|
||||
test_release:
|
||||
name: Test Release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Code checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Go
|
||||
id: go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: stable
|
||||
cache: false
|
||||
|
||||
- name: Cache Go artifacts
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/bin
|
||||
~/go/pkg/mod
|
||||
key: go-artifacts-${{ runner.os }}-test-release-${{ steps.go.outputs.go-version }}-${{ hashFiles('Makefile', '**/Dockerfile*', '**/Makefile') }}
|
||||
restore-keys: go-artifacts-${{ runner.os }}-test-release-
|
||||
|
||||
- name: Run make release
|
||||
run: |
|
||||
make clean
|
||||
make release
|
||||
|
||||
- name: Run release tests
|
||||
run: make test-release-victoria-metrics
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -7,7 +7,6 @@
|
||||
.vscode
|
||||
*.test
|
||||
*.swp
|
||||
/vmdocs
|
||||
/gocache-for-docker
|
||||
/victoria-logs-data
|
||||
/victoria-metrics-data
|
||||
@@ -26,4 +25,3 @@ _site
|
||||
/docs/.jekyll-metadata
|
||||
coverage.txt
|
||||
cspell.json
|
||||
*~
|
||||
|
||||
@@ -4,4 +4,3 @@ allowlist:
|
||||
- BSD-3-Clause
|
||||
- BSD-2-Clause
|
||||
- ISC
|
||||
- MPL-2.0
|
||||
|
||||
284
Makefile
284
Makefile
@@ -13,7 +13,10 @@ PKG_TAG := $(BUILDINFO_TAG)
|
||||
endif
|
||||
|
||||
GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
|
||||
TAR_OWNERSHIP ?= --owner=1000 --group=1000
|
||||
|
||||
VICTORIA_LOGS_COMPONENTS = victoria-logs
|
||||
VICTORIA_METRICS_COMPONENTS = victoria-metrics
|
||||
VICTORIA_METRICS_UTILS_COMPONENTS = vmagent vmalert vmalert-tool vmauth vmbackup vmrestore vmctl
|
||||
|
||||
.PHONY: $(MAKECMDGOALS)
|
||||
|
||||
@@ -24,10 +27,76 @@ include deployment/*/Makefile
|
||||
include dashboards/Makefile
|
||||
include package/release/Makefile
|
||||
|
||||
define RELEASE_GOOS_GOARCH
|
||||
$(eval PKG_NAME := $(1))
|
||||
$(eval PKG_COMPONENTS := $(2))
|
||||
|
||||
# Build
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
|
||||
|
||||
# Generate SBOM
|
||||
mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS),
|
||||
cyclonedx-gomod app -assert-licenses -json -licenses -packages \
|
||||
-main app/$(pkg_component) \
|
||||
-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
|
||||
|
||||
# Pack and compress
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS),
|
||||
cd bin && tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar \
|
||||
$(pkg_component)-$(GOOS)-$(GOARCH)-prod)
|
||||
cd bin && gzip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar
|
||||
cd bin && tar -czf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
|
||||
|
||||
# Generate checksums
|
||||
cd bin && \
|
||||
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
|
||||
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS),
|
||||
cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ >> \
|
||||
$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
|
||||
|
||||
# Clean up
|
||||
cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
|
||||
endef
|
||||
|
||||
define RELEASE_WINDOWS_GOARCH
|
||||
$(eval PKG_NAME := $(1))
|
||||
$(eval PKG_COMPONENTS := $(2))
|
||||
|
||||
# Build
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
|
||||
|
||||
# Generate SBOM
|
||||
mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS),
|
||||
cyclonedx-gomod app -assert-licenses -json -licenses -packages \
|
||||
-main app/$(pkg_component) \
|
||||
-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
|
||||
|
||||
# Pack and compress
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS),
|
||||
cd bin && zip -u $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip \
|
||||
$(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
|
||||
cd bin && zip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip -r $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
|
||||
|
||||
# Generate checksums
|
||||
cd bin && \
|
||||
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
|
||||
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS),
|
||||
cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
|
||||
|
||||
# Clean up
|
||||
cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
|
||||
$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
|
||||
endef
|
||||
|
||||
|
||||
all: \
|
||||
victoria-metrics-prod \
|
||||
victoria-logs-prod \
|
||||
vlogscli-prod \
|
||||
vmagent-prod \
|
||||
vmalert-prod \
|
||||
vmalert-tool-prod \
|
||||
@@ -52,7 +121,6 @@ publish: \
|
||||
package: \
|
||||
package-victoria-metrics \
|
||||
package-victoria-logs \
|
||||
package-vlogscli \
|
||||
package-vmagent \
|
||||
package-vmalert \
|
||||
package-vmalert-tool \
|
||||
@@ -244,26 +312,13 @@ release-victoria-metrics-openbsd-amd64:
|
||||
GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
|
||||
|
||||
release-victoria-metrics-windows-amd64:
|
||||
GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
|
||||
GOOS=windows GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
|
||||
|
||||
release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
victoria-metrics-$(GOOS)-$(GOARCH)-prod \
|
||||
&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
victoria-metrics-$(GOOS)-$(GOARCH)-prod \
|
||||
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
|
||||
release-victoria-metrics-goos-goarch:
|
||||
$(call RELEASE_GOOS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
|
||||
|
||||
release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
|
||||
victoria-metrics-windows-$(GOARCH)-prod.exe \
|
||||
&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
|
||||
victoria-metrics-windows-$(GOARCH)-prod.exe \
|
||||
> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
cd bin && rm -rf \
|
||||
victoria-metrics-windows-$(GOARCH)-prod.exe
|
||||
release-victoria-metrics-windows-goarch:
|
||||
$(call RELEASE_WINDOWS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
|
||||
|
||||
release-victoria-logs:
|
||||
$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
|
||||
@@ -305,7 +360,7 @@ release-victoria-logs-windows-amd64:
|
||||
|
||||
release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
victoria-logs-$(GOOS)-$(GOARCH)-prod \
|
||||
&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
victoria-logs-$(GOOS)-$(GOARCH)-prod \
|
||||
@@ -322,63 +377,6 @@ release-victoria-logs-windows-goarch: victoria-logs-windows-$(GOARCH)-prod
|
||||
cd bin && rm -rf \
|
||||
victoria-logs-windows-$(GOARCH)-prod.exe
|
||||
|
||||
release-vlogscli:
|
||||
$(MAKE_PARALLEL) release-vlogscli-linux-386 \
|
||||
release-vlogscli-linux-amd64 \
|
||||
release-vlogscli-linux-arm \
|
||||
release-vlogscli-linux-arm64 \
|
||||
release-vlogscli-darwin-amd64 \
|
||||
release-vlogscli-darwin-arm64 \
|
||||
release-vlogscli-freebsd-amd64 \
|
||||
release-vlogscli-openbsd-amd64 \
|
||||
release-vlogscli-windows-amd64
|
||||
|
||||
release-vlogscli-linux-386:
|
||||
GOOS=linux GOARCH=386 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-linux-amd64:
|
||||
GOOS=linux GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-linux-arm:
|
||||
GOOS=linux GOARCH=arm $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-linux-arm64:
|
||||
GOOS=linux GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-darwin-amd64:
|
||||
GOOS=darwin GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-darwin-arm64:
|
||||
GOOS=darwin GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-freebsd-amd64:
|
||||
GOOS=freebsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-openbsd-amd64:
|
||||
GOOS=openbsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-windows-amd64:
|
||||
GOARCH=amd64 $(MAKE) release-vlogscli-windows-goarch
|
||||
|
||||
release-vlogscli-goos-goarch: vlogscli-$(GOOS)-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
vlogscli-$(GOOS)-$(GOARCH)-prod \
|
||||
&& sha256sum vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
vlogscli-$(GOOS)-$(GOARCH)-prod \
|
||||
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
cd bin && rm -rf vlogscli-$(GOOS)-$(GOARCH)-prod
|
||||
|
||||
release-vlogscli-windows-goarch: vlogscli-windows-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
zip vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
|
||||
vlogscli-windows-$(GOARCH)-prod.exe \
|
||||
&& sha256sum vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
|
||||
vlogscli-windows-$(GOARCH)-prod.exe \
|
||||
> vlogscli-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
cd bin && rm -rf \
|
||||
vlogscli-windows-$(GOARCH)-prod.exe
|
||||
|
||||
release-vmutils: \
|
||||
release-vmutils-linux-386 \
|
||||
release-vmutils-linux-amd64 \
|
||||
@@ -392,7 +390,7 @@ release-vmutils: \
|
||||
|
||||
release-vmutils-linux-386:
|
||||
GOOS=linux GOARCH=386 $(MAKE) release-vmutils-goos-goarch
|
||||
|
||||
|
||||
release-vmutils-linux-amd64:
|
||||
GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
|
||||
|
||||
@@ -415,77 +413,13 @@ release-vmutils-openbsd-amd64:
|
||||
GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
|
||||
|
||||
release-vmutils-windows-amd64:
|
||||
GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
|
||||
GOOS=windows GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
|
||||
|
||||
release-vmutils-goos-goarch: \
|
||||
vmagent-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
|
||||
vmauth-$(GOOS)-$(GOARCH)-prod \
|
||||
vmbackup-$(GOOS)-$(GOARCH)-prod \
|
||||
vmrestore-$(GOOS)-$(GOARCH)-prod \
|
||||
vmctl-$(GOOS)-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
vmagent-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
|
||||
vmauth-$(GOOS)-$(GOARCH)-prod \
|
||||
vmbackup-$(GOOS)-$(GOARCH)-prod \
|
||||
vmrestore-$(GOOS)-$(GOARCH)-prod \
|
||||
vmctl-$(GOOS)-$(GOARCH)-prod \
|
||||
&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
vmagent-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
|
||||
vmauth-$(GOOS)-$(GOARCH)-prod \
|
||||
vmbackup-$(GOOS)-$(GOARCH)-prod \
|
||||
vmrestore-$(GOOS)-$(GOARCH)-prod \
|
||||
vmctl-$(GOOS)-$(GOARCH)-prod \
|
||||
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
cd bin && rm -rf \
|
||||
vmagent-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
|
||||
vmauth-$(GOOS)-$(GOARCH)-prod \
|
||||
vmbackup-$(GOOS)-$(GOARCH)-prod \
|
||||
vmrestore-$(GOOS)-$(GOARCH)-prod \
|
||||
vmctl-$(GOOS)-$(GOARCH)-prod
|
||||
release-vmutils-goos-goarch:
|
||||
$(call RELEASE_GOOS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
|
||||
|
||||
release-vmutils-windows-goarch: \
|
||||
vmagent-windows-$(GOARCH)-prod \
|
||||
vmalert-windows-$(GOARCH)-prod \
|
||||
vmalert-tool-windows-$(GOARCH)-prod \
|
||||
vmauth-windows-$(GOARCH)-prod \
|
||||
vmbackup-windows-$(GOARCH)-prod \
|
||||
vmrestore-windows-$(GOARCH)-prod \
|
||||
vmctl-windows-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
|
||||
vmagent-windows-$(GOARCH)-prod.exe \
|
||||
vmalert-windows-$(GOARCH)-prod.exe \
|
||||
vmalert-tool-windows-$(GOARCH)-prod.exe \
|
||||
vmauth-windows-$(GOARCH)-prod.exe \
|
||||
vmbackup-windows-$(GOARCH)-prod.exe \
|
||||
vmrestore-windows-$(GOARCH)-prod.exe \
|
||||
vmctl-windows-$(GOARCH)-prod.exe \
|
||||
&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
|
||||
vmagent-windows-$(GOARCH)-prod.exe \
|
||||
vmalert-windows-$(GOARCH)-prod.exe \
|
||||
vmalert-tool-windows-$(GOARCH)-prod.exe \
|
||||
vmauth-windows-$(GOARCH)-prod.exe \
|
||||
vmbackup-windows-$(GOARCH)-prod.exe \
|
||||
vmrestore-windows-$(GOARCH)-prod.exe \
|
||||
vmctl-windows-$(GOARCH)-prod.exe \
|
||||
> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
cd bin && rm -rf \
|
||||
vmagent-windows-$(GOARCH)-prod.exe \
|
||||
vmalert-windows-$(GOARCH)-prod.exe \
|
||||
vmalert-tool-windows-$(GOARCH)-prod.exe \
|
||||
vmauth-windows-$(GOARCH)-prod.exe \
|
||||
vmbackup-windows-$(GOARCH)-prod.exe \
|
||||
vmrestore-windows-$(GOARCH)-prod.exe \
|
||||
vmctl-windows-$(GOARCH)-prod.exe
|
||||
release-vmutils-windows-goarch:
|
||||
$(call RELEASE_WINDOWS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
|
||||
|
||||
pprof-cpu:
|
||||
go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
|
||||
@@ -503,19 +437,19 @@ check-all: fmt vet golangci-lint govulncheck
|
||||
clean-checkers: remove-golangci-lint remove-govulncheck
|
||||
|
||||
test:
|
||||
DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/...
|
||||
go test ./lib/... ./app/...
|
||||
|
||||
test-race:
|
||||
DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/...
|
||||
go test -race ./lib/... ./app/...
|
||||
|
||||
test-pure:
|
||||
DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/...
|
||||
CGO_ENABLED=0 go test ./lib/... ./app/...
|
||||
|
||||
test-full:
|
||||
DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
|
||||
test-full-386:
|
||||
DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
|
||||
benchmark:
|
||||
go test -bench=. ./lib/...
|
||||
@@ -526,9 +460,9 @@ benchmark-pure:
|
||||
CGO_ENABLED=0 go test -bench=. ./app/...
|
||||
|
||||
vendor-update:
|
||||
go get -u ./lib/...
|
||||
go get -u ./app/...
|
||||
go mod tidy -compat=1.23
|
||||
go get -u -d ./lib/...
|
||||
go get -u -d ./app/...
|
||||
go mod tidy -compat=1.22
|
||||
go mod vendor
|
||||
|
||||
app-local:
|
||||
@@ -554,7 +488,7 @@ golangci-lint: install-golangci-lint
|
||||
golangci-lint run
|
||||
|
||||
install-golangci-lint:
|
||||
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.3
|
||||
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
|
||||
|
||||
remove-golangci-lint:
|
||||
rm -rf `which golangci-lint`
|
||||
@@ -573,3 +507,37 @@ install-wwhrd:
|
||||
|
||||
check-licenses: install-wwhrd
|
||||
wwhrd check -f .wwhrd.yml
|
||||
|
||||
cyclonedx-gomod-install:
|
||||
which cyclonedx-gomod || go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest
|
||||
|
||||
copy-docs:
|
||||
# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
|
||||
# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
|
||||
echo "---" > ${DST}
|
||||
@if [ ${ORDER} -ne 0 ]; then \
|
||||
echo "sort: ${ORDER}" >> ${DST}; \
|
||||
echo "weight: ${ORDER}" >> ${DST}; \
|
||||
printf "menu:\n docs:\n parent: 'victoriametrics'\n weight: ${ORDER}\n" >> ${DST}; \
|
||||
fi
|
||||
|
||||
echo "title: ${TITLE}" >> ${DST}
|
||||
@if [ ${OLD_URL} ]; then \
|
||||
printf "aliases:\n - ${OLD_URL}\n" >> ${DST}; \
|
||||
fi
|
||||
echo "---" >> ${DST}
|
||||
cat ${SRC} >> ${DST}
|
||||
sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
|
||||
sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"/' ${DST}
|
||||
rm -rf docs/*.tmp
|
||||
|
||||
# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
|
||||
# For ORDER=0 it adds no order tag/weight tag.
|
||||
# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
|
||||
# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
|
||||
# Images starting with <img src="docs/ are replaced with <img src="
|
||||
# Cluster docs are supposed to be ordered as 2nd.
|
||||
# The rest of docs is ordered manually.
|
||||
docs-sync:
|
||||
SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
|
||||
SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs
|
||||
|
||||
@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
|
||||
| Version | Supported |
|
||||
|---------|--------------------|
|
||||
| [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
|
||||
| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
|
||||
| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
|
||||
| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
|
||||
| other releases | :x: |
|
||||
|
||||
See [this page](https://victoriametrics.com/security/) for more details.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please report any security issues to <security@victoriametrics.com>
|
||||
Please report any security issues to security@victoriametrics.com
|
||||
|
||||
BIN
VM_logo.zip
BIN
VM_logo.zip
Binary file not shown.
@@ -1,8 +1,8 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 9428
|
||||
|
||||
ENTRYPOINT ["/victoria-logs-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./victoria-logs-prod
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 8428
|
||||
|
||||
ENTRYPOINT ["/victoria-metrics-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./victoria-metrics-prod
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -55,12 +57,6 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
|
||||
// Return fake response for Logstash APIs requests.
|
||||
// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
switch path {
|
||||
case "/":
|
||||
switch r.Method {
|
||||
@@ -249,8 +245,19 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
|
||||
// Try parsing timestamp in seconds or milliseconds
|
||||
return insertutils.ParseUnixTimestamp(s)
|
||||
// Try parsing timestamp in milliseconds
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
|
||||
}
|
||||
if n > int64(math.MaxInt64)/1e6 {
|
||||
return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
|
||||
}
|
||||
if n < int64(math.MinInt64)/1e6 {
|
||||
return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
|
||||
}
|
||||
n *= 1e6
|
||||
return n, nil
|
||||
}
|
||||
if len(s) == len("YYYY-MM-DD") {
|
||||
t, err := time.Parse("2006-01-02", s)
|
||||
|
||||
@@ -76,20 +76,17 @@ func TestReadBulkRequest_Success(t *testing.T) {
|
||||
data := `{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
|
||||
{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"@timestamp":"2023-06-06 04:48:12.735+01:00","message":"baz"}
|
||||
{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
|
||||
{"index":{"_index":"filebeat-8.8.0"}}
|
||||
{"message":"xyz","@timestamp":"1686026893735","x":"y"}
|
||||
{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"message":"qwe rty","@timestamp":"1686026893"}
|
||||
{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
|
||||
`
|
||||
timeField := "@timestamp"
|
||||
msgField := "message"
|
||||
rowsExpected := 4
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
|
||||
rowsExpected := 3
|
||||
timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
|
||||
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"@timestamp":"","_msg":"baz"}
|
||||
{"_msg":"xyz","@timestamp":"","x":"y"}
|
||||
{"_msg":"qwe rty","@timestamp":""}`
|
||||
{"_msg":"xyz","@timestamp":"","x":"y"}`
|
||||
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ package insertutils
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -39,46 +38,22 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Extract time field name from _time_field query arg or header
|
||||
timeField := "_time"
|
||||
// Extract time field name from _time_field query arg
|
||||
var timeField = "_time"
|
||||
if tf := r.FormValue("_time_field"); tf != "" {
|
||||
timeField = tf
|
||||
} else if tf = r.Header.Get("VL-Time-Field"); tf != "" {
|
||||
timeField = tf
|
||||
}
|
||||
|
||||
// Extract message field name from _msg_field query arg or header
|
||||
msgField := ""
|
||||
// Extract message field name from _msg_field query arg
|
||||
var msgField = ""
|
||||
if msgf := r.FormValue("_msg_field"); msgf != "" {
|
||||
msgField = msgf
|
||||
} else if msgf = r.Header.Get("VL-Msg-Field"); msgf != "" {
|
||||
msgField = msgf
|
||||
}
|
||||
|
||||
streamFields := httputils.GetArray(r, "_stream_fields")
|
||||
if len(streamFields) == 0 {
|
||||
if sf := r.Header.Get("VL-Stream-Fields"); len(sf) > 0 {
|
||||
streamFields = strings.Split(sf, ",")
|
||||
}
|
||||
}
|
||||
ignoreFields := httputils.GetArray(r, "ignore_fields")
|
||||
if len(ignoreFields) == 0 {
|
||||
if f := r.Header.Get("VL-Ignore-Fields"); len(f) > 0 {
|
||||
ignoreFields = strings.Split(f, ",")
|
||||
}
|
||||
}
|
||||
|
||||
debug := httputils.GetBool(r, "debug")
|
||||
if !debug {
|
||||
if dh := r.Header.Get("VL-Debug"); len(dh) > 0 {
|
||||
hv := strings.ToLower(dh)
|
||||
switch hv {
|
||||
case "", "0", "f", "false", "no":
|
||||
default:
|
||||
debug = true
|
||||
}
|
||||
}
|
||||
}
|
||||
debugRequestURI := ""
|
||||
debugRemoteAddr := ""
|
||||
if debug {
|
||||
@@ -96,7 +71,6 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
||||
DebugRequestURI: debugRequestURI,
|
||||
DebugRemoteAddr: debugRemoteAddr,
|
||||
}
|
||||
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
@@ -176,27 +150,11 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Fiel
|
||||
return
|
||||
}
|
||||
|
||||
// _msg field must be non-empty according to VictoriaLogs data model.
|
||||
// See https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field
|
||||
msgExist := false
|
||||
for i := range fields {
|
||||
if fields[i].Name == "_msg" {
|
||||
msgExist = len(fields[i].Value) > 0
|
||||
break
|
||||
}
|
||||
}
|
||||
if !msgExist {
|
||||
rf := logstorage.RowFormatter(fields)
|
||||
logger.Warnf("dropping log line without _msg field; %s", rf)
|
||||
rowsDroppedTotalMsgNotValid.Inc()
|
||||
return
|
||||
}
|
||||
|
||||
lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields)
|
||||
if lmp.cp.Debug {
|
||||
s := lmp.lr.GetRowString(0)
|
||||
lmp.lr.ResetKeepSettings()
|
||||
logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
|
||||
logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
|
||||
rowsDroppedTotalDebug.Inc()
|
||||
return
|
||||
}
|
||||
@@ -238,8 +196,5 @@ func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
|
||||
return lmp
|
||||
}
|
||||
|
||||
var (
|
||||
rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
|
||||
rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
|
||||
rowsDroppedTotalMsgNotValid = metrics.NewCounter(`vl_rows_dropped_total{reason="msg_not_exist"}`)
|
||||
)
|
||||
var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
|
||||
var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
|
||||
|
||||
@@ -2,8 +2,6 @@ package insertutils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
@@ -21,49 +19,15 @@ func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage
|
||||
if f.Name != timeField {
|
||||
continue
|
||||
}
|
||||
nsecs, err := parseTimestamp(f.Value)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", timeField, err)
|
||||
if f.Value == "" || f.Value == "0" {
|
||||
return time.Now().UnixNano(), nil
|
||||
}
|
||||
nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp from %s=%q", timeField, f.Value)
|
||||
}
|
||||
f.Value = ""
|
||||
if nsecs == 0 {
|
||||
nsecs = time.Now().UnixNano()
|
||||
}
|
||||
return nsecs, nil
|
||||
}
|
||||
return time.Now().UnixNano(), nil
|
||||
}
|
||||
|
||||
func parseTimestamp(s string) (int64, error) {
|
||||
if s == "" || s == "0" {
|
||||
return time.Now().UnixNano(), nil
|
||||
}
|
||||
if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
|
||||
return ParseUnixTimestamp(s)
|
||||
}
|
||||
nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp %q", s)
|
||||
}
|
||||
return nsecs, nil
|
||||
}
|
||||
|
||||
// ParseUnixTimestamp parses s as unix timestamp in either seconds or milliseconds and returns the parsed timestamp in nanoseconds.
|
||||
func ParseUnixTimestamp(s string) (int64, error) {
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
|
||||
}
|
||||
if n < (1<<31) && n >= (-1<<31) {
|
||||
// The timestamp is in seconds. Convert it to milliseconds
|
||||
n *= 1e3
|
||||
}
|
||||
if n > int64(math.MaxInt64)/1e6 {
|
||||
return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
|
||||
}
|
||||
if n < int64(math.MinInt64)/1e6 {
|
||||
return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
|
||||
}
|
||||
n *= 1e6
|
||||
return n, nil
|
||||
}
|
||||
|
||||
@@ -27,41 +27,25 @@ func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// UTC time
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "2024-06-18T23:37:20Z"},
|
||||
}, 1718753840000000000)
|
||||
|
||||
// Time with timezone
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
|
||||
}, 1718725040000000000)
|
||||
|
||||
// SQL datetime format
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "2024-06-18 23:37:20.123-05:30"},
|
||||
{Name: "time", Value: "2024-06-18T23:37:20.123-05:30"},
|
||||
}, 1718773640123000000)
|
||||
|
||||
// Time with nanosecond precision
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
|
||||
{Name: "foo", Value: "bar"},
|
||||
}, 1718773640123456789)
|
||||
|
||||
// Unix timestamp in milliseconds
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "1718773640123"},
|
||||
}, 1718773640123000000)
|
||||
|
||||
// Unix timestamp in seconds
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "1718773640"},
|
||||
}, 1718773640000000000)
|
||||
}
|
||||
|
||||
func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
|
||||
@@ -82,6 +66,9 @@ func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
|
||||
|
||||
f("foobar")
|
||||
|
||||
// no Z at the end
|
||||
f("2024-06-18T23:37:20")
|
||||
|
||||
// incomplete time
|
||||
f("2024-06-18")
|
||||
f("2024-06-18T23:37")
|
||||
|
||||
@@ -23,13 +23,13 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
||||
}
|
||||
|
||||
data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
|
||||
{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
|
||||
{"message":"xyz","@timestamp":"2023-06-06 04:48:13.735Z","x":"y"}
|
||||
{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
|
||||
{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
|
||||
`
|
||||
timeField := "@timestamp"
|
||||
msgField := "message"
|
||||
rowsExpected := 3
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
|
||||
timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
|
||||
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"@timestamp":"","_msg":"baz"}
|
||||
{"_msg":"xyz","@timestamp":"","x":"y"}`
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/opentelemetry"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
|
||||
)
|
||||
|
||||
@@ -42,9 +41,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
case strings.HasPrefix(path, "/loki/"):
|
||||
path = strings.TrimPrefix(path, "/loki")
|
||||
return loki.RequestHandler(path, w, r)
|
||||
case strings.HasPrefix(path, "/opentelemetry/"):
|
||||
path = strings.TrimPrefix(path, "/opentelemetry")
|
||||
return opentelemetry.RequestHandler(path, w, r)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
package opentelemetry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
// RequestHandler processes Opentelemetry insert requests
|
||||
func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
// use the same path as opentelemetry collector
|
||||
// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
|
||||
case "/v1/logs":
|
||||
if r.Header.Get("Content-Type") == "application/json" {
|
||||
httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
|
||||
return true
|
||||
}
|
||||
handleProtobuf(r, w)
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
startTime := time.Now()
|
||||
requestsProtobufTotal.Inc()
|
||||
reader := r.Body
|
||||
if r.Header.Get("Content-Encoding") == "gzip" {
|
||||
zr, err := common.GetGzipReader(reader)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
|
||||
return
|
||||
}
|
||||
defer common.PutGzipReader(zr)
|
||||
reader = zr
|
||||
}
|
||||
|
||||
wcr := writeconcurrencylimiter.GetReader(reader)
|
||||
data, err := io.ReadAll(wcr)
|
||||
writeconcurrencylimiter.PutReader(wcr)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read request body: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
|
||||
return
|
||||
}
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor()
|
||||
n, err := pushProtobufRequest(data, lmp)
|
||||
lmp.MustClose()
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse OpenTelemetry protobuf request: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
rowsIngestedProtobufTotal.Add(n)
|
||||
|
||||
// update requestProtobufDuration only for successfully parsed requests
|
||||
// There is no need in updating requestProtobufDuration for request errors,
|
||||
// since their timings are usually much smaller than the timing for successful request parsing.
|
||||
requestProtobufDuration.UpdateDuration(startTime)
|
||||
}
|
||||
|
||||
var (
|
||||
rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="opentelemetry",format="protobuf"}`)
|
||||
|
||||
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
|
||||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
)
|
||||
|
||||
func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
|
||||
var req pb.ExportLogsServiceRequest
|
||||
if err := req.UnmarshalProtobuf(data); err != nil {
|
||||
errorsTotal.Inc()
|
||||
return 0, fmt.Errorf("cannot unmarshal request from %d bytes: %w", len(data), err)
|
||||
}
|
||||
|
||||
var rowsIngested int
|
||||
var commonFields []logstorage.Field
|
||||
for _, rl := range req.ResourceLogs {
|
||||
attributes := rl.Resource.Attributes
|
||||
commonFields = slicesutil.SetLength(commonFields, len(attributes))
|
||||
for i, attr := range attributes {
|
||||
commonFields[i].Name = attr.Key
|
||||
commonFields[i].Value = attr.Value.FormatString()
|
||||
}
|
||||
commonFieldsLen := len(commonFields)
|
||||
for _, sc := range rl.ScopeLogs {
|
||||
var scopeIngested int
|
||||
commonFields, scopeIngested = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp)
|
||||
rowsIngested += scopeIngested
|
||||
}
|
||||
}
|
||||
|
||||
return rowsIngested, nil
|
||||
}
|
||||
|
||||
func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutils.LogMessageProcessor) ([]logstorage.Field, int) {
|
||||
fields := commonFields
|
||||
for _, lr := range sc.LogRecords {
|
||||
fields = fields[:len(commonFields)]
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: "_msg",
|
||||
Value: lr.Body.FormatString(),
|
||||
})
|
||||
for _, attr := range lr.Attributes {
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: attr.Key,
|
||||
Value: attr.Value.FormatString(),
|
||||
})
|
||||
}
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: "severity",
|
||||
Value: lr.FormatSeverity(),
|
||||
})
|
||||
|
||||
lmp.AddRow(lr.ExtractTimestampNano(), fields)
|
||||
}
|
||||
return fields, len(sc.LogRecords)
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
package opentelemetry
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
|
||||
)
|
||||
|
||||
func TestPushProtoOk(t *testing.T) {
|
||||
f := func(src []pb.ResourceLogs, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
lr := pb.ExportLogsServiceRequest{
|
||||
ResourceLogs: src,
|
||||
}
|
||||
|
||||
pData := lr.MarshalProtobuf(nil)
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
n, err := pushProtobufRequest(pData, tlp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// single line without resource attributes
|
||||
f([]pb.ResourceLogs{
|
||||
{
|
||||
ScopeLogs: []pb.ScopeLogs{
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]int64{1234},
|
||||
`{"_msg":"log-line-message","severity":"Trace"}`,
|
||||
)
|
||||
// multi-line with resource attributes
|
||||
f([]pb.ResourceLogs{
|
||||
{
|
||||
Resource: pb.Resource{
|
||||
Attributes: []*pb.KeyValue{
|
||||
{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
|
||||
{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
|
||||
{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
|
||||
Values: []*pb.KeyValue{
|
||||
{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
|
||||
{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
|
||||
},
|
||||
}}},
|
||||
},
|
||||
},
|
||||
ScopeLogs: []pb.ScopeLogs{
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 21, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]int64{1234, 1235, 1236},
|
||||
`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
|
||||
{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}
|
||||
{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
|
||||
)
|
||||
|
||||
// multi-scope with resource attributes and multi-line
|
||||
f([]pb.ResourceLogs{
|
||||
{
|
||||
Resource: pb.Resource{
|
||||
Attributes: []*pb.KeyValue{
|
||||
{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
|
||||
{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
|
||||
{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
|
||||
Values: []*pb.KeyValue{
|
||||
{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
|
||||
{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
|
||||
},
|
||||
}}},
|
||||
},
|
||||
},
|
||||
ScopeLogs: []pb.ScopeLogs{
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
{TimeUnixNano: 1235, SeverityNumber: 5, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ScopeLogs: []pb.ScopeLogs{
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{TimeUnixNano: 2345, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-0")}},
|
||||
{TimeUnixNano: 2346, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-1")}},
|
||||
},
|
||||
},
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
|
||||
{ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]int64{1234, 1235, 2345, 2346, 2347, 2348},
|
||||
`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
|
||||
{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Debug"}
|
||||
{"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
|
||||
{"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
|
||||
{"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
|
||||
{"_msg":"log-line-resource-scope-1-1-1","severity":"Info4"}`,
|
||||
)
|
||||
}
|
||||
|
||||
func ptrTo[T any](s T) *T {
|
||||
return &s
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package opentelemetry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
|
||||
)
|
||||
|
||||
func BenchmarkParseProtobufRequest(b *testing.B) {
|
||||
for _, scopes := range []int{1, 2} {
|
||||
for _, rows := range []int{100, 1000} {
|
||||
for _, attributes := range []int{5, 10} {
|
||||
b.Run(fmt.Sprintf("scopes_%d/rows_%d/attributes_%d", scopes, rows, attributes), func(b *testing.B) {
|
||||
benchmarkParseProtobufRequest(b, scopes, rows, attributes)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
|
||||
blp := &insertutils.BenchmarkLogMessageProcessor{}
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(streams * rows))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
body := getProtobufBody(streams, rows, labels)
|
||||
for pb.Next() {
|
||||
_, err := pushProtobufRequest(body, blp)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func getProtobufBody(scopesCount, rowsCount, attributesCount int) []byte {
|
||||
msg := "12345678910"
|
||||
|
||||
attrValues := []*pb.AnyValue{
|
||||
{StringValue: ptrTo("string-attribute")},
|
||||
{IntValue: ptrTo[int64](12345)},
|
||||
{DoubleValue: ptrTo(3.14)},
|
||||
}
|
||||
attrs := make([]*pb.KeyValue, attributesCount)
|
||||
for j := 0; j < attributesCount; j++ {
|
||||
attrs[j] = &pb.KeyValue{
|
||||
Key: fmt.Sprintf("key-%d", j),
|
||||
Value: attrValues[j%3],
|
||||
}
|
||||
}
|
||||
entries := make([]pb.LogRecord, rowsCount)
|
||||
for j := 0; j < rowsCount; j++ {
|
||||
entries[j] = pb.LogRecord{
|
||||
TimeUnixNano: 12345678910, ObservedTimeUnixNano: 12345678910, Body: pb.AnyValue{StringValue: &msg},
|
||||
}
|
||||
}
|
||||
scopes := make([]pb.ScopeLogs, scopesCount)
|
||||
|
||||
for j := 0; j < scopesCount; j++ {
|
||||
scopes[j] = pb.ScopeLogs{
|
||||
LogRecords: entries,
|
||||
}
|
||||
}
|
||||
|
||||
pr := pb.ExportLogsServiceRequest{
|
||||
ResourceLogs: []pb.ResourceLogs{
|
||||
{
|
||||
Resource: pb.Resource{
|
||||
Attributes: attrs,
|
||||
},
|
||||
ScopeLogs: scopes,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return pr.MarshalProtobuf(nil)
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
# All these commands must run from repository root.
|
||||
|
||||
vlogscli:
|
||||
APP_NAME=vlogscli $(MAKE) app-local
|
||||
|
||||
vlogscli-race:
|
||||
APP_NAME=vlogscli RACE=-race $(MAKE) app-local
|
||||
|
||||
vlogscli-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker
|
||||
|
||||
vlogscli-pure-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-pure
|
||||
|
||||
vlogscli-linux-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-amd64
|
||||
|
||||
vlogscli-linux-arm-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm
|
||||
|
||||
vlogscli-linux-arm64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm64
|
||||
|
||||
vlogscli-linux-ppc64le-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-ppc64le
|
||||
|
||||
vlogscli-linux-386-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-386
|
||||
|
||||
vlogscli-darwin-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-amd64
|
||||
|
||||
vlogscli-darwin-arm64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-arm64
|
||||
|
||||
vlogscli-freebsd-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-freebsd-amd64
|
||||
|
||||
vlogscli-openbsd-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-openbsd-amd64
|
||||
|
||||
vlogscli-windows-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-windows-amd64
|
||||
|
||||
package-vlogscli:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker
|
||||
|
||||
package-vlogscli-pure:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-pure
|
||||
|
||||
package-vlogscli-amd64:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-amd64
|
||||
|
||||
package-vlogscli-arm:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-arm
|
||||
|
||||
package-vlogscli-arm64:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-arm64
|
||||
|
||||
package-vlogscli-ppc64le:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-ppc64le
|
||||
|
||||
package-vlogscli-386:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-386
|
||||
|
||||
publish-vlogscli:
|
||||
APP_NAME=vlogscli $(MAKE) publish-via-docker
|
||||
|
||||
vlogscli-linux-amd64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-arm:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-arm64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-ppc64le:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-s390x:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-loong64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-386:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-darwin-amd64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-darwin-arm64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-freebsd-amd64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-openbsd-amd64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-windows-amd64:
|
||||
GOARCH=amd64 APP_NAME=vlogscli $(MAKE) app-local-windows-goarch
|
||||
|
||||
vlogscli-pure:
|
||||
APP_NAME=vlogscli $(MAKE) app-local-pure
|
||||
|
||||
run-vlogscli:
|
||||
APP_NAME=vlogscli $(MAKE) run-via-docker
|
||||
@@ -1,5 +0,0 @@
|
||||
# vlogscli
|
||||
|
||||
Command-line utility for querying [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
|
||||
|
||||
See [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/).
|
||||
@@ -1,6 +0,0 @@
|
||||
ARG base_image=non-existing
|
||||
FROM $base_image
|
||||
|
||||
ENTRYPOINT ["/vlogscli-prod"]
|
||||
ARG src_binary=non-existing
|
||||
COPY $src_binary ./vlogscli-prod
|
||||
@@ -1,73 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type jsonPrettifier struct {
|
||||
rOriginal io.ReadCloser
|
||||
|
||||
d *json.Decoder
|
||||
|
||||
pr *io.PipeReader
|
||||
pw *io.PipeWriter
|
||||
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func newJSONPrettifier(r io.ReadCloser) *jsonPrettifier {
|
||||
d := json.NewDecoder(r)
|
||||
pr, pw := io.Pipe()
|
||||
|
||||
jp := &jsonPrettifier{
|
||||
rOriginal: r,
|
||||
d: d,
|
||||
pr: pr,
|
||||
pw: pw,
|
||||
}
|
||||
|
||||
jp.wg.Add(1)
|
||||
go func() {
|
||||
defer jp.wg.Done()
|
||||
err := jp.prettifyJSONLines()
|
||||
jp.closePipesWithError(err)
|
||||
}()
|
||||
|
||||
return jp
|
||||
}
|
||||
|
||||
func (jp *jsonPrettifier) closePipesWithError(err error) {
|
||||
_ = jp.pr.CloseWithError(err)
|
||||
_ = jp.pw.CloseWithError(err)
|
||||
}
|
||||
|
||||
func (jp *jsonPrettifier) prettifyJSONLines() error {
|
||||
for jp.d.More() {
|
||||
var v any
|
||||
if err := jp.d.Decode(&v); err != nil {
|
||||
return err
|
||||
}
|
||||
line, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("BUG: cannot marshal %v to JSON: %w", v, err))
|
||||
}
|
||||
if _, err := fmt.Fprintf(jp.pw, "%s\n", line); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (jp *jsonPrettifier) Close() error {
|
||||
jp.closePipesWithError(io.ErrUnexpectedEOF)
|
||||
err := jp.rOriginal.Close()
|
||||
jp.wg.Wait()
|
||||
return err
|
||||
}
|
||||
|
||||
func (jp *jsonPrettifier) Read(p []byte) (int, error) {
|
||||
return jp.pr.Read(p)
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"github.com/mattn/go-isatty"
|
||||
)
|
||||
|
||||
func isTerminal() bool {
|
||||
return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
|
||||
}
|
||||
|
||||
func readWithLess(r io.Reader) error {
|
||||
if !isTerminal() {
|
||||
// Just write everything to stdout if no terminal is available.
|
||||
_, err := io.Copy(os.Stdout, r)
|
||||
if err != nil && !isErrPipe(err) {
|
||||
return fmt.Errorf("error when forwarding data to stdout: %w", err)
|
||||
}
|
||||
if err := os.Stdout.Sync(); err != nil {
|
||||
return fmt.Errorf("cannot sync data to stdout: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
pr, pw, err := os.Pipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create pipe: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = pr.Close()
|
||||
_ = pw.Close()
|
||||
}()
|
||||
|
||||
// Ignore Ctrl+C in the current process, so 'less' could handle it properly
|
||||
cancel := ignoreSignals(os.Interrupt)
|
||||
defer cancel()
|
||||
|
||||
// Start 'less' process
|
||||
path, err := exec.LookPath("less")
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot find 'less' command: %w", err)
|
||||
}
|
||||
p, err := os.StartProcess(path, []string{"less", "-F", "-X"}, &os.ProcAttr{
|
||||
Env: append(os.Environ(), "LESSCHARSET=utf-8"),
|
||||
Files: []*os.File{pr, os.Stdout, os.Stderr},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot start 'less' process: %w", err)
|
||||
}
|
||||
|
||||
// Close pr after 'less' finishes in a parallel goroutine
|
||||
// in order to unblock forwarding data to stopped 'less' below.
|
||||
waitch := make(chan *os.ProcessState)
|
||||
go func() {
|
||||
// Wait for 'less' process to finish.
|
||||
ps, err := p.Wait()
|
||||
if err != nil {
|
||||
fatalf("unexpected error when waiting for 'less' process: %w", err)
|
||||
}
|
||||
_ = pr.Close()
|
||||
waitch <- ps
|
||||
}()
|
||||
|
||||
// Forward data from r to 'less'
|
||||
_, err = io.Copy(pw, r)
|
||||
_ = pw.Sync()
|
||||
_ = pw.Close()
|
||||
|
||||
// Wait until 'less' finished
|
||||
ps := <-waitch
|
||||
|
||||
// Verify 'less' status.
|
||||
if !ps.Success() {
|
||||
return fmt.Errorf("'less' finished with unexpected code %d", ps.ExitCode())
|
||||
}
|
||||
|
||||
if err != nil && !isErrPipe(err) {
|
||||
return fmt.Errorf("error when forwarding data to 'less': %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isErrPipe(err error) bool {
|
||||
return errors.Is(err, syscall.EPIPE) || errors.Is(err, io.ErrClosedPipe)
|
||||
}
|
||||
|
||||
func ignoreSignals(sigs ...os.Signal) func() {
|
||||
ch := make(chan os.Signal, 1)
|
||||
signal.Notify(ch, sigs...)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
_, ok := <-ch
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
signal.Stop(ch)
|
||||
close(ch)
|
||||
wg.Wait()
|
||||
}
|
||||
}
|
||||
@@ -1,301 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/ergochat/readline"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
)
|
||||
|
||||
var (
|
||||
datasourceURL = flag.String("datasource.url", "http://localhost:9428/select/logsql/query", "URL for querying VictoriaLogs; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/querying/#querying-logs")
|
||||
historyFile = flag.String("historyFile", "vlogscli-history", "Path to file with command history")
|
||||
header = flagutil.NewArrayString("header", "Optional header to pass in request -datasource.url in the form 'HeaderName: value'")
|
||||
)
|
||||
|
||||
const (
|
||||
firstLinePrompt = ";> "
|
||||
nextLinePrompt = ""
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Write flags and help message to stdout, since it is easier to grep or pipe.
|
||||
flag.CommandLine.SetOutput(os.Stdout)
|
||||
flag.Usage = usage
|
||||
envflag.Parse()
|
||||
buildinfo.Init()
|
||||
logger.InitNoLogFlags()
|
||||
|
||||
hes, err := parseHeaders(*header)
|
||||
if err != nil {
|
||||
fatalf("cannot parse -header command-line flag: %s", err)
|
||||
}
|
||||
headers = hes
|
||||
|
||||
isEmptyLine := true
|
||||
cfg := &readline.Config{
|
||||
Prompt: firstLinePrompt,
|
||||
DisableAutoSaveHistory: true,
|
||||
Listener: func(line []rune, pos int, _ rune) ([]rune, int, bool) {
|
||||
isEmptyLine = len(line) == 0
|
||||
return line, pos, false
|
||||
},
|
||||
}
|
||||
rl, err := readline.NewFromConfig(cfg)
|
||||
if err != nil {
|
||||
fatalf("cannot initialize readline: %s", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(rl, "sending queries to %s\n", *datasourceURL)
|
||||
|
||||
runReadlineLoop(rl, &isEmptyLine)
|
||||
|
||||
if err := rl.Close(); err != nil {
|
||||
fatalf("cannot close readline: %s", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func runReadlineLoop(rl *readline.Instance, isEmptyLine *bool) {
|
||||
historyLines, err := loadFromHistory(*historyFile)
|
||||
if err != nil {
|
||||
fatalf("cannot load query history: %s", err)
|
||||
}
|
||||
for _, line := range historyLines {
|
||||
if err := rl.SaveToHistory(line); err != nil {
|
||||
fatalf("cannot initialize query history: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
s := ""
|
||||
for {
|
||||
line, err := rl.ReadLine()
|
||||
if err != nil {
|
||||
switch err {
|
||||
case io.EOF:
|
||||
if s != "" {
|
||||
// This is non-interactive query execution.
|
||||
if err := executeQuery(context.Background(), rl, s); err != nil {
|
||||
fmt.Fprintf(rl, "%s\n", err)
|
||||
}
|
||||
}
|
||||
return
|
||||
case readline.ErrInterrupt:
|
||||
if s == "" && *isEmptyLine {
|
||||
fmt.Fprintf(rl, "interrupted\n")
|
||||
os.Exit(128 + int(syscall.SIGINT))
|
||||
}
|
||||
// Default value for Ctrl+C - clear the prompt
|
||||
s = ""
|
||||
rl.SetPrompt(firstLinePrompt)
|
||||
continue
|
||||
default:
|
||||
fatalf("unexpected error in readline: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
s += line
|
||||
if isQuitCommand(s) {
|
||||
fmt.Fprintf(rl, "bye!\n")
|
||||
return
|
||||
}
|
||||
if s == "" {
|
||||
// Skip empty lines
|
||||
continue
|
||||
}
|
||||
if line != "" && !strings.HasSuffix(line, ";") {
|
||||
// Assume the query is incomplete and allow the user finishing the query on the next line
|
||||
s += "\n"
|
||||
rl.SetPrompt(nextLinePrompt)
|
||||
continue
|
||||
}
|
||||
|
||||
// Execute the query
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
err = executeQuery(ctx, rl, s)
|
||||
cancel()
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
fmt.Fprintf(rl, "\n")
|
||||
} else {
|
||||
fmt.Fprintf(rl, "%s\n", err)
|
||||
}
|
||||
// Save queries in the history even if they weren't finished successfully
|
||||
}
|
||||
|
||||
s = strings.TrimSpace(s)
|
||||
if len(historyLines) == 0 || historyLines[len(historyLines)-1] != s {
|
||||
historyLines = append(historyLines, s)
|
||||
if len(historyLines) > 500 {
|
||||
historyLines = historyLines[len(historyLines)-500:]
|
||||
}
|
||||
if err := saveToHistory(*historyFile, historyLines); err != nil {
|
||||
fatalf("cannot save query history: %s", err)
|
||||
}
|
||||
}
|
||||
if err := rl.SaveToHistory(s); err != nil {
|
||||
fatalf("cannot update query history: %s", err)
|
||||
}
|
||||
s = ""
|
||||
rl.SetPrompt(firstLinePrompt)
|
||||
}
|
||||
}
|
||||
|
||||
func loadFromHistory(filePath string) ([]string, error) {
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
linesQuoted := strings.Split(string(data), "\n")
|
||||
lines := make([]string, 0, len(linesQuoted))
|
||||
i := 0
|
||||
for _, lineQuoted := range linesQuoted {
|
||||
i++
|
||||
if lineQuoted == "" {
|
||||
continue
|
||||
}
|
||||
line, err := strconv.Unquote(lineQuoted)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse line #%d at %s: %w; line: [%s]", i, filePath, err, line)
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
func saveToHistory(filePath string, lines []string) error {
|
||||
linesQuoted := make([]string, len(lines))
|
||||
for i, line := range lines {
|
||||
lineQuoted := strconv.Quote(line)
|
||||
linesQuoted[i] = lineQuoted
|
||||
}
|
||||
data := strings.Join(linesQuoted, "\n")
|
||||
return os.WriteFile(filePath, []byte(data), 0600)
|
||||
}
|
||||
|
||||
func isQuitCommand(s string) bool {
|
||||
switch s {
|
||||
case "q", "quit", "exit":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func executeQuery(ctx context.Context, output io.Writer, s string) error {
|
||||
// Parse the query and convert it to canonical view.
|
||||
s = strings.TrimSuffix(s, ";")
|
||||
q, err := logstorage.ParseQuery(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse query: %w", err)
|
||||
}
|
||||
qStr := q.String()
|
||||
fmt.Fprintf(output, "executing [%s]...", qStr)
|
||||
|
||||
// Prepare HTTP request for VictoriaLogs
|
||||
args := make(url.Values)
|
||||
args.Set("query", qStr)
|
||||
data := strings.NewReader(args.Encode())
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", *datasourceURL, data)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("BUG: cannot prepare request to server: %w", err))
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
for _, h := range headers {
|
||||
req.Header.Set(h.Name, h.Value)
|
||||
}
|
||||
|
||||
// Execute HTTP request at VictoriaLogs
|
||||
startTime := time.Now()
|
||||
resp, err := httpClient.Do(req)
|
||||
queryDuration := time.Since(startTime)
|
||||
fmt.Fprintf(output, "; duration: %.3fs\n", queryDuration.Seconds())
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot execute query: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
body = []byte(fmt.Sprintf("cannot read response body: %s", err))
|
||||
}
|
||||
return fmt.Errorf("unexpected status code: %d; response body:\n%s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
// Prettify the response and stream it to 'less'.
|
||||
jp := newJSONPrettifier(resp.Body)
|
||||
defer func() {
|
||||
_ = jp.Close()
|
||||
}()
|
||||
|
||||
if err := readWithLess(jp); err != nil {
|
||||
return fmt.Errorf("error when reading query response: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var httpClient = &http.Client{}
|
||||
|
||||
var headers []headerEntry
|
||||
|
||||
type headerEntry struct {
|
||||
Name string
|
||||
Value string
|
||||
}
|
||||
|
||||
func parseHeaders(a []string) ([]headerEntry, error) {
|
||||
hes := make([]headerEntry, len(a))
|
||||
for i, s := range a {
|
||||
a := strings.SplitN(s, ":", 2)
|
||||
if len(a) != 2 {
|
||||
return nil, fmt.Errorf("cannot parse header=%q; it must contain at least one ':'; for example, 'Cookie: foo'", s)
|
||||
}
|
||||
hes[i] = headerEntry{
|
||||
Name: strings.TrimSpace(a[0]),
|
||||
Value: strings.TrimSpace(a[1]),
|
||||
}
|
||||
}
|
||||
return hes, nil
|
||||
}
|
||||
|
||||
func fatalf(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func usage() {
|
||||
const s = `
|
||||
vlogscli is a command-line tool for querying VictoriaLogs.
|
||||
|
||||
See the docs at https://docs.victoriametrics.com/victorialogs/querying/vlogscli/
|
||||
`
|
||||
flagutil.Usage(s)
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
FROM $root_image
|
||||
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/vlogscli-prod"]
|
||||
ARG TARGETARCH
|
||||
COPY vlogscli-linux-${TARGETARCH}-prod ./vlogscli-prod
|
||||
@@ -230,7 +230,7 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
|
||||
for i := 0; i < activeStreams; i++ {
|
||||
ip := toIPv4(rand.Uint32())
|
||||
uuid := toUUID(rand.Uint64(), rand.Uint64())
|
||||
fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
|
||||
fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
|
||||
timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
|
||||
fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
|
||||
for j := 0; j < *constFieldsPerLog; j++ {
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -45,7 +44,6 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
if step <= 0 {
|
||||
httpserver.Errorf(w, r, "'step' must be bigger than zero")
|
||||
return
|
||||
}
|
||||
|
||||
// Obtain offset
|
||||
@@ -382,8 +380,6 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
// ProcessLiveTailRequest processes live tailing request to /select/logsq/tail
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#live-tailing
|
||||
func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
liveTailRequests.Inc()
|
||||
defer liveTailRequests.Dec()
|
||||
@@ -417,17 +413,13 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
if !ok {
|
||||
logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
|
||||
}
|
||||
qOrig := q
|
||||
for {
|
||||
start := end - tailOffsetNsecs
|
||||
end = time.Now().UnixNano()
|
||||
|
||||
q = qOrig.Clone(end)
|
||||
q.AddTimeFilter(start, end)
|
||||
// q.Optimize() call is needed for converting '*' into filterNoop.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6785#issuecomment-2358547733
|
||||
q.Optimize()
|
||||
if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, tp.writeBlock); err != nil {
|
||||
qCopy := q.Clone()
|
||||
qCopy.AddTimeFilter(start, end)
|
||||
if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, qCopy, tp.writeBlock); err != nil {
|
||||
httpserver.Errorf(w, r, "cannot execute tail query [%s]: %s", q, err)
|
||||
return
|
||||
}
|
||||
@@ -568,212 +560,9 @@ func (tp *tailProcessor) getTailRows() ([][]logstorage.Field, error) {
|
||||
return tailRows, nil
|
||||
}
|
||||
|
||||
// ProcessStatsQueryRangeRequest handles /select/logsql/stats_query_range request.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-range-stats
|
||||
func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
q, tenantIDs, err := parseCommonArgs(r)
|
||||
if err != nil {
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Obtain step
|
||||
stepStr := r.FormValue("step")
|
||||
if stepStr == "" {
|
||||
stepStr = "1d"
|
||||
}
|
||||
step, err := promutils.ParseDuration(stepStr)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("cannot parse 'step' arg: %s", err)
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
if step <= 0 {
|
||||
err := fmt.Errorf("'step' must be bigger than zero")
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Obtain `by(...)` fields from the last `| stats` pipe in q.
|
||||
// Add `_time:step` to the `by(...)` list.
|
||||
byFields, err := q.GetStatsByFieldsAddGroupingByTime(int64(step))
|
||||
if err != nil {
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
q.Optimize()
|
||||
|
||||
m := make(map[string]*statsSeries)
|
||||
var mLock sync.Mutex
|
||||
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
clonedColumnNames := make([]string, len(columns))
|
||||
for i, c := range columns {
|
||||
clonedColumnNames[i] = strings.Clone(c.Name)
|
||||
}
|
||||
for i := range timestamps {
|
||||
timestamp := q.GetTimestamp()
|
||||
labels := make([]logstorage.Field, 0, len(byFields))
|
||||
for j, c := range columns {
|
||||
if c.Name == "_time" {
|
||||
nsec, ok := logstorage.TryParseTimestampRFC3339Nano(c.Values[i])
|
||||
if ok {
|
||||
timestamp = nsec
|
||||
continue
|
||||
}
|
||||
}
|
||||
if slices.Contains(byFields, c.Name) {
|
||||
labels = append(labels, logstorage.Field{
|
||||
Name: clonedColumnNames[j],
|
||||
Value: strings.Clone(c.Values[i]),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var dst []byte
|
||||
for j, c := range columns {
|
||||
if !slices.Contains(byFields, c.Name) {
|
||||
name := clonedColumnNames[j]
|
||||
dst = dst[:0]
|
||||
dst = append(dst, name...)
|
||||
dst = logstorage.MarshalFieldsToJSON(dst, labels)
|
||||
key := string(dst)
|
||||
p := statsPoint{
|
||||
Timestamp: timestamp,
|
||||
Value: strings.Clone(c.Values[i]),
|
||||
}
|
||||
|
||||
mLock.Lock()
|
||||
ss := m[key]
|
||||
if ss == nil {
|
||||
ss = &statsSeries{
|
||||
key: key,
|
||||
Name: name,
|
||||
Labels: labels,
|
||||
}
|
||||
m[key] = ss
|
||||
}
|
||||
ss.Points = append(ss.Points, p)
|
||||
mLock.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
|
||||
err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Sort the collected stats by time
|
||||
rows := make([]*statsSeries, 0, len(m))
|
||||
for _, ss := range m {
|
||||
points := ss.Points
|
||||
sort.Slice(points, func(i, j int) bool {
|
||||
return points[i].Timestamp < points[j].Timestamp
|
||||
})
|
||||
rows = append(rows, ss)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
return rows[i].key < rows[j].key
|
||||
})
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
WriteStatsQueryRangeResponse(w, rows)
|
||||
}
|
||||
|
||||
type statsSeries struct {
|
||||
key string
|
||||
|
||||
Name string
|
||||
Labels []logstorage.Field
|
||||
Points []statsPoint
|
||||
}
|
||||
|
||||
type statsPoint struct {
|
||||
Timestamp int64
|
||||
Value string
|
||||
}
|
||||
|
||||
// ProcessStatsQueryRequest handles /select/logsql/stats_query request.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-stats
|
||||
func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
q, tenantIDs, err := parseCommonArgs(r)
|
||||
if err != nil {
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Obtain `by(...)` fields from the last `| stats` pipe in q.
|
||||
byFields, err := q.GetStatsByFields()
|
||||
if err != nil {
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
q.Optimize()
|
||||
|
||||
var rows []statsRow
|
||||
var rowsLock sync.Mutex
|
||||
|
||||
timestamp := q.GetTimestamp()
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
clonedColumnNames := make([]string, len(columns))
|
||||
for i, c := range columns {
|
||||
clonedColumnNames[i] = strings.Clone(c.Name)
|
||||
}
|
||||
for i := range timestamps {
|
||||
labels := make([]logstorage.Field, 0, len(byFields))
|
||||
for j, c := range columns {
|
||||
if slices.Contains(byFields, c.Name) {
|
||||
labels = append(labels, logstorage.Field{
|
||||
Name: clonedColumnNames[j],
|
||||
Value: strings.Clone(c.Values[i]),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for j, c := range columns {
|
||||
if !slices.Contains(byFields, c.Name) {
|
||||
r := statsRow{
|
||||
Name: clonedColumnNames[j],
|
||||
Labels: labels,
|
||||
Timestamp: timestamp,
|
||||
Value: strings.Clone(c.Values[i]),
|
||||
}
|
||||
|
||||
rowsLock.Lock()
|
||||
rows = append(rows, r)
|
||||
rowsLock.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
|
||||
err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
WriteStatsQueryResponse(w, rows)
|
||||
}
|
||||
|
||||
type statsRow struct {
|
||||
Name string
|
||||
Labels []logstorage.Field
|
||||
Timestamp int64
|
||||
Value string
|
||||
}
|
||||
|
||||
// ProcessQueryRequest handles /select/logsql/query request.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#querying-logs
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#http-api
|
||||
func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
q, tenantIDs, err := parseCommonArgs(r)
|
||||
if err != nil {
|
||||
@@ -848,7 +637,6 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
|
||||
limitUpper := 2 * limit
|
||||
q.AddPipeLimit(uint64(limitUpper))
|
||||
q.Optimize()
|
||||
|
||||
rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -859,62 +647,32 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// Slow path - adjust time range for selecting up to limitUpper rows
|
||||
// Slow path - search for the time range containing up to limitUpper rows.
|
||||
start, end := q.GetFilterTimeRange()
|
||||
d := end/2 - start/2
|
||||
start += d
|
||||
|
||||
qOrig := q
|
||||
for {
|
||||
timestamp := qOrig.GetTimestamp()
|
||||
q = qOrig.Clone(timestamp)
|
||||
q = qOrig.Clone()
|
||||
q.AddTimeFilter(start, end)
|
||||
// q.Optimize() call is needed for converting '*' into filterNoop.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6785#issuecomment-2358547733
|
||||
q.Optimize()
|
||||
rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if d == 0 || start >= end {
|
||||
// The [start ... end] time range equals one nanosecond.
|
||||
// Just return up to limit rows.
|
||||
if len(rows) > limit {
|
||||
rows = rows[:limit]
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
dLastBit := d & 1
|
||||
d /= 2
|
||||
|
||||
if len(rows) >= limitUpper {
|
||||
// The number of found rows on the [start ... end] time range exceeds limitUpper,
|
||||
// so reduce the time range to [start+d ... end].
|
||||
start += d
|
||||
continue
|
||||
}
|
||||
if len(rows) >= limit {
|
||||
// The number of found rows is in the range [limit ... limitUpper).
|
||||
// This means that found rows contains the needed limit rows with the biggest timestamps.
|
||||
if len(rows) >= limit && len(rows) < limitUpper || d == 0 {
|
||||
rows = getLastNRows(rows, limit)
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// The number of found rows on [start ... end] time range is below the limit.
|
||||
// This means the time range doesn't cover the needed logs, so it must be extended.
|
||||
|
||||
if len(rows) == 0 {
|
||||
// The [start ... end] time range doesn't contain any rows, so change it to [start-d ... start).
|
||||
end = start - 1
|
||||
start -= d + dLastBit
|
||||
continue
|
||||
lastBit := d & 1
|
||||
d /= 2
|
||||
if len(rows) > limit {
|
||||
start += d
|
||||
} else {
|
||||
start -= d + lastBit
|
||||
}
|
||||
|
||||
// The number of found rows on [start ... end] time range is bigger than 0 but smaller than limit.
|
||||
// Increase the time range to [start-d ... end].
|
||||
start -= d + dLastBit
|
||||
}
|
||||
}
|
||||
|
||||
@@ -935,25 +693,20 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
|
||||
var rows []row
|
||||
var rowsLock sync.Mutex
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
clonedColumnNames := make([]string, len(columns))
|
||||
for i, c := range columns {
|
||||
clonedColumnNames[i] = strings.Clone(c.Name)
|
||||
}
|
||||
rowsLock.Lock()
|
||||
defer rowsLock.Unlock()
|
||||
|
||||
for i, timestamp := range timestamps {
|
||||
fields := make([]logstorage.Field, len(columns))
|
||||
for j := range columns {
|
||||
f := &fields[j]
|
||||
f.Name = clonedColumnNames[j]
|
||||
f.Name = strings.Clone(columns[j].Name)
|
||||
f.Value = strings.Clone(columns[j].Values[i])
|
||||
}
|
||||
|
||||
rowsLock.Lock()
|
||||
rows = append(rows, row{
|
||||
timestamp: timestamp,
|
||||
fields: fields,
|
||||
})
|
||||
rowsLock.Unlock()
|
||||
}
|
||||
|
||||
if len(rows) >= limit {
|
||||
@@ -975,23 +728,9 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
|
||||
}
|
||||
tenantIDs := []logstorage.TenantID{tenantID}
|
||||
|
||||
// Parse optional time arg
|
||||
timestamp, okTime, err := getTimeNsec(r, "time")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !okTime {
|
||||
// If time arg is missing, then evaluate query at the current timestamp
|
||||
timestamp = time.Now().UnixNano()
|
||||
}
|
||||
|
||||
// decrease timestamp by one nanosecond in order to avoid capturing logs belonging
|
||||
// to the first nanosecond at the next period of time (month, week, day, hour, etc.)
|
||||
timestamp--
|
||||
|
||||
// Parse query
|
||||
qStr := r.FormValue("query")
|
||||
q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
|
||||
q, err := logstorage.ParseQuery(qStr)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
{% stripspace %}
|
||||
|
||||
// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
|
||||
{% func StatsQueryRangeResponse(rows []*statsSeries) %}
|
||||
{
|
||||
"status":"success",
|
||||
"data":{
|
||||
"resultType":"matrix",
|
||||
"result":[
|
||||
{% if len(rows) > 0 %}
|
||||
{%= formatStatsSeries(rows[0]) %}
|
||||
{% code rows = rows[1:] %}
|
||||
{% for i := range rows %}
|
||||
,{%= formatStatsSeries(rows[i]) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
]
|
||||
}
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% func formatStatsSeries(ss *statsSeries) %}
|
||||
{
|
||||
"metric":{
|
||||
"__name__":{%q= ss.Name %}
|
||||
{% if len(ss.Labels) > 0 %}
|
||||
{% for _, label := range ss.Labels %}
|
||||
,{%q= label.Name %}:{%q= label.Value %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
},
|
||||
"values":[
|
||||
{% code points := ss.Points %}
|
||||
{% if len(points) > 0 %}
|
||||
{%= formatStatsPoint(&points[0]) %}
|
||||
{% code points = points[1:] %}
|
||||
{% for i := range points %}
|
||||
,{%= formatStatsPoint(&points[i]) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
]
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% func formatStatsPoint(p *statsPoint) %}
|
||||
[
|
||||
{%f= float64(p.Timestamp)/1e9 %},
|
||||
{%q= p.Value %}
|
||||
]
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
@@ -1,188 +0,0 @@
|
||||
// Code generated by qtc from "stats_query_range_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
package logsql
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
func StreamStatsQueryRangeResponse(qw422016 *qt422016.Writer, rows []*statsSeries) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:10
|
||||
if len(rows) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:11
|
||||
streamformatStatsSeries(qw422016, rows[0])
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:12
|
||||
rows = rows[1:]
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:13
|
||||
for i := range rows {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:13
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:14
|
||||
streamformatStatsSeries(qw422016, rows[i])
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:15
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:16
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:16
|
||||
qw422016.N().S(`]}}`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
func WriteStatsQueryRangeResponse(qq422016 qtio422016.Writer, rows []*statsSeries) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
StreamStatsQueryRangeResponse(qw422016, rows)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
func StatsQueryRangeResponse(rows []*statsSeries) string {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
WriteStatsQueryRangeResponse(qb422016, rows)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:22
|
||||
func streamformatStatsSeries(qw422016 *qt422016.Writer, ss *statsSeries) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:22
|
||||
qw422016.N().S(`{"metric":{"__name__":`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:25
|
||||
qw422016.N().Q(ss.Name)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:26
|
||||
if len(ss.Labels) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:27
|
||||
for _, label := range ss.Labels {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:27
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:28
|
||||
qw422016.N().Q(label.Name)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:28
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:28
|
||||
qw422016.N().Q(label.Value)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:29
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:30
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:30
|
||||
qw422016.N().S(`},"values":[`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:33
|
||||
points := ss.Points
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:34
|
||||
if len(points) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:35
|
||||
streamformatStatsPoint(qw422016, &points[0])
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:36
|
||||
points = points[1:]
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:37
|
||||
for i := range points {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:37
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:38
|
||||
streamformatStatsPoint(qw422016, &points[i])
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:39
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:40
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:40
|
||||
qw422016.N().S(`]}`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
func writeformatStatsSeries(qq422016 qtio422016.Writer, ss *statsSeries) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
streamformatStatsSeries(qw422016, ss)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
func formatStatsSeries(ss *statsSeries) string {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
writeformatStatsSeries(qb422016, ss)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:45
|
||||
func streamformatStatsPoint(qw422016 *qt422016.Writer, p *statsPoint) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:45
|
||||
qw422016.N().S(`[`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:47
|
||||
qw422016.N().F(float64(p.Timestamp) / 1e9)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:47
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:48
|
||||
qw422016.N().Q(p.Value)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:48
|
||||
qw422016.N().S(`]`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
func writeformatStatsPoint(qq422016 qtio422016.Writer, p *statsPoint) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
streamformatStatsPoint(qw422016, p)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
func formatStatsPoint(p *statsPoint) string {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
writeformatStatsPoint(qb422016, p)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
{% stripspace %}
|
||||
|
||||
// StatsQueryResponse generates response for /select/logsql/stats_query
|
||||
{% func StatsQueryResponse(rows []statsRow) %}
|
||||
{
|
||||
"status":"success",
|
||||
"data":{
|
||||
"resultType":"vector",
|
||||
"result":[
|
||||
{% if len(rows) > 0 %}
|
||||
{%= formatStatsRow(&rows[0]) %}
|
||||
{% code rows = rows[1:] %}
|
||||
{% for i := range rows %}
|
||||
,{%= formatStatsRow(&rows[i]) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
]
|
||||
}
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% func formatStatsRow(r *statsRow) %}
|
||||
{
|
||||
"metric":{
|
||||
"__name__":{%q= r.Name %}
|
||||
{% if len(r.Labels) > 0 %}
|
||||
{% for _, label := range r.Labels %}
|
||||
,{%q= label.Name %}:{%q= label.Value %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
},
|
||||
"value":[{%f= float64(r.Timestamp)/1e9 %},{%q= r.Value %}]
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
@@ -1,133 +0,0 @@
|
||||
// Code generated by qtc from "stats_query_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
// StatsQueryResponse generates response for /select/logsql/stats_query
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
package logsql
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
func StreamStatsQueryResponse(qw422016 *qt422016.Writer, rows []statsRow) {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
qw422016.N().S(`{"status":"success","data":{"resultType":"vector","result":[`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:10
|
||||
if len(rows) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:11
|
||||
streamformatStatsRow(qw422016, &rows[0])
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:12
|
||||
rows = rows[1:]
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:13
|
||||
for i := range rows {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:13
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:14
|
||||
streamformatStatsRow(qw422016, &rows[i])
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:15
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:16
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:16
|
||||
qw422016.N().S(`]}}`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
func WriteStatsQueryResponse(qq422016 qtio422016.Writer, rows []statsRow) {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
StreamStatsQueryResponse(qw422016, rows)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
func StatsQueryResponse(rows []statsRow) string {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
WriteStatsQueryResponse(qb422016, rows)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:22
|
||||
func streamformatStatsRow(qw422016 *qt422016.Writer, r *statsRow) {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:22
|
||||
qw422016.N().S(`{"metric":{"__name__":`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:25
|
||||
qw422016.N().Q(r.Name)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:26
|
||||
if len(r.Labels) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:27
|
||||
for _, label := range r.Labels {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:27
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:28
|
||||
qw422016.N().Q(label.Name)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:28
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:28
|
||||
qw422016.N().Q(label.Value)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:29
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:30
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:30
|
||||
qw422016.N().S(`},"value":[`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:32
|
||||
qw422016.N().F(float64(r.Timestamp) / 1e9)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:32
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:32
|
||||
qw422016.N().Q(r.Value)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:32
|
||||
qw422016.N().S(`]}`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
func writeformatStatsRow(qq422016 qtio422016.Writer, r *statsRow) {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
streamformatStatsRow(qw422016, r)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
func formatStatsRow(r *statsRow) string {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
writeformatStatsRow(qb422016, r)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
}
|
||||
@@ -193,14 +193,6 @@ func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
|
||||
logsqlQueryRequests.Inc()
|
||||
logsql.ProcessQueryRequest(ctx, w, r)
|
||||
return true
|
||||
case "/select/logsql/stats_query":
|
||||
logsqlStatsQueryRequests.Inc()
|
||||
logsql.ProcessStatsQueryRequest(ctx, w, r)
|
||||
return true
|
||||
case "/select/logsql/stats_query_range":
|
||||
logsqlStatsQueryRangeRequests.Inc()
|
||||
logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
|
||||
return true
|
||||
case "/select/logsql/stream_field_names":
|
||||
logsqlStreamFieldNamesRequests.Inc()
|
||||
logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
|
||||
@@ -240,8 +232,6 @@ var (
|
||||
logsqlFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
|
||||
logsqlHitsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
|
||||
logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
|
||||
logsqlStatsQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
|
||||
logsqlStatsQueryRangeRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
|
||||
logsqlStreamFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
|
||||
logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
|
||||
logsqlStreamIDsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
|
||||
|
||||
BIN
app/vlselect/vmui/apple-touch-icon.png
Normal file
BIN
app/vlselect/vmui/apple-touch-icon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"files": {
|
||||
"main.css": "./static/css/main.cbbca000.css",
|
||||
"main.js": "./static/js/main.3d2eb957.js",
|
||||
"static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
|
||||
"static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
|
||||
"main.css": "./static/css/main.1041c3d4.css",
|
||||
"main.js": "./static/js/main.62a82f4a.js",
|
||||
"static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
|
||||
"static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
|
||||
"index.html": "./index.html"
|
||||
},
|
||||
"entrypoints": [
|
||||
"static/css/main.cbbca000.css",
|
||||
"static/js/main.3d2eb957.js"
|
||||
"static/css/main.1041c3d4.css",
|
||||
"static/js/main.62a82f4a.js"
|
||||
]
|
||||
}
|
||||
BIN
app/vlselect/vmui/favicon-32x32.png
Normal file
BIN
app/vlselect/vmui/favicon-32x32.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
@@ -1 +0,0 @@
|
||||
<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB |
@@ -1 +1 @@
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.3d2eb957.js"></script><link href="./static/css/main.cbbca000.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.62a82f4a.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
@@ -1,11 +1,16 @@
|
||||
{
|
||||
"short_name": "vmui",
|
||||
"name": "vmui",
|
||||
"short_name": "Victoria Metrics UI",
|
||||
"name": "Victoria Metrics UI is a metric explorer for Victoria Metrics",
|
||||
"icons": [
|
||||
{
|
||||
"src": "favicon.svg",
|
||||
"sizes": "any",
|
||||
"type": "image/svg+xml"
|
||||
"src": "favicon-32x32.png",
|
||||
"sizes": "32x32",
|
||||
"type": "image/png"
|
||||
},
|
||||
{
|
||||
"src": "apple-touch-icon.png",
|
||||
"type": "image/png",
|
||||
"sizes": "192x192"
|
||||
}
|
||||
],
|
||||
"start_url": ".",
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 6.5 KiB After Width: | Height: | Size: 67 KiB |
1
app/vlselect/vmui/static/css/main.1041c3d4.css
Normal file
1
app/vlselect/vmui/static/css/main.1041c3d4.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/vlselect/vmui/static/js/685.bebe1265.chunk.js
Normal file
1
app/vlselect/vmui/static/js/685.bebe1265.chunk.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
2
app/vlselect/vmui/static/js/main.62a82f4a.js
Normal file
2
app/vlselect/vmui/static/js/main.62a82f4a.js
Normal file
File diff suppressed because one or more lines are too long
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
|
||||
/**
|
||||
* @remix-run/router v1.19.2
|
||||
* @remix-run/router v1.17.0
|
||||
*
|
||||
* Copyright (c) Remix Software Inc.
|
||||
*
|
||||
@@ -16,7 +16,7 @@
|
||||
*/
|
||||
|
||||
/**
|
||||
* React Router DOM v6.26.2
|
||||
* React Router DOM v6.24.0
|
||||
*
|
||||
* Copyright (c) Remix Software Inc.
|
||||
*
|
||||
@@ -27,7 +27,7 @@
|
||||
*/
|
||||
|
||||
/**
|
||||
* React Router v6.26.2
|
||||
* React Router v6.24.0
|
||||
*
|
||||
* Copyright (c) Remix Software Inc.
|
||||
*
|
||||
@@ -1,4 +1,5 @@
|
||||
---
|
||||
sort: 23
|
||||
weight: 23
|
||||
title: MetricsQL
|
||||
menu:
|
||||
@@ -9,6 +10,9 @@ aliases:
|
||||
- /ExtendedPromQL.html
|
||||
- /MetricsQL.html
|
||||
---
|
||||
|
||||
# MetricsQL
|
||||
|
||||
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
|
||||
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
|
||||
MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
|
||||
@@ -103,7 +107,7 @@ The list of MetricsQL features on top of PromQL:
|
||||
* Trailing commas on all the lists are allowed - label filters, function args and with expressions.
|
||||
For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`.
|
||||
This simplifies maintenance of multi-line queries.
|
||||
* Metric names and label names may contain any unicode letter. For example `ტემპერატურა{πόλη="Київ"}` is a valid MetricsQL expression.
|
||||
* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a valid MetricsQL expression.
|
||||
* Metric names and labels names may contain escaped chars. For example, `foo\-bar{baz\=aa="b"}` is valid expression.
|
||||
It returns time series with name `foo-bar` containing label `baz=aa` with value `b`.
|
||||
Additionally, the following escape sequences are supported:
|
||||
@@ -216,11 +220,9 @@ See also [descent_over_time](#descent_over_time).
|
||||
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
|
||||
See also [median_over_time](#median_over_time).
|
||||
|
||||
#### changes
|
||||
|
||||
@@ -260,8 +262,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
|
||||
|
||||
#### count_gt_over_time
|
||||
@@ -272,8 +272,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
|
||||
|
||||
#### count_le_over_time
|
||||
@@ -284,8 +282,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
|
||||
|
||||
#### count_ne_over_time
|
||||
@@ -296,9 +292,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
|
||||
See also [count_over_time](#count_over_time).
|
||||
|
||||
#### count_over_time
|
||||
|
||||
@@ -319,8 +313,6 @@ The results are calculated independently per each time series returned from the
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
|
||||
|
||||
#### decreases_over_time
|
||||
@@ -440,8 +432,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### histogram_over_time
|
||||
|
||||
`histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
@@ -453,15 +443,11 @@ For example, the following query calculates median temperature by country over t
|
||||
|
||||
`histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### hoeffding_bound_lower
|
||||
|
||||
`hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [hoeffding_bound_upper](#hoeffding_bound_upper).
|
||||
|
||||
#### hoeffding_bound_upper
|
||||
@@ -469,8 +455,6 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
|
||||
`hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [hoeffding_bound_lower](#hoeffding_bound_lower).
|
||||
|
||||
#### holt_winters
|
||||
@@ -478,9 +462,8 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
|
||||
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
|
||||
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
|
||||
Both `sf` and `tf` must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
||||
returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
@@ -512,14 +495,13 @@ See also [deriv](#deriv).
|
||||
|
||||
`increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
|
||||
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
|
||||
@@ -528,13 +510,12 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
|
||||
|
||||
`increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
|
||||
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [increase_pure](#increase_pure) and [increase](#increase).
|
||||
|
||||
#### increase_pure
|
||||
@@ -543,10 +524,6 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
|
||||
of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
|
||||
while [increase](#increase) ignores the first value in a series if it is too big.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [increase](#increase) and [increase_prometheus](#increase_prometheus).
|
||||
|
||||
#### increases_over_time
|
||||
|
||||
`increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -563,18 +540,15 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### irate
|
||||
|
||||
`irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
|
||||
the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [rate](#rate) and [rollup_rate](#rollup_rate).
|
||||
@@ -613,8 +587,6 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
|
||||
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
|
||||
|
||||
#### max_over_time
|
||||
@@ -622,11 +594,9 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
|
||||
`max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
|
||||
See also [tmax_over_time](#tmax_over_time).
|
||||
|
||||
#### median_over_time
|
||||
|
||||
@@ -634,8 +604,6 @@ See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
|
||||
on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [avg_over_time](#avg_over_time).
|
||||
|
||||
#### min_over_time
|
||||
@@ -643,11 +611,9 @@ See also [avg_over_time](#avg_over_time).
|
||||
`min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
|
||||
See also [tmin_over_time](#tmin_over_time).
|
||||
|
||||
#### mode_over_time
|
||||
|
||||
@@ -656,8 +622,6 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
values are discrete.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### outlier_iqr_over_time
|
||||
|
||||
`outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
|
||||
@@ -668,8 +632,6 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
|
||||
The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
|
||||
For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [outliers_iqr](#outliers_iqr).
|
||||
|
||||
#### predict_linear
|
||||
@@ -697,8 +659,6 @@ This function is supported by PromQL.
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
The `phi` value must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [quantiles_over_time](#quantiles_over_time).
|
||||
@@ -710,8 +670,6 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [quantile_over_time](#quantile_over_time).
|
||||
|
||||
#### range_over_time
|
||||
@@ -722,12 +680,11 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### rate
|
||||
|
||||
`rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
|
||||
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
|
||||
passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
|
||||
@@ -746,8 +703,6 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
on the given lookbehind window `d`. The calculations are performed individually per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
#### resets
|
||||
@@ -755,11 +710,10 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
`resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
|
||||
of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
|
||||
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
#### rollup
|
||||
@@ -771,10 +725,6 @@ These values are calculated individually per each time series returned from the
|
||||
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_candlestick
|
||||
|
||||
`rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
|
||||
@@ -786,8 +736,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### rollup_delta
|
||||
|
||||
`rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -814,8 +762,6 @@ See also [label_match](#label_match).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_increase
|
||||
|
||||
`rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -828,17 +774,12 @@ See also [label_match](#label_match).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_rate
|
||||
|
||||
`rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
|
||||
for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
|
||||
and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
|
||||
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
|
||||
when to use `rollup_rate()`.
|
||||
@@ -846,12 +787,10 @@ when to use `rollup_rate()`.
|
||||
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
|
||||
|
||||
#### rollup_scrape_interval
|
||||
|
||||
`rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
|
||||
@@ -885,8 +824,6 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
|
||||
|
||||
#### share_le_over_time
|
||||
@@ -901,8 +838,6 @@ the share of time series values for the last 24 hours when memory usage was belo
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
|
||||
|
||||
#### share_eq_over_time
|
||||
@@ -914,8 +849,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### stale_samples_over_time
|
||||
@@ -933,8 +866,6 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [stdvar_over_time](#stdvar_over_time).
|
||||
@@ -946,43 +877,35 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [stddev_over_time](#stddev_over_time).
|
||||
|
||||
#### sum_eq_over_time
|
||||
|
||||
`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
values equal to `eq` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### sum_gt_over_time
|
||||
|
||||
`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
values bigger than `gt` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
|
||||
|
||||
#### sum_le_over_time
|
||||
|
||||
`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
values smaller or equal to `le` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
|
||||
|
||||
#### sum_over_time
|
||||
@@ -992,8 +915,6 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
#### sum2_over_time
|
||||
@@ -1003,8 +924,6 @@ values on the given lookbehind window `d` per each time series returned from the
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### timestamp
|
||||
|
||||
`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
|
||||
@@ -1082,8 +1001,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
|
||||
|
||||
|
||||
@@ -2313,8 +2230,7 @@ Any [rollup function](#rollup-functions) for something other than [series select
|
||||
Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
|
||||
For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
|
||||
since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
|
||||
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
|
||||
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
|
||||
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
|
||||
|
||||
VictoriaMetrics performs subqueries in the following way:
|
||||
|
||||
@@ -2347,6 +2263,5 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
|
||||
For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
|
||||
* If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
||||
is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
|
||||
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
|
||||
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
|
||||
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
|
||||
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
|
||||
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
|
||||
@@ -1,8 +1,8 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 8429
|
||||
|
||||
ENTRYPOINT ["/vmagent-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./vmagent-prod
|
||||
|
||||
@@ -10,8 +10,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
|
||||
@@ -44,7 +42,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -318,10 +316,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
influxQueryRequests.Inc()
|
||||
influxutils.WriteDatabaseNames(w)
|
||||
return true
|
||||
case "/influx/health":
|
||||
influxHealthRequests.Inc()
|
||||
influxutils.WriteHealthCheckResponse(w)
|
||||
return true
|
||||
case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
|
||||
opentelemetryPushRequests.Inc()
|
||||
if err := opentelemetry.InsertHandler(nil, r); err != nil {
|
||||
@@ -456,7 +450,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
var bb bytesutil.ByteBuffer
|
||||
promscrape.WriteConfigData(&bb)
|
||||
fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
|
||||
fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
|
||||
return true
|
||||
case "/prometheus/-/reload", "/-/reload":
|
||||
if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
|
||||
@@ -568,10 +562,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
|
||||
influxQueryRequests.Inc()
|
||||
influxutils.WriteDatabaseNames(w)
|
||||
return true
|
||||
case "influx/health":
|
||||
influxHealthRequests.Inc()
|
||||
influxutils.WriteHealthCheckResponse(w)
|
||||
return true
|
||||
case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
|
||||
opentelemetryPushRequests.Inc()
|
||||
if err := opentelemetry.InsertHandler(at, r); err != nil {
|
||||
@@ -682,8 +672,7 @@ var (
|
||||
influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
|
||||
influxWriteErrors = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
|
||||
|
||||
influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
|
||||
influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
|
||||
influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
|
||||
|
||||
datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
|
||||
datadogv1WriteErrors = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -7,11 +7,12 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
@@ -22,7 +23,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -34,10 +34,8 @@ var (
|
||||
rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
|
||||
"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
|
||||
"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
|
||||
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
|
||||
retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxInterval")
|
||||
retryMaxTime = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
|
||||
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
|
||||
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
|
||||
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
|
||||
"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
|
||||
|
||||
tlsHandshakeTimeout = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
|
||||
@@ -92,9 +90,6 @@ type client struct {
|
||||
fq *persistentqueue.FastQueue
|
||||
hc *http.Client
|
||||
|
||||
retryMinInterval time.Duration
|
||||
retryMaxTime time.Duration
|
||||
|
||||
sendBlock func(block []byte) bool
|
||||
authCfg *promauth.Config
|
||||
awsCfg *awsapi.Config
|
||||
@@ -148,15 +143,13 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
|
||||
Timeout: sendTimeout.GetOptionalArg(argIdx),
|
||||
}
|
||||
c := &client{
|
||||
sanitizedURL: sanitizedURL,
|
||||
remoteWriteURL: remoteWriteURL,
|
||||
authCfg: authCfg,
|
||||
awsCfg: awsCfg,
|
||||
fq: fq,
|
||||
hc: hc,
|
||||
retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
|
||||
retryMaxTime: retryMaxTime.GetOptionalArg(argIdx),
|
||||
stopCh: make(chan struct{}),
|
||||
sanitizedURL: sanitizedURL,
|
||||
remoteWriteURL: remoteWriteURL,
|
||||
authCfg: authCfg,
|
||||
awsCfg: awsCfg,
|
||||
fq: fq,
|
||||
hc: hc,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
c.sendBlock = c.sendBlockHTTP
|
||||
|
||||
@@ -403,11 +396,11 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
|
||||
// sendBlockHTTP sends the given block to c.remoteWriteURL.
|
||||
//
|
||||
// The function returns false only if c.stopCh is closed.
|
||||
// Otherwise, it tries sending the block to remote storage indefinitely.
|
||||
// Otherwise it tries sending the block to remote storage indefinitely.
|
||||
func (c *client) sendBlockHTTP(block []byte) bool {
|
||||
c.rl.Register(len(block))
|
||||
maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
|
||||
retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
|
||||
maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
|
||||
retryDuration := timeutil.AddJitterToDuration(time.Second)
|
||||
retriesCount := 0
|
||||
|
||||
again:
|
||||
@@ -463,10 +456,10 @@ again:
|
||||
|
||||
// Unexpected status code returned
|
||||
retriesCount++
|
||||
retryAfterHeader := parseRetryAfterHeader(resp.Header.Get("Retry-After"))
|
||||
retryDuration = getRetryDuration(retryAfterHeader, retryDuration, maxRetryDuration)
|
||||
|
||||
// Handle response
|
||||
retryDuration *= 2
|
||||
if retryDuration > maxRetryDuration {
|
||||
retryDuration = maxRetryDuration
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
@@ -488,49 +481,3 @@ again:
|
||||
}
|
||||
|
||||
var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
|
||||
|
||||
// getRetryDuration returns retry duration.
|
||||
// retryAfterDuration has the highest priority.
|
||||
// If retryAfterDuration is not specified, retryDuration gets doubled.
|
||||
// retryDuration can't exceed maxRetryDuration.
|
||||
//
|
||||
// Also see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6097
|
||||
func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.Duration) time.Duration {
|
||||
// retryAfterDuration has the highest priority duration
|
||||
if retryAfterDuration > 0 {
|
||||
return timeutil.AddJitterToDuration(retryAfterDuration)
|
||||
}
|
||||
|
||||
// default backoff retry policy
|
||||
retryDuration *= 2
|
||||
if retryDuration > maxRetryDuration {
|
||||
retryDuration = maxRetryDuration
|
||||
}
|
||||
|
||||
return retryDuration
|
||||
}
|
||||
|
||||
// parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
|
||||
// retryAfterString should be in either HTTP-date or a number of seconds.
|
||||
// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
|
||||
func parseRetryAfterHeader(retryAfterString string) (retryAfterDuration time.Duration) {
|
||||
if retryAfterString == "" {
|
||||
return retryAfterDuration
|
||||
}
|
||||
|
||||
defer func() {
|
||||
v := retryAfterDuration.Seconds()
|
||||
logger.Infof("'Retry-After: %s' parsed into %.2f second(s)", retryAfterString, v)
|
||||
}()
|
||||
|
||||
// Retry-After could be in "Mon, 02 Jan 2006 15:04:05 GMT" format.
|
||||
if parsedTime, err := time.Parse(http.TimeFormat, retryAfterString); err == nil {
|
||||
return time.Duration(time.Until(parsedTime).Seconds()) * time.Second
|
||||
}
|
||||
// Retry-After could be in seconds.
|
||||
if seconds, err := strconv.Atoi(retryAfterString); err == nil {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"math"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCalculateRetryDuration(t *testing.T) {
|
||||
// `testFunc` call `calculateRetryDuration` for `n` times
|
||||
// and evaluate if the result of `calculateRetryDuration` is
|
||||
// 1. >= expectMinDuration
|
||||
// 2. <= expectMinDuration + 10% (see timeutil.AddJitterToDuration)
|
||||
f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
|
||||
}
|
||||
|
||||
expectMaxDuration := helper(expectMinDuration)
|
||||
expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
|
||||
|
||||
if !(retryDuration >= expectMinDuration && retryDuration <= expectMaxDuration) {
|
||||
t.Fatalf(
|
||||
"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
|
||||
expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
|
||||
retryDuration.Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Call calculateRetryDuration for 1 time.
|
||||
{
|
||||
// default backoff policy
|
||||
f(0, time.Second, 1, 2*time.Second)
|
||||
// default backoff policy exceed max limit"
|
||||
f(0, 10*time.Minute, 1, time.Minute)
|
||||
|
||||
// retry after > default backoff policy
|
||||
f(10*time.Second, 1*time.Second, 1, 10*time.Second)
|
||||
// retry after < default backoff policy
|
||||
f(1*time.Second, 10*time.Second, 1, 1*time.Second)
|
||||
// retry after invalid and < default backoff policy
|
||||
f(0, time.Second, 1, 2*time.Second)
|
||||
|
||||
}
|
||||
|
||||
// Call calculateRetryDuration for multiple times.
|
||||
{
|
||||
// default backoff policy 2 times
|
||||
f(0, time.Second, 2, 4*time.Second)
|
||||
// default backoff policy 3 times
|
||||
f(0, time.Second, 3, 8*time.Second)
|
||||
// default backoff policy N times exceed max limit
|
||||
f(0, time.Second, 10, time.Minute)
|
||||
|
||||
// retry after 120s 1 times
|
||||
f(120*time.Second, time.Second, 1, 120*time.Second)
|
||||
// retry after 120s 2 times
|
||||
f(120*time.Second, time.Second, 2, 120*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRetryAfterHeader(t *testing.T) {
|
||||
f := func(retryAfterString string, expectResult time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
result := parseRetryAfterHeader(retryAfterString)
|
||||
// expect `expectResult == result` when retryAfterString is in seconds or invalid
|
||||
// expect the difference between result and expectResult to be lower than 10%
|
||||
if !(expectResult == result || math.Abs(float64(expectResult-result))/float64(expectResult) < 0.10) {
|
||||
t.Fatalf(
|
||||
"incorrect retry after duration, want (ms): %d, got (ms): %d",
|
||||
expectResult.Milliseconds(), result.Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// retry after header in seconds
|
||||
f("10", 10*time.Second)
|
||||
// retry after header in date time
|
||||
f(time.Now().Add(30*time.Second).UTC().Format(http.TimeFormat), 30*time.Second)
|
||||
// retry after header invalid
|
||||
f("invalid-retry-after", 0)
|
||||
// retry after header not in GMT
|
||||
f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
|
||||
}
|
||||
|
||||
// helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
|
||||
func helper(d time.Duration) time.Duration {
|
||||
dv := d / 10
|
||||
if dv > 10*time.Second {
|
||||
dv = 10 * time.Second
|
||||
}
|
||||
|
||||
return d + dv
|
||||
}
|
||||
@@ -441,7 +441,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
|
||||
var rctx *relabelCtx
|
||||
rcs := allRelabelConfigs.Load()
|
||||
pcsGlobal := rcs.global
|
||||
if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
|
||||
if pcsGlobal.Len() > 0 {
|
||||
rctx = getRelabelCtx()
|
||||
defer putRelabelCtx(rctx)
|
||||
}
|
||||
@@ -494,8 +494,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
|
||||
tssBlock = dropAggregatedSeries(tssBlock, matchIdxs.B, *streamAggrGlobalDropInput)
|
||||
}
|
||||
matchIdxsPool.Put(matchIdxs)
|
||||
}
|
||||
if deduplicatorGlobal != nil {
|
||||
} else if deduplicatorGlobal != nil {
|
||||
deduplicatorGlobal.Push(tssBlock)
|
||||
tssBlock = tssBlock[:0]
|
||||
}
|
||||
@@ -923,8 +922,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
|
||||
tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
|
||||
}
|
||||
matchIdxsPool.Put(matchIdxs)
|
||||
}
|
||||
if rwctx.deduplicator != nil {
|
||||
} else if rwctx.deduplicator != nil {
|
||||
rwctx.deduplicator.Push(tss)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package remotewrite
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
@@ -51,13 +50,12 @@ var (
|
||||
streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
|
||||
"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
|
||||
"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
|
||||
streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
|
||||
streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
|
||||
"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
|
||||
"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
|
||||
"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
|
||||
streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
|
||||
"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
|
||||
"Multiple labels per remoteWrite.url must be delimited by '^^': -remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
|
||||
"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
|
||||
)
|
||||
|
||||
@@ -132,10 +130,11 @@ func initStreamAggrConfigGlobal() {
|
||||
sasGlobal.Store(sas)
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
|
||||
}
|
||||
dedupInterval := streamAggrGlobalDedupInterval.Duration()
|
||||
if dedupInterval > 0 {
|
||||
deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
|
||||
} else {
|
||||
dedupInterval := streamAggrGlobalDedupInterval.Duration()
|
||||
if dedupInterval > 0 {
|
||||
deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrDropInputLabels, "dedup-global")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,15 +152,12 @@ func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
|
||||
rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
|
||||
}
|
||||
dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
|
||||
if dedupInterval > 0 {
|
||||
alias := fmt.Sprintf("dedup-%d", idx+1)
|
||||
var dropLabels []string
|
||||
if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
|
||||
dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
|
||||
} else {
|
||||
dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
|
||||
if dedupInterval > 0 {
|
||||
alias := fmt.Sprintf("dedup-%d", idx+1)
|
||||
rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
|
||||
}
|
||||
rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, dropLabels, alias)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -206,7 +202,6 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
|
||||
DropInputLabels: *streamAggrGlobalDropInputLabels,
|
||||
IgnoreOldSamples: *streamAggrGlobalIgnoreOldSamples,
|
||||
IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
|
||||
KeepInput: *streamAggrGlobalKeepInput,
|
||||
}
|
||||
|
||||
sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
|
||||
@@ -230,16 +225,11 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag
|
||||
if *showRemoteWriteURL {
|
||||
alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
|
||||
}
|
||||
var dropLabels []string
|
||||
if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
|
||||
dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
|
||||
}
|
||||
opts := &streamaggr.Options{
|
||||
DedupInterval: streamAggrDedupInterval.GetOptionalArg(idx),
|
||||
DropInputLabels: dropLabels,
|
||||
DropInputLabels: *streamAggrDropInputLabels,
|
||||
IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(idx),
|
||||
IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
|
||||
KeepInput: streamAggrKeepInput.GetOptionalArg(idx),
|
||||
IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
|
||||
}
|
||||
|
||||
sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
|
||||
|
||||
@@ -41,19 +41,9 @@ Examples:
|
||||
Usage: "disable adding group's Name as label to generated alerts and time series.",
|
||||
Required: false,
|
||||
},
|
||||
&cli.StringSliceFlag{
|
||||
Name: "external.label",
|
||||
Usage: `Optional label in the form 'name=value' to add to all generated recording rules and alerts. Supports an array of values separated by comma or specified via multiple flags.`,
|
||||
Required: false,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "external.url",
|
||||
Usage: `Optional external URL to template in rule's labels or annotations.`,
|
||||
Required: false,
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url")); failed {
|
||||
if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
|
||||
return fmt.Errorf("unittest failed")
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ tests:
|
||||
- expr: suquery_interval_test
|
||||
eval_time: 4m
|
||||
exp_samples:
|
||||
- labels: '{__name__="suquery_interval_test", instance="localhost:9090", job="vmagent2"}'
|
||||
- labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
|
||||
value: 1
|
||||
|
||||
alert_rule_test:
|
||||
@@ -24,16 +24,20 @@ tests:
|
||||
job: vmagent2
|
||||
severity: page
|
||||
instance: localhost:9090
|
||||
datacenter: dc-123
|
||||
exp_annotations:
|
||||
summary: "Instance localhost:9090 down"
|
||||
description: "localhost:9090 of job vmagent2 in cluster has been down for more than 5 minutes."
|
||||
dashboard: "/d/dashboard?orgId=1"
|
||||
description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
|
||||
|
||||
- eval_time: 0
|
||||
alertname: AlwaysFiring
|
||||
exp_alerts:
|
||||
- {}
|
||||
- exp_labels:
|
||||
datacenter: dc-123
|
||||
|
||||
- eval_time: 0
|
||||
alertname: InstanceDown
|
||||
exp_alerts: []
|
||||
|
||||
external_labels:
|
||||
datacenter: dc-123
|
||||
|
||||
@@ -8,8 +8,7 @@ groups:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Instance {{ $labels.instance }} down"
|
||||
description: "{{ $labels.instance }} of job {{ $labels.job }} in cluster {{ $externalLabels.cluster }} has been down for more than 5 minutes."
|
||||
dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
|
||||
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
|
||||
- alert: AlwaysFiring
|
||||
expr: 1
|
||||
- alert: SameAlertNameWithDifferentGroup
|
||||
|
||||
21
app/vmalert-tool/unittest/testdata/test1.yaml
vendored
21
app/vmalert-tool/unittest/testdata/test1.yaml
vendored
@@ -16,8 +16,7 @@ tests:
|
||||
groupname: group1
|
||||
alertname: SameAlertNameWithDifferentGroup
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: prod
|
||||
- {}
|
||||
- eval_time: 1m
|
||||
groupname: group2
|
||||
alertname: SameAlertNameWithDifferentGroup
|
||||
@@ -26,8 +25,7 @@ tests:
|
||||
groupname: group1
|
||||
alertname: SameAlertNameWithDifferentGroup
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: prod
|
||||
- {}
|
||||
- eval_time: 6m
|
||||
groupname: group1
|
||||
alertname: SameAlertNameWithDifferentGroup
|
||||
@@ -63,18 +61,18 @@ tests:
|
||||
eval_time: 4m
|
||||
exp_samples:
|
||||
- value: 4
|
||||
labels: '{__name__="t1", cluster="prod"}'
|
||||
labels: '{__name__="t1", datacenter="dc-123"}'
|
||||
- expr: t2
|
||||
eval_time: 4m
|
||||
exp_samples:
|
||||
- value: 4
|
||||
labels: '{__name__="t2", cluster="prod"}'
|
||||
labels: '{__name__="t2", datacenter="dc-123"}'
|
||||
- expr: t3
|
||||
eval_time: 4m
|
||||
exp_samples:
|
||||
# t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
|
||||
- value: 3
|
||||
labels: '{__name__="t3", cluster="prod"}'
|
||||
labels: '{__name__="t3", datacenter="dc-123"}'
|
||||
|
||||
alert_rule_test:
|
||||
- eval_time: 10m
|
||||
@@ -85,21 +83,22 @@ tests:
|
||||
job: vmagent1
|
||||
severity: page
|
||||
instance: localhost:9090
|
||||
cluster: prod
|
||||
datacenter: dc-123
|
||||
exp_annotations:
|
||||
summary: "Instance localhost:9090 down"
|
||||
description: "localhost:9090 of job vmagent1 in cluster prod has been down for more than 5 minutes."
|
||||
dashboard: "http://grafana:3000/d/dashboard?orgId=1"
|
||||
description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
|
||||
|
||||
- eval_time: 0
|
||||
groupname: group1
|
||||
alertname: AlwaysFiring
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: prod
|
||||
datacenter: dc-123
|
||||
|
||||
- eval_time: 0
|
||||
groupname: alerts
|
||||
alertname: InstanceDown
|
||||
exp_alerts: []
|
||||
|
||||
external_labels:
|
||||
datacenter: dc-123
|
||||
|
||||
11
app/vmalert-tool/unittest/testdata/test2.yaml
vendored
11
app/vmalert-tool/unittest/testdata/test2.yaml
vendored
@@ -13,7 +13,7 @@ tests:
|
||||
- expr: suquery_interval_test
|
||||
eval_time: 4m
|
||||
exp_samples:
|
||||
- labels: '{__name__="suquery_interval_test", cluster="prod", instance="localhost:9090", job="vmagent2"}'
|
||||
- labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
|
||||
value: 1
|
||||
|
||||
alert_rule_test:
|
||||
@@ -25,21 +25,22 @@ tests:
|
||||
job: vmagent2
|
||||
severity: page
|
||||
instance: localhost:9090
|
||||
cluster: prod
|
||||
datacenter: dc-123
|
||||
exp_annotations:
|
||||
summary: "Instance localhost:9090 down"
|
||||
description: "localhost:9090 of job vmagent2 in cluster prod has been down for more than 5 minutes."
|
||||
dashboard: "http://grafana:3000/d/dashboard?orgId=1"
|
||||
description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
|
||||
|
||||
- eval_time: 0
|
||||
groupname: group1
|
||||
alertname: AlwaysFiring
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: prod
|
||||
datacenter: dc-123
|
||||
|
||||
- eval_time: 0
|
||||
groupname: group1
|
||||
alertname: InstanceDown
|
||||
exp_alerts: []
|
||||
|
||||
external_labels:
|
||||
datacenter: dc-123
|
||||
|
||||
@@ -55,7 +55,7 @@ const (
|
||||
)
|
||||
|
||||
// UnitTest runs unittest for files
|
||||
func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL string) bool {
|
||||
func UnitTest(files []string, disableGroupLabel bool) bool {
|
||||
if err := templates.Load([]string{}, true); err != nil {
|
||||
logger.Fatalf("failed to load template: %v", err)
|
||||
}
|
||||
@@ -71,34 +71,14 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
|
||||
|
||||
testfiles, err := config.ReadFromFS(files)
|
||||
if err != nil {
|
||||
logger.Fatalf("failed to load test files %q: %v", files, err)
|
||||
fmt.Println(" FAILED")
|
||||
fmt.Printf("\nfailed to read test files: \n%v", err)
|
||||
}
|
||||
if len(testfiles) == 0 {
|
||||
fmt.Println("no test file found")
|
||||
return false
|
||||
}
|
||||
|
||||
labels := make(map[string]string)
|
||||
for _, s := range externalLabels {
|
||||
if len(s) == 0 {
|
||||
continue
|
||||
}
|
||||
n := strings.IndexByte(s, '=')
|
||||
if n < 0 {
|
||||
logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
|
||||
}
|
||||
labels[s[:n]] = s[n+1:]
|
||||
}
|
||||
_, err = notifier.Init(nil, labels, externalURL)
|
||||
if err != nil {
|
||||
logger.Fatalf("failed to init notifier: %v", err)
|
||||
}
|
||||
|
||||
var failed bool
|
||||
for fileName, file := range testfiles {
|
||||
if err := ruleUnitTest(fileName, file, labels); err != nil {
|
||||
if err := ruleUnitTest(fileName, file); err != nil {
|
||||
fmt.Println(" FAILED")
|
||||
fmt.Printf("\nfailed to run unit test for file %q: \n%v", fileName, err)
|
||||
fmt.Printf("\nfailed to run unit test for file %q: \n%v", file, err)
|
||||
failed = true
|
||||
} else {
|
||||
fmt.Println(" SUCCESS")
|
||||
@@ -108,7 +88,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
|
||||
return failed
|
||||
}
|
||||
|
||||
func ruleUnitTest(filename string, content []byte, externalLabels map[string]string) []error {
|
||||
func ruleUnitTest(filename string, content []byte) []error {
|
||||
fmt.Println("\nUnit Testing: ", filename)
|
||||
var unitTestInp unitTestFile
|
||||
if err := yaml.UnmarshalStrict(content, &unitTestInp); err != nil {
|
||||
@@ -146,7 +126,7 @@ func ruleUnitTest(filename string, content []byte, externalLabels map[string]str
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups, externalLabels)
|
||||
testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups)
|
||||
errs = append(errs, testErrs...)
|
||||
}
|
||||
|
||||
@@ -183,10 +163,6 @@ func verifyTestGroup(group testGroup) error {
|
||||
return fmt.Errorf("\n%s missing required field \"eval_time\"", testGroupName)
|
||||
}
|
||||
}
|
||||
if group.ExternalLabels != nil {
|
||||
fmt.Printf("\n%s warning: filed `external_labels` will be deprecated soon, please use `-external.label` cmd-line flag instead. "+
|
||||
"Check https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6735 for details.\n", testGroupName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -203,7 +179,6 @@ func processFlags() {
|
||||
{flag: "retentionPeriod", value: "100y"},
|
||||
{flag: "datasource.url", value: testDataSourcePath},
|
||||
{flag: "remoteWrite.url", value: testRemoteWritePath},
|
||||
{flag: "notifier.blackhole", value: "true"},
|
||||
} {
|
||||
// panics if flag doesn't exist
|
||||
if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
|
||||
@@ -264,7 +239,7 @@ func tearDown() {
|
||||
fs.MustRemoveAll(storagePath)
|
||||
}
|
||||
|
||||
func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group, externalLabels map[string]string) (checkErrs []error) {
|
||||
func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
|
||||
// set up vmstorage and http server for ingest and read queries
|
||||
setUp()
|
||||
// tear down vmstorage and clean the data dir
|
||||
@@ -313,14 +288,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
|
||||
// create groups with given rule
|
||||
var groups []*rule.Group
|
||||
for _, group := range testGroups {
|
||||
mergedExternalLabels := make(map[string]string)
|
||||
for k, v := range tg.ExternalLabels {
|
||||
mergedExternalLabels[k] = v
|
||||
}
|
||||
for k, v := range externalLabels {
|
||||
mergedExternalLabels[k] = v
|
||||
}
|
||||
ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
|
||||
ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels)
|
||||
groups = append(groups, ng)
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestUnitTest_Failure(t *testing.T) {
|
||||
f := func(files []string) {
|
||||
t.Helper()
|
||||
|
||||
failed := UnitTest(files, false, nil, "")
|
||||
failed := UnitTest(files, false)
|
||||
if !failed {
|
||||
t.Fatalf("expecting failed test")
|
||||
}
|
||||
@@ -29,19 +29,18 @@ func TestUnitTest_Failure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestUnitTest_Success(t *testing.T) {
|
||||
f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL string) {
|
||||
f := func(disableGroupLabel bool, files []string) {
|
||||
t.Helper()
|
||||
|
||||
failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL)
|
||||
failed := UnitTest(files, disableGroupLabel)
|
||||
if failed {
|
||||
t.Fatalf("unexpected failed test")
|
||||
}
|
||||
}
|
||||
|
||||
// run multi files
|
||||
f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000")
|
||||
f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"})
|
||||
|
||||
// disable group label
|
||||
// template with null external values
|
||||
f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "")
|
||||
f(true, []string{"./testdata/disable-group-label.yaml"})
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
|
||||
var (
|
||||
addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
|
||||
"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
|
||||
"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
|
||||
"See also -remoteRead.disablePathAppend and -datasource.showURL")
|
||||
appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
|
||||
showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
|
||||
@@ -99,7 +99,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
|
||||
|
||||
tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
|
||||
return nil, fmt.Errorf("failed to create transport: %w", err)
|
||||
}
|
||||
tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
|
||||
tr.DisableKeepAlives = *disableKeepAlive
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -57,23 +57,20 @@ var jsonParserPool fastjson.ParserPool
|
||||
// [{"metric":{"__name__":"up","job":"prometheus"},value": [ 1435781451.781,"1"]},
|
||||
// {"metric":{"__name__":"up","job":"node"},value": [ 1435781451.781,"0"]}]
|
||||
func (pi *promInstant) Unmarshal(b []byte) error {
|
||||
var metrics []json.RawMessage
|
||||
// metrics slice could be large, so parsing it with fastjson could consume a lot of memory.
|
||||
// We parse the slice with standard lib to keep mem usage low.
|
||||
// And each metric object will be parsed with fastjson to reduce allocations.
|
||||
if err := json.Unmarshal(b, &metrics); err != nil {
|
||||
return fmt.Errorf("cannot unmarshal metrics: %w", err)
|
||||
}
|
||||
|
||||
p := jsonParserPool.Get()
|
||||
defer jsonParserPool.Put(p)
|
||||
|
||||
pi.ms = make([]Metric, len(metrics))
|
||||
for i, data := range metrics {
|
||||
row, err := p.ParseBytes(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse metric object: %w", err)
|
||||
}
|
||||
v, err := p.ParseBytes(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := v.Array()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot find the top-level array of result objects: %w", err)
|
||||
}
|
||||
pi.ms = make([]Metric, len(rows))
|
||||
for i, row := range rows {
|
||||
metric := row.Get("metric")
|
||||
if metric == nil {
|
||||
return fmt.Errorf("can't find `metric` object in %q", row)
|
||||
|
||||
@@ -1,24 +1,43 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkPromInstantUnmarshal(b *testing.B) {
|
||||
func BenchmarkMetrics(b *testing.B) {
|
||||
payload := []byte(`[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests", "foo":"bar", "baz": "qux"},"value":[1583786140,"2000"]}]`)
|
||||
|
||||
var pi promInstant
|
||||
if err := pi.Unmarshal(payload); err != nil {
|
||||
b.Fatalf(err.Error())
|
||||
}
|
||||
b.Run("Instant", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = pi.metrics()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkParsePrometheusResponse(b *testing.B) {
|
||||
req, _ := http.NewRequest("GET", "", nil)
|
||||
resp := &http.Response{StatusCode: http.StatusOK}
|
||||
data, err := os.ReadFile("testdata/instant_response.json")
|
||||
if err != nil {
|
||||
b.Fatalf("error while reading file: %s", err)
|
||||
}
|
||||
resp.Body = io.NopCloser(bytes.NewReader(data))
|
||||
|
||||
// BenchmarkParsePrometheusResponse/Instant_std+fastjson-10 1760 668959 ns/op 280147 B/op 5781 allocs/op
|
||||
b.Run("Instant std+fastjson", func(b *testing.B) {
|
||||
b.Run("Instant", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
var pi promInstant
|
||||
err = pi.Unmarshal(data)
|
||||
_, err := parsePrometheusResponse(req, resp)
|
||||
if err != nil {
|
||||
b.Fatalf("unexpected parse err: %s", err)
|
||||
}
|
||||
resp.Body = io.NopCloser(bytes.NewReader(data))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 8880
|
||||
|
||||
ENTRYPOINT ["/vmalert-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./vmalert-prod
|
||||
|
||||
@@ -76,7 +76,7 @@ absolute path to all .tpl files in root.
|
||||
`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
|
||||
`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
|
||||
externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
|
||||
"In case of conflicts, original labels are kept with prefix `exported_`.")
|
||||
"Pass multiple -label flags in order to add multiple label sets.")
|
||||
|
||||
remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup. DEPRECATED - this flag has no effect and will be removed in the next releases.")
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -187,7 +187,7 @@ func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
|
||||
func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
|
||||
var labels []prompbmarshal.Label
|
||||
for k, v := range a.Labels {
|
||||
labels = append(labels, prompbmarshal.Label{
|
||||
|
||||
@@ -187,7 +187,7 @@ func TestAlert_toPromLabels(t *testing.T) {
|
||||
fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
|
||||
t.Helper()
|
||||
a := Alert{Labels: labels}
|
||||
got := a.applyRelabelingIfNeeded(relabel)
|
||||
got := a.toPromLabels(relabel)
|
||||
if !reflect.DeepEqual(got, exp) {
|
||||
t.Fatalf("expected to have: \n%v;\ngot:\n%v",
|
||||
exp, got)
|
||||
|
||||
@@ -76,6 +76,9 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for key, value := range headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
if am.timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
@@ -91,11 +94,6 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
|
||||
return err
|
||||
}
|
||||
}
|
||||
// external headers have higher priority
|
||||
for key, value := range headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
resp, err := am.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -132,8 +130,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
|
||||
}
|
||||
tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create transport for alertmanager URL=%q: %w", alertManagerURL, err)
|
||||
|
||||
return nil, fmt.Errorf("failed to create transport: %w", err)
|
||||
}
|
||||
|
||||
ba := new(promauth.BasicAuthConfig)
|
||||
@@ -148,9 +145,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
|
||||
aCfg, err := utils.AuthConfig(
|
||||
utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
|
||||
utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
|
||||
utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
|
||||
utils.WithHeaders(strings.Join(authCfg.Headers, "^^")),
|
||||
)
|
||||
utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to configure auth: %w", err)
|
||||
}
|
||||
|
||||
@@ -8,8 +8,6 @@
|
||||
{% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %}
|
||||
[
|
||||
{% for i, alert := range alerts %}
|
||||
{% code lbls := alert.applyRelabelingIfNeeded(relabelCfg) %}
|
||||
{% if len(lbls) == 0 %} {% continue %} {% endif %}
|
||||
{
|
||||
"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
|
||||
"generatorURL": {%q= generatorURL(alert) %},
|
||||
@@ -17,6 +15,7 @@
|
||||
"endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
|
||||
{% endif %}
|
||||
"labels": {
|
||||
{% code lbls := alert.toPromLabels(relabelCfg) %}
|
||||
{% code ll := len(lbls) %}
|
||||
{% for idx, l := range lbls %}
|
||||
{%q= l.Name %}:{%q= l.Value %}{% if idx != ll-1 %}, {% endif %}
|
||||
|
||||
@@ -30,117 +30,111 @@ func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL fun
|
||||
qw422016.N().S(`[`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:10
|
||||
for i, alert := range alerts {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:11
|
||||
lbls := alert.applyRelabelingIfNeeded(relabelCfg)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
if len(lbls) == 0 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
continue
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:10
|
||||
qw422016.N().S(`{"startsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
qw422016.N().S(`,"generatorURL":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:13
|
||||
qw422016.N().Q(generatorURL(alert))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:13
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
if !alert.End.IsZero() {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
qw422016.N().S(`"endsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:17
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:17
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:18
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:18
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
qw422016.N().S(`"labels": {`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:20
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:18
|
||||
lbls := alert.toPromLabels(relabelCfg)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:19
|
||||
ll := len(lbls)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:20
|
||||
for idx, l := range lbls {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
qw422016.N().Q(l.Name)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
qw422016.N().Q(l.Value)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
if idx != ll-1 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:23
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:23
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
qw422016.N().S(`},"annotations": {`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:26
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:25
|
||||
c := len(alert.Annotations)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:27
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:26
|
||||
for k, v := range alert.Annotations {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:27
|
||||
c = c - 1
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
qw422016.N().Q(k)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
qw422016.N().Q(v)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
if c > 0 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:30
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:30
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
qw422016.N().S(`}}`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:32
|
||||
if i != len(alerts)-1 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:32
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:32
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:34
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:34
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
qw422016.N().S(`]`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
}
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
streamamRequest(qw422016, alerts, generatorURL, relabelCfg)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
}
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
writeamRequest(qb422016, alerts, generatorURL, relabelCfg)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
return qs422016
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package notifier
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
@@ -11,7 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
)
|
||||
|
||||
func TestAlertManager_Addr(t *testing.T) {
|
||||
@@ -50,9 +48,6 @@ func TestAlertManager_Send(t *testing.T) {
|
||||
conn, _, _ := w.(http.Hijacker).Hijack()
|
||||
_ = conn.Close()
|
||||
case 1:
|
||||
if r.Header.Get(headerKey) != headerValue {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
|
||||
}
|
||||
w.WriteHeader(500)
|
||||
case 2:
|
||||
var a []struct {
|
||||
@@ -77,33 +72,9 @@ func TestAlertManager_Send(t *testing.T) {
|
||||
if a[0].EndAt.IsZero() {
|
||||
t.Fatalf("expected non-zero end time")
|
||||
}
|
||||
if len(a[0].Labels) != 1 {
|
||||
t.Fatalf("expected 1 labels got %d", len(a[0].Labels))
|
||||
}
|
||||
if len(a[0].Annotations) != 2 {
|
||||
t.Fatalf("expected 2 annotations got %d", len(a[0].Annotations))
|
||||
}
|
||||
if r.Header.Get(headerKey) != "bar" {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
|
||||
}
|
||||
case 3:
|
||||
var a []struct {
|
||||
Labels map[string]string `json:"labels"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
|
||||
t.Fatalf("can not unmarshal data into alert %s", err)
|
||||
}
|
||||
if len(a) != 1 {
|
||||
t.Fatalf("expected 1 alert in array got %d", len(a))
|
||||
}
|
||||
if len(a[0].Labels) != 3 {
|
||||
t.Fatalf("expected 3 labels got %d", len(a[0].Labels))
|
||||
}
|
||||
if a[0].Labels["env"] != "prod" {
|
||||
t.Fatalf("expected env label to be prod during relabeling, got %s", a[0].Labels["env"])
|
||||
}
|
||||
if r.Header.Get(headerKey) != "bar" {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, "bar", r.Header.Get(headerKey))
|
||||
if r.Header.Get(headerKey) != headerValue {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -115,60 +86,32 @@ func TestAlertManager_Send(t *testing.T) {
|
||||
Username: baUser,
|
||||
Password: promauth.NewSecret(baPass),
|
||||
},
|
||||
Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
|
||||
}
|
||||
parsedConfigs, err := promrelabel.ParseRelabelConfigsData([]byte(`
|
||||
- action: drop
|
||||
if: '{tenant="0"}'
|
||||
regex: ".*"
|
||||
- target_label: "env"
|
||||
replacement: "prod"
|
||||
if: '{tenant="1"}'
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error when parse relabeling config: %s", err)
|
||||
}
|
||||
am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
|
||||
return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
|
||||
}, aCfg, parsedConfigs, 0)
|
||||
}, aCfg, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
|
||||
if err := am.Send(context.Background(), []Alert{{}, {}}, nil); err == nil {
|
||||
t.Fatalf("expected connection error got nil")
|
||||
}
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
|
||||
if err := am.Send(context.Background(), []Alert{}, nil); err == nil {
|
||||
t.Fatalf("expected wrong http code error got nil")
|
||||
}
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{{
|
||||
GroupID: 0,
|
||||
Name: "alert0",
|
||||
Start: time.Now().UTC(),
|
||||
End: time.Now().UTC(),
|
||||
Labels: map[string]string{"alertname": "alert0"},
|
||||
Annotations: map[string]string{"a": "b", "c": "d"},
|
||||
}}, map[string]string{headerKey: "bar"}); err != nil {
|
||||
Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
|
||||
}}, nil); err != nil {
|
||||
t.Fatalf("unexpected error %s", err)
|
||||
}
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{
|
||||
// drop tenant0 alert message during relabeling
|
||||
{
|
||||
Name: "alert1",
|
||||
Labels: map[string]string{"rule": "test", "tenant": "0"},
|
||||
},
|
||||
{
|
||||
Name: "alert2",
|
||||
Labels: map[string]string{"rule": "test", "tenant": "1"},
|
||||
},
|
||||
}, map[string]string{headerKey: "bar"}); err != nil {
|
||||
t.Fatalf("unexpected error %s", err)
|
||||
if c != 2 {
|
||||
t.Fatalf("expected 2 calls(count from zero) to server got %d", c)
|
||||
}
|
||||
|
||||
if c != 3 {
|
||||
t.Fatalf("expected 3 calls(count from zero) to server got %d", c)
|
||||
if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
|
||||
t.Fatalf("unexpected error %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,9 +25,6 @@ var (
|
||||
"Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). "+
|
||||
"-notifier.url, -notifier.config and -notifier.blackhole are mutually exclusive.")
|
||||
|
||||
headers = flagutil.NewArrayString("notifier.headers", "Optional HTTP headers to send with each request to the corresponding -notifier.url. "+
|
||||
"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
|
||||
"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
|
||||
basicAuthUsername = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
|
||||
basicAuthPassword = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
|
||||
basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
|
||||
@@ -174,7 +171,6 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
|
||||
Scopes: strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
|
||||
TokenURL: oauth2TokenURL.GetOptionalArg(i),
|
||||
},
|
||||
Headers: []string{headers.GetOptionalArg(i)},
|
||||
}
|
||||
|
||||
addr = strings.TrimSuffix(addr, "/")
|
||||
|
||||
@@ -17,7 +17,7 @@ var (
|
||||
addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
|
||||
"Remote read is used to restore alerts state."+
|
||||
"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
|
||||
"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
|
||||
"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
|
||||
"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
|
||||
|
||||
showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
|
||||
@@ -68,7 +68,7 @@ func Init() (datasource.QuerierBuilder, error) {
|
||||
}
|
||||
tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
|
||||
return nil, fmt.Errorf("failed to create transport: %w", err)
|
||||
}
|
||||
tr.IdleConnTimeout = *idleConnectionTimeout
|
||||
tr.DialContext = netutil.NewStatDialFunc("vmalert_remoteread")
|
||||
|
||||
@@ -32,7 +32,7 @@ func NewDebugClient() (*DebugClient, error) {
|
||||
|
||||
t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
|
||||
return nil, fmt.Errorf("failed to create transport: %w", err)
|
||||
}
|
||||
c := &DebugClient{
|
||||
c: &http.Client{
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
var (
|
||||
addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
|
||||
"and recording rules results in form of timeseries. "+
|
||||
"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
|
||||
"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
|
||||
"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
|
||||
"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
|
||||
showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
|
||||
@@ -34,9 +34,9 @@ var (
|
||||
|
||||
idleConnectionTimeout = flag.Duration("remoteWrite.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider settings this value less to the value of "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
|
||||
|
||||
maxQueueSize = flag.Int("remoteWrite.maxQueueSize", 1e6, "Defines the max number of pending datapoints to remote write endpoint")
|
||||
maxBatchSize = flag.Int("remoteWrite.maxBatchSize", 1e4, "Defines max number of timeseries to be flushed at once")
|
||||
concurrency = flag.Int("remoteWrite.concurrency", 4, "Defines number of writers for concurrent writing into remote write endpoint")
|
||||
maxQueueSize = flag.Int("remoteWrite.maxQueueSize", 1e5, "Defines the max number of pending datapoints to remote write endpoint")
|
||||
maxBatchSize = flag.Int("remoteWrite.maxBatchSize", 1e3, "Defines max number of timeseries to be flushed at once")
|
||||
concurrency = flag.Int("remoteWrite.concurrency", 1, "Defines number of writers for concurrent writing into remote write endpoint")
|
||||
flushInterval = flag.Duration("remoteWrite.flushInterval", 5*time.Second, "Defines interval of flushes to remote write endpoint")
|
||||
|
||||
tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
|
||||
@@ -72,7 +72,7 @@ func Init(ctx context.Context) (*Client, error) {
|
||||
|
||||
t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
|
||||
return nil, fmt.Errorf("failed to create transport: %w", err)
|
||||
}
|
||||
t.IdleConnTimeout = *idleConnectionTimeout
|
||||
t.DialContext = netutil.NewStatDialFunc("vmalert_remotewrite")
|
||||
|
||||
@@ -335,8 +335,6 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
|
||||
// reset to Pending if there are gaps > EvalInterval between DPs
|
||||
a.State = notifier.StatePending
|
||||
a.ActiveAt = at
|
||||
// re-template the annotations as active timestamp is changed
|
||||
_, a.Annotations, _ = ar.expandTemplates(s, qFn, at)
|
||||
a.Start = time.Time{}
|
||||
} else if at.Sub(a.ActiveAt) >= ar.For && a.State != notifier.StateFiring {
|
||||
a.State = notifier.StateFiring
|
||||
@@ -443,6 +441,9 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
|
||||
}
|
||||
a.Value = m.Values[0]
|
||||
a.Annotations = annotations
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.KeepFiringSince = time.Time{}
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -289,7 +289,7 @@ func TestAlertingRule_Exec(t *testing.T) {
|
||||
4: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}},
|
||||
})
|
||||
|
||||
f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>firing", defaultStep, 0, defaultStep, nil), [][]datasource.Metric{
|
||||
f(newTestAlertingRuleWithKeepFiring("for-pending=>firing=>keepfiring=>firing", defaultStep, defaultStep), [][]datasource.Metric{
|
||||
{metricWithLabels(t, "name", "foo")},
|
||||
{metricWithLabels(t, "name", "foo")},
|
||||
// empty step to keep firing
|
||||
@@ -302,7 +302,7 @@ func TestAlertingRule_Exec(t *testing.T) {
|
||||
3: {{labels: []string{"name", "foo"}, alert: ¬ifier.Alert{State: notifier.StateFiring}}},
|
||||
})
|
||||
|
||||
f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>keepfiring=>inactive=>pending=>firing", defaultStep, 0, 2*defaultStep, nil), [][]datasource.Metric{
|
||||
f(newTestAlertingRuleWithKeepFiring("for-pending=>firing=>keepfiring=>keepfiring=>inactive=>pending=>firing", defaultStep, 2*defaultStep), [][]datasource.Metric{
|
||||
{metricWithLabels(t, "name", "foo")},
|
||||
{metricWithLabels(t, "name", "foo")},
|
||||
// empty step to keep firing
|
||||
@@ -395,7 +395,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
{State: notifier.StateFiring, ActiveAt: time.Unix(3e3, 0)},
|
||||
}, nil)
|
||||
|
||||
f(newTestAlertingRuleWithCustomFields("for-pending", time.Second, 0, 0, map[string]string{"activeAt": "{{ $activeAt.UnixMilli }}"}), []datasource.Metric{
|
||||
f(newTestAlertingRule("for-pending", time.Second), []datasource.Metric{
|
||||
{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
|
||||
}, []*notifier.Alert{
|
||||
{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
|
||||
@@ -406,7 +406,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
GroupID: fakeGroup.ID(),
|
||||
Name: "for-pending",
|
||||
Labels: map[string]string{"alertname": "for-pending"},
|
||||
Annotations: map[string]string{"activeAt": "5000"},
|
||||
Annotations: map[string]string{},
|
||||
State: notifier.StatePending,
|
||||
ActiveAt: time.Unix(5, 0),
|
||||
Value: 1,
|
||||
@@ -414,7 +414,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
f(newTestAlertingRuleWithCustomFields("for-firing", 3*time.Second, 0, 0, map[string]string{"activeAt": "{{ $activeAt.UnixMilli }}"}), []datasource.Metric{
|
||||
f(newTestAlertingRule("for-firing", 3*time.Second), []datasource.Metric{
|
||||
{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
|
||||
}, []*notifier.Alert{
|
||||
{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
|
||||
@@ -425,7 +425,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
GroupID: fakeGroup.ID(),
|
||||
Name: "for-firing",
|
||||
Labels: map[string]string{"alertname": "for-firing"},
|
||||
Annotations: map[string]string{"activeAt": "1000"},
|
||||
Annotations: map[string]string{},
|
||||
State: notifier.StateFiring,
|
||||
ActiveAt: time.Unix(1, 0),
|
||||
Start: time.Unix(5, 0),
|
||||
@@ -434,7 +434,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
f(newTestAlertingRuleWithCustomFields("for-hold-pending", time.Second, 0, 0, map[string]string{"activeAt": "{{ $activeAt.UnixMilli }}"}), []datasource.Metric{
|
||||
f(newTestAlertingRule("for-hold-pending", time.Second), []datasource.Metric{
|
||||
{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 2, 5}},
|
||||
}, []*notifier.Alert{
|
||||
{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
|
||||
@@ -445,7 +445,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
GroupID: fakeGroup.ID(),
|
||||
Name: "for-hold-pending",
|
||||
Labels: map[string]string{"alertname": "for-hold-pending"},
|
||||
Annotations: map[string]string{"activeAt": "5000"},
|
||||
Annotations: map[string]string{},
|
||||
State: notifier.StatePending,
|
||||
ActiveAt: time.Unix(5, 0),
|
||||
Value: 1,
|
||||
@@ -453,7 +453,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
f(newTestAlertingRuleWithCustomFields("firing=>inactive=>inactive=>firing=>firing", 0, time.Second, 0, nil), []datasource.Metric{
|
||||
f(newTestAlertingRuleWithEvalInterval("firing=>inactive=>inactive=>firing=>firing", 0, time.Second), []datasource.Metric{
|
||||
{Values: []float64{1, 1, 1, 1}, Timestamps: []int64{1, 4, 5, 6}},
|
||||
}, []*notifier.Alert{
|
||||
{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
|
||||
@@ -538,6 +538,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
"source": "vm",
|
||||
},
|
||||
},
|
||||
//
|
||||
{
|
||||
State: notifier.StateFiring, ActiveAt: time.Unix(1, 0),
|
||||
Labels: map[string]string{
|
||||
@@ -1035,13 +1036,15 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
|
||||
return &rule
|
||||
}
|
||||
|
||||
func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, keepFiringFor time.Duration, annotation map[string]string) *AlertingRule {
|
||||
func newTestAlertingRuleWithEvalInterval(name string, waitFor, evalInterval time.Duration) *AlertingRule {
|
||||
rule := newTestAlertingRule(name, waitFor)
|
||||
rule.EvalInterval = evalInterval
|
||||
return rule
|
||||
}
|
||||
|
||||
func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.Duration) *AlertingRule {
|
||||
rule := newTestAlertingRule(name, waitFor)
|
||||
if evalInterval != 0 {
|
||||
rule.EvalInterval = evalInterval
|
||||
}
|
||||
rule.KeepFiringFor = keepFiringFor
|
||||
rule.Annotations = annotation
|
||||
return rule
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
func BenchmarkGetStaleSeries(b *testing.B) {
|
||||
ts := time.Now()
|
||||
n := 100
|
||||
payload := make([]prompbmarshal.TimeSeries, 0, n)
|
||||
payload := make([]prompbmarshal.TimeSeries, n)
|
||||
for i := 0; i < n; i++ {
|
||||
s := fmt.Sprintf("%d", i)
|
||||
labels := toPromLabels(b,
|
||||
|
||||
@@ -50,12 +50,12 @@ func TestRequestToCurl(t *testing.T) {
|
||||
f(req, "curl -k -X POST 'https://user:xxxxx@foo.com?query=up&step=10'")
|
||||
|
||||
req = newReq("https://user:pass@foo.com")
|
||||
req.Header.Set("Authorization", "Bearer 123456")
|
||||
f(req, "curl -k -X POST -H 'Authorization: <secret>' 'https://user:xxxxx@foo.com'")
|
||||
req.Header.Set("Authorisation", "Bearer 123456")
|
||||
f(req, "curl -k -X POST -H 'Authorisation: <secret>' 'https://user:xxxxx@foo.com'")
|
||||
|
||||
req = newReq("https://user:pass@foo.com")
|
||||
req.Header.Set("Authorization", "Basic 123456")
|
||||
f(req, "curl -k -X POST -H 'Authorization: <secret>' 'https://user:xxxxx@foo.com'")
|
||||
req.Header.Set("Authorisation", "Basic 123456")
|
||||
f(req, "curl -k -X POST -H 'Authorisation: <secret>' 'https://user:xxxxx@foo.com'")
|
||||
|
||||
req = newReq("https://foo.com")
|
||||
req.Header.Set("My-Password", "mypassword")
|
||||
@@ -66,7 +66,7 @@ func TestRequestToCurl(t *testing.T) {
|
||||
f(req, "curl -k -X POST -H 'Key-For: <secret>' 'https://foo.com'")
|
||||
|
||||
req = newReq("https://foo.com")
|
||||
req.Header.Set("My-Secret-Org", "secret-organization")
|
||||
req.Header.Set("My-Secret-Org", "secret-organisation")
|
||||
f(req, "curl -k -X POST -H 'My-Secret-Org: <secret>' 'https://foo.com'")
|
||||
|
||||
req = newReq("https://foo.com")
|
||||
|
||||
@@ -83,6 +83,7 @@ type UserInfo struct {
|
||||
|
||||
concurrencyLimitCh chan struct{}
|
||||
concurrencyLimitReached *metrics.Counter
|
||||
overrideHostHeader bool
|
||||
|
||||
rt http.RoundTripper
|
||||
|
||||
@@ -93,9 +94,8 @@ type UserInfo struct {
|
||||
|
||||
// HeadersConf represents config for request and response headers.
|
||||
type HeadersConf struct {
|
||||
RequestHeaders []*Header `yaml:"headers,omitempty"`
|
||||
ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
|
||||
KeepOriginalHost *bool `yaml:"keep_original_host,omitempty"`
|
||||
RequestHeaders []*Header `yaml:"headers,omitempty"`
|
||||
ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
|
||||
}
|
||||
|
||||
func (ui *UserInfo) beginConcurrencyLimit() error {
|
||||
@@ -150,6 +150,15 @@ func (h *Header) MarshalYAML() (any, error) {
|
||||
return h.sOriginal, nil
|
||||
}
|
||||
|
||||
func overrideHostHeader(headers []*Header) bool {
|
||||
for _, h := range headers {
|
||||
if h.Name == "Host" && h.Value == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// URLMap is a mapping from source paths to target urls.
|
||||
type URLMap struct {
|
||||
// SrcPaths is an optional list of regular expressions, which must match the request path.
|
||||
@@ -301,18 +310,12 @@ func (up *URLPrefix) getBackendsCount() int {
|
||||
|
||||
// getBackendURL returns the backendURL depending on the load balance policy.
|
||||
//
|
||||
// It can return nil if there are no backend urls available at the moment.
|
||||
//
|
||||
// backendURL.put() must be called on the returned backendURL after the request is complete.
|
||||
func (up *URLPrefix) getBackendURL() *backendURL {
|
||||
up.discoverBackendAddrsIfNeeded()
|
||||
|
||||
pbus := up.bus.Load()
|
||||
bus := *pbus
|
||||
if len(bus) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if up.loadBalancingPolicy == "first_available" {
|
||||
return getFirstAvailableBackendURL(bus)
|
||||
}
|
||||
@@ -394,7 +397,7 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
|
||||
buCopy.Host = addr
|
||||
if port != "" {
|
||||
if n := strings.IndexByte(buCopy.Host, ':'); n >= 0 {
|
||||
// Drop the discovered port and substitute it the port specified in bu.
|
||||
// Drop the discovered port and substitute it the the port specified in bu.
|
||||
buCopy.Host = buCopy.Host[:n]
|
||||
}
|
||||
buCopy.Host += ":" + port
|
||||
@@ -461,7 +464,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
|
||||
}
|
||||
|
||||
// Slow path - select other backend urls.
|
||||
n := atomicCounter.Add(1) - 1
|
||||
n := atomicCounter.Add(1)
|
||||
|
||||
for i := uint32(0); i < uint32(len(bus)); i++ {
|
||||
idx := (n + i) % uint32(len(bus))
|
||||
@@ -597,7 +600,7 @@ func initAuthConfig() {
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
|
||||
sighupCh := procutil.NewSighupChan()
|
||||
|
||||
_, err := reloadAuthConfig()
|
||||
_, err := loadAuthConfig()
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot load auth config: %s", err)
|
||||
}
|
||||
@@ -629,7 +632,7 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
|
||||
|
||||
updateFn := func() {
|
||||
configReloads.Inc()
|
||||
updated, err := reloadAuthConfig()
|
||||
updated, err := loadAuthConfig()
|
||||
if err != nil {
|
||||
logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
|
||||
configSuccess.Set(0)
|
||||
@@ -655,45 +658,27 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
|
||||
}
|
||||
}
|
||||
|
||||
// authConfigData stores the yaml definition for this config.
|
||||
// authConfigData needs to be updated each time authConfig is updated.
|
||||
var authConfigData atomic.Pointer[[]byte]
|
||||
|
||||
var (
|
||||
// authConfigData stores the yaml definition for this config.
|
||||
// authConfigData needs to be updated each time authConfig is updated.
|
||||
authConfigData atomic.Pointer[[]byte]
|
||||
|
||||
// authConfig contains the currently loaded auth config
|
||||
authConfig atomic.Pointer[AuthConfig]
|
||||
|
||||
// authUsers contains the currently loaded auth users
|
||||
authUsers atomic.Pointer[map[string]*UserInfo]
|
||||
|
||||
authConfig atomic.Pointer[AuthConfig]
|
||||
authUsers atomic.Pointer[map[string]*UserInfo]
|
||||
authConfigWG sync.WaitGroup
|
||||
stopCh chan struct{}
|
||||
)
|
||||
|
||||
// reloadAuthConfig loads and applies the config from *authConfigPath.
|
||||
// loadAuthConfig loads and applies the config from *authConfigPath.
|
||||
// It returns bool value to identify if new config was applied.
|
||||
// The config can be not applied if there is a parsing error
|
||||
// or if there are no changes to the current authConfig.
|
||||
func reloadAuthConfig() (bool, error) {
|
||||
func loadAuthConfig() (bool, error) {
|
||||
data, err := fscore.ReadFileOrHTTP(*authConfigPath)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
|
||||
}
|
||||
|
||||
ok, err := reloadAuthConfigData(data)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
|
||||
}
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
mp := authUsers.Load()
|
||||
logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func reloadAuthConfigData(data []byte) (bool, error) {
|
||||
oldData := authConfigData.Load()
|
||||
if oldData != nil && bytes.Equal(data, *oldData) {
|
||||
// there are no updates in the config - skip reloading.
|
||||
@@ -702,13 +687,14 @@ func reloadAuthConfigData(data []byte) (bool, error) {
|
||||
|
||||
ac, err := parseAuthConfig(data)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse auth config: %w", err)
|
||||
return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
|
||||
}
|
||||
|
||||
m, err := parseAuthConfigUsers(ac)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse users from auth config: %w", err)
|
||||
return false, fmt.Errorf("failed to parse users from -auth.config=%q: %w", *authConfigPath, err)
|
||||
}
|
||||
logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
|
||||
|
||||
acPrev := authConfig.Load()
|
||||
if acPrev != nil {
|
||||
@@ -755,6 +741,7 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
|
||||
if err := ui.initURLs(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ui.overrideHostHeader = overrideHostHeader(ui.HeadersConf.RequestHeaders)
|
||||
|
||||
metricLabels, err := ui.getMetricLabels()
|
||||
if err != nil {
|
||||
@@ -819,6 +806,7 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
|
||||
_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
|
||||
return float64(len(ui.concurrencyLimitCh))
|
||||
})
|
||||
ui.overrideHostHeader = overrideHostHeader(ui.HeadersConf.RequestHeaders)
|
||||
|
||||
rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
|
||||
if err != nil {
|
||||
@@ -1009,14 +997,6 @@ func getAuthTokensFromRequest(r *http.Request) []string {
|
||||
}
|
||||
}
|
||||
|
||||
// Authorization via http://user:pass@hosname/path
|
||||
if u := r.URL.User; u != nil && u.Username() != "" {
|
||||
username := u.Username()
|
||||
password, _ := u.Password()
|
||||
at := getHTTPAuthBasicToken(username, password)
|
||||
ats = append(ats, at)
|
||||
}
|
||||
|
||||
return ats
|
||||
}
|
||||
|
||||
@@ -1042,6 +1022,10 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
|
||||
}
|
||||
|
||||
func sanitizeURLPrefix(urlPrefix *url.URL) (*url.URL, error) {
|
||||
// Remove trailing '/' from urlPrefix
|
||||
for strings.HasSuffix(urlPrefix.Path, "/") {
|
||||
urlPrefix.Path = urlPrefix.Path[:len(urlPrefix.Path)-1]
|
||||
}
|
||||
// Validate urlPrefix
|
||||
if urlPrefix.Scheme != "http" && urlPrefix.Scheme != "https" {
|
||||
return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, urlPrefix.Scheme)
|
||||
|
||||
@@ -82,14 +82,6 @@ users:
|
||||
headers: foobar
|
||||
`)
|
||||
|
||||
// Invalid keep_original_host value
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
url_prefix: http://foo.bar
|
||||
keep_original_host: foobar
|
||||
`)
|
||||
|
||||
// empty url_prefix
|
||||
f(`
|
||||
users:
|
||||
@@ -418,7 +410,7 @@ users:
|
||||
HeadersConf: HeadersConf{
|
||||
RequestHeaders: []*Header{
|
||||
mustNewHeader("'foo: bar'"),
|
||||
mustNewHeader("'xxx:'"),
|
||||
mustNewHeader("'xxx: y'"),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -437,7 +429,7 @@ users:
|
||||
url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
|
||||
headers:
|
||||
- "foo: bar"
|
||||
- "xxx:"
|
||||
- "xxx: y"
|
||||
`, map[string]*UserInfo{
|
||||
getHTTPAuthBearerToken("foo"): sharedUserInfo,
|
||||
getHTTPAuthBasicToken("foo", ""): sharedUserInfo,
|
||||
@@ -466,7 +458,6 @@ users:
|
||||
})
|
||||
|
||||
// with default url
|
||||
keepOriginalHost := true
|
||||
f(`
|
||||
users:
|
||||
- bearer_token: foo
|
||||
@@ -478,7 +469,6 @@ users:
|
||||
headers:
|
||||
- "foo: bar"
|
||||
- "xxx: y"
|
||||
keep_original_host: true
|
||||
default_url:
|
||||
- http://default1/select/0/prometheus
|
||||
- http://default2/select/0/prometheus
|
||||
@@ -501,7 +491,6 @@ users:
|
||||
mustNewHeader("'foo: bar'"),
|
||||
mustNewHeader("'xxx: y'"),
|
||||
},
|
||||
KeepOriginalHost: &keepOriginalHost,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -528,7 +517,6 @@ users:
|
||||
mustNewHeader("'foo: bar'"),
|
||||
mustNewHeader("'xxx: y'"),
|
||||
},
|
||||
KeepOriginalHost: &keepOriginalHost,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -548,17 +536,12 @@ users:
|
||||
metric_labels:
|
||||
dc: eu
|
||||
team: dev
|
||||
keep_original_host: true
|
||||
- username: foo-same
|
||||
password: bar
|
||||
url_prefix: https://bar/x
|
||||
metric_labels:
|
||||
backend_env: test
|
||||
team: accounting
|
||||
headers:
|
||||
- "foo: bar"
|
||||
response_headers:
|
||||
- "Abc: def"
|
||||
`, map[string]*UserInfo{
|
||||
getHTTPAuthBasicToken("foo-same", "baz"): {
|
||||
Username: "foo-same",
|
||||
@@ -568,9 +551,6 @@ users:
|
||||
"dc": "eu",
|
||||
"team": "dev",
|
||||
},
|
||||
HeadersConf: HeadersConf{
|
||||
KeepOriginalHost: &keepOriginalHost,
|
||||
},
|
||||
},
|
||||
getHTTPAuthBasicToken("foo-same", "bar"): {
|
||||
Username: "foo-same",
|
||||
@@ -580,14 +560,6 @@ users:
|
||||
"backend_env": "test",
|
||||
"team": "accounting",
|
||||
},
|
||||
HeadersConf: HeadersConf{
|
||||
RequestHeaders: []*Header{
|
||||
mustNewHeader("'foo: bar'"),
|
||||
},
|
||||
ResponseHeaders: []*Header{
|
||||
mustNewHeader("'Abc: def'"),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -723,9 +695,9 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
|
||||
|
||||
fn := func(ns ...int) {
|
||||
t.Helper()
|
||||
pbus := up.bus.Load()
|
||||
bus := *pbus
|
||||
for i, b := range bus {
|
||||
bus := up.bus.Load()
|
||||
pbus := *bus
|
||||
for i, b := range pbus {
|
||||
got := int(b.concurrentRequests.Load())
|
||||
exp := ns[i]
|
||||
if got != exp {
|
||||
@@ -735,39 +707,39 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
|
||||
}
|
||||
|
||||
up.getBackendURL()
|
||||
fn(1, 0, 0)
|
||||
fn(0, 1, 0)
|
||||
up.getBackendURL()
|
||||
fn(1, 1, 0)
|
||||
fn(0, 1, 1)
|
||||
up.getBackendURL()
|
||||
fn(1, 1, 1)
|
||||
|
||||
up.getBackendURL()
|
||||
up.getBackendURL()
|
||||
fn(2, 2, 1)
|
||||
fn(1, 2, 2)
|
||||
|
||||
bus := up.bus.Load()
|
||||
pbus := *bus
|
||||
pbus[0].concurrentRequests.Add(2)
|
||||
pbus[2].concurrentRequests.Add(5)
|
||||
fn(4, 2, 6)
|
||||
fn(3, 2, 7)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(4, 3, 6)
|
||||
fn(3, 3, 7)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(4, 4, 6)
|
||||
fn(3, 4, 7)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(4, 5, 6)
|
||||
fn(4, 4, 7)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(5, 5, 6)
|
||||
fn(5, 4, 7)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(6, 5, 6)
|
||||
fn(5, 5, 7)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(6, 6, 6)
|
||||
fn(6, 5, 7)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(6, 6, 7)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 8427
|
||||
|
||||
ENTRYPOINT ["/vmauth-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./vmauth-prod
|
||||
|
||||
@@ -50,7 +50,7 @@ var (
|
||||
`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
|
||||
failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
|
||||
maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
|
||||
"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
|
||||
"Bigger values may require more memory. Negative or zero values disable request body caching and retries.")
|
||||
backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
|
||||
"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
|
||||
backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
|
||||
@@ -117,7 +117,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
handleMissingAuthorizationError(w)
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
|
||||
http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -188,27 +189,26 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
// to a route that is not in the configuration for unauthorized user.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5236
|
||||
if ui.BearerToken == "" && ui.Username == "" && len(*authUsers.Load()) > 0 {
|
||||
handleMissingAuthorizationError(w)
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
|
||||
http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
missingRouteRequests.Inc()
|
||||
httpserver.Errorf(w, r, "missing route for %s", u.String())
|
||||
httpserver.Errorf(w, r, "missing route for %q", u.String())
|
||||
return
|
||||
}
|
||||
up, hc = ui.DefaultURL, ui.HeadersConf
|
||||
isDefault = true
|
||||
}
|
||||
|
||||
rtb := getReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
|
||||
defer putReadTrackingBody(rtb)
|
||||
r.Body = rtb
|
||||
|
||||
// caching makes sense only for positive non zero size
|
||||
if maxRequestBodySizeToRetry.IntN() > 0 {
|
||||
rtb := getReadTrackingBody(r.Body, int(r.ContentLength))
|
||||
defer putReadTrackingBody(rtb)
|
||||
r.Body = rtb
|
||||
}
|
||||
maxAttempts := up.getBackendsCount()
|
||||
for i := 0; i < maxAttempts; i++ {
|
||||
bu := up.getBackendURL()
|
||||
if bu == nil {
|
||||
break
|
||||
}
|
||||
targetURL := bu.url
|
||||
// Don't change path and add request_path query param for default route.
|
||||
if isDefault {
|
||||
@@ -218,15 +218,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
} else { // Update path for regular routes.
|
||||
targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
|
||||
}
|
||||
|
||||
wasLocalRetry := false
|
||||
again:
|
||||
ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
|
||||
if needLocalRetry && !wasLocalRetry {
|
||||
wasLocalRetry = true
|
||||
goto again
|
||||
}
|
||||
|
||||
ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
|
||||
bu.put()
|
||||
if ok {
|
||||
return
|
||||
@@ -234,28 +226,29 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
bu.setBroken()
|
||||
}
|
||||
err := &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
|
||||
StatusCode: http.StatusBadGateway,
|
||||
Err: fmt.Errorf("all the backends for the user %q are unavailable", ui.name()),
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
ui.backendErrors.Inc()
|
||||
}
|
||||
|
||||
func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
|
||||
func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
|
||||
// This code has been copied from net/http/httputil/reverseproxy.go
|
||||
req := sanitizeRequestHeaders(r)
|
||||
|
||||
req.URL = targetURL
|
||||
req.Header.Set("User-Agent", "vmauth")
|
||||
updateHeadersByConfig(req.Header, hc.RequestHeaders)
|
||||
if hc.KeepOriginalHost == nil || !*hc.KeepOriginalHost {
|
||||
if host := getHostHeader(hc.RequestHeaders); host != "" {
|
||||
req.Host = host
|
||||
} else {
|
||||
req.Host = targetURL.Host
|
||||
}
|
||||
}
|
||||
|
||||
if req.URL.Scheme == "https" || ui.overrideHostHeader {
|
||||
// Override req.Host only for https requests, since https server verifies hostnames during TLS handshake,
|
||||
// so it expects the targetURL.Host in the request.
|
||||
// There is no need in overriding the req.Host for http requests, since it is expected that backend server
|
||||
// may properly process queries with the original req.Host.
|
||||
req.Host = targetURL.Host
|
||||
}
|
||||
updateHeadersByConfig(req.Header, hc.RequestHeaders)
|
||||
var trivialRetries int
|
||||
rtb, rtbOK := req.Body.(*readTrackingBody)
|
||||
again:
|
||||
res, err := ui.rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
@@ -267,7 +260,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
|
||||
// Timed out request must be counted as errors, since this usually means that the backend is slow.
|
||||
ui.backendErrors.Inc()
|
||||
}
|
||||
return true, false
|
||||
return true
|
||||
}
|
||||
if !rtbOK || !rtb.canRetry() {
|
||||
// Request body cannot be re-sent to another backend. Return the error to the client then.
|
||||
@@ -277,19 +270,19 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
ui.backendErrors.Inc()
|
||||
return true, false
|
||||
return true
|
||||
}
|
||||
if netutil.IsTrivialNetworkError(err) {
|
||||
// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
|
||||
return false, true
|
||||
// one time retry trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
|
||||
if (netutil.IsTrivialNetworkError(err) || errors.Is(err, io.EOF)) && trivialRetries < 1 {
|
||||
trivialRetries++
|
||||
goto again
|
||||
}
|
||||
|
||||
// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
|
||||
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
|
||||
// NOTE: do not use httpserver.GetRequestURI
|
||||
// it explicitly reads request body, which may fail retries.
|
||||
logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
|
||||
return false, false
|
||||
return false
|
||||
}
|
||||
if slices.Contains(retryStatusCodes, res.StatusCode) {
|
||||
_ = res.Body.Close()
|
||||
@@ -303,7 +296,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
ui.backendErrors.Inc()
|
||||
return true, false
|
||||
return true
|
||||
}
|
||||
// Retry requests at other backends if it matches retryStatusCodes.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
|
||||
@@ -312,7 +305,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
|
||||
// it explicitly reads request body, which may fail retries.
|
||||
logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
|
||||
remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
|
||||
return false, false
|
||||
return false
|
||||
}
|
||||
removeHopHeaders(res.Header)
|
||||
copyHeader(w.Header(), res.Header)
|
||||
@@ -328,9 +321,9 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
|
||||
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
|
||||
requestURI := httpserver.GetRequestURI(r)
|
||||
logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
|
||||
return true, false
|
||||
return true
|
||||
}
|
||||
return true, false
|
||||
return true
|
||||
}
|
||||
|
||||
var copyBufPool bytesutil.ByteBufferPool
|
||||
@@ -343,21 +336,12 @@ func copyHeader(dst, src http.Header) {
|
||||
}
|
||||
}
|
||||
|
||||
func getHostHeader(headers []*Header) string {
|
||||
for _, h := range headers {
|
||||
if h.Name == "Host" {
|
||||
return h.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func updateHeadersByConfig(dst http.Header, src []*Header) {
|
||||
for _, h := range src {
|
||||
func updateHeadersByConfig(headers http.Header, config []*Header) {
|
||||
for _, h := range config {
|
||||
if h.Value == "" {
|
||||
dst.Del(h.Name)
|
||||
headers.Del(h.Name)
|
||||
} else {
|
||||
dst.Set(h.Name, h.Value)
|
||||
headers.Set(h.Name, h.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -497,11 +481,6 @@ See the docs at https://docs.victoriametrics.com/vmauth/ .
|
||||
flagutil.Usage(s)
|
||||
}
|
||||
|
||||
func handleMissingAuthorizationError(w http.ResponseWriter) {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
|
||||
http.Error(w, "missing 'Authorization' request header", http.StatusUnauthorized)
|
||||
}
|
||||
|
||||
func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
w.Header().Add("Retry-After", "10")
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
@@ -511,124 +490,85 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
}
|
||||
|
||||
// readTrackingBody must be obtained via getReadTrackingBody()
|
||||
type readTrackingBody struct {
|
||||
// maxBodySize is the maximum body size to cache in buf.
|
||||
//
|
||||
// Bigger bodies cannot be retried.
|
||||
maxBodySize int
|
||||
|
||||
// r contains reader for initial data reading
|
||||
r io.ReadCloser
|
||||
|
||||
// buf is a buffer for data read from r. Buf size is limited by maxBodySize.
|
||||
// If more than maxBodySize is read from r, then cannotRetry is set to true.
|
||||
// buf is a buffer for data read from r. Buf size is limited by maxRequestBodySizeToRetry.
|
||||
// If more than maxRequestBodySizeToRetry is read from r, then cannotRetry is set to true.
|
||||
buf []byte
|
||||
|
||||
// readBuf points to the cached data at buf, which must be read in the next call to Read().
|
||||
readBuf []byte
|
||||
|
||||
// cannotRetry is set to true when more than maxBodySize bytes are read from r.
|
||||
// cannotRetry is set to true when more than maxRequestBodySizeToRetry are read from r.
|
||||
// In this case the read data cannot fit buf, so it cannot be re-read from buf.
|
||||
cannotRetry bool
|
||||
|
||||
// bufComplete is set to true when buf contains complete request body read from r.
|
||||
bufComplete bool
|
||||
|
||||
// offset is an offset at buf for the next data read if needReadBuf is set to true.
|
||||
offset int
|
||||
}
|
||||
|
||||
func (rtb *readTrackingBody) reset() {
|
||||
rtb.maxBodySize = 0
|
||||
rtb.r = nil
|
||||
rtb.buf = rtb.buf[:0]
|
||||
rtb.readBuf = nil
|
||||
rtb.cannotRetry = false
|
||||
rtb.bufComplete = false
|
||||
}
|
||||
|
||||
func getReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
|
||||
v := readTrackingBodyPool.Get()
|
||||
if v == nil {
|
||||
v = &readTrackingBody{}
|
||||
}
|
||||
rtb := v.(*readTrackingBody)
|
||||
|
||||
if maxBodySize < 0 {
|
||||
maxBodySize = 0
|
||||
}
|
||||
rtb.maxBodySize = maxBodySize
|
||||
|
||||
if r == nil {
|
||||
// This is GET request without request body
|
||||
r = (*zeroReader)(nil)
|
||||
}
|
||||
rtb.r = r
|
||||
return rtb
|
||||
}
|
||||
|
||||
type zeroReader struct{}
|
||||
|
||||
func (r *zeroReader) Read(_ []byte) (int, error) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
func (r *zeroReader) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func putReadTrackingBody(rtb *readTrackingBody) {
|
||||
rtb.reset()
|
||||
readTrackingBodyPool.Put(rtb)
|
||||
}
|
||||
|
||||
var readTrackingBodyPool sync.Pool
|
||||
|
||||
// Read implements io.Reader interface.
|
||||
// Read implements io.Reader interface
|
||||
// tracks body reading requests
|
||||
func (rtb *readTrackingBody) Read(p []byte) (int, error) {
|
||||
if len(rtb.readBuf) > 0 {
|
||||
n := copy(p, rtb.readBuf)
|
||||
rtb.readBuf = rtb.readBuf[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
if rtb.r == nil {
|
||||
if rtb.bufComplete {
|
||||
return 0, io.EOF
|
||||
if rtb.offset < len(rtb.buf) {
|
||||
if rtb.cannotRetry {
|
||||
return 0, fmt.Errorf("cannot retry reading data from buf")
|
||||
}
|
||||
return 0, fmt.Errorf("cannot read client request body after closing client reader")
|
||||
nb := copy(p, rtb.buf[rtb.offset:])
|
||||
rtb.offset += nb
|
||||
if rtb.bufComplete {
|
||||
if rtb.offset == len(rtb.buf) {
|
||||
return nb, io.EOF
|
||||
}
|
||||
return nb, nil
|
||||
}
|
||||
if nb < len(p) {
|
||||
nr, err := rtb.readFromStream(p[nb:])
|
||||
return nb + nr, err
|
||||
}
|
||||
return nb, nil
|
||||
}
|
||||
if rtb.bufComplete {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return rtb.readFromStream(p)
|
||||
}
|
||||
|
||||
func (rtb *readTrackingBody) readFromStream(p []byte) (int, error) {
|
||||
if rtb.r == nil {
|
||||
return 0, fmt.Errorf("cannot read data after closing the reader")
|
||||
}
|
||||
n, err := rtb.r.Read(p)
|
||||
if rtb.cannotRetry {
|
||||
return n, err
|
||||
}
|
||||
|
||||
if len(rtb.buf)+n > rtb.maxBodySize {
|
||||
if rtb.offset+n > maxRequestBodySizeToRetry.IntN() {
|
||||
rtb.cannotRetry = true
|
||||
}
|
||||
if n > 0 {
|
||||
rtb.offset += n
|
||||
rtb.buf = append(rtb.buf, p[:n]...)
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
rtb.bufComplete = true
|
||||
return n, err
|
||||
}
|
||||
rtb.cannotRetry = true
|
||||
return n, err
|
||||
}
|
||||
rtb.buf = append(rtb.buf, p[:n]...)
|
||||
if err == io.EOF {
|
||||
rtb.bufComplete = true
|
||||
}
|
||||
return n, err
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (rtb *readTrackingBody) canRetry() bool {
|
||||
if rtb.cannotRetry {
|
||||
return false
|
||||
}
|
||||
if rtb.bufComplete {
|
||||
return true
|
||||
}
|
||||
return rtb.r != nil
|
||||
return !rtb.cannotRetry
|
||||
}
|
||||
|
||||
// Close implements io.Closer interface.
|
||||
func (rtb *readTrackingBody) Close() error {
|
||||
if !rtb.cannotRetry {
|
||||
rtb.readBuf = rtb.buf
|
||||
} else {
|
||||
rtb.readBuf = nil
|
||||
}
|
||||
rtb.offset = 0
|
||||
|
||||
// Close rtb.r only if the request body is completely read or if it is too big.
|
||||
// http.Roundtrip performs body.Close call even without any Read calls,
|
||||
@@ -644,3 +584,38 @@ func (rtb *readTrackingBody) Close() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var readTrackingBodyPool sync.Pool
|
||||
|
||||
func getReadTrackingBody(origin io.ReadCloser, b int) *readTrackingBody {
|
||||
bufSize := 1024
|
||||
if b > 0 && b < maxRequestBodySizeToRetry.IntN() {
|
||||
bufSize = b
|
||||
}
|
||||
v := readTrackingBodyPool.Get()
|
||||
if v == nil {
|
||||
v = &readTrackingBody{
|
||||
buf: make([]byte, 0, bufSize),
|
||||
}
|
||||
}
|
||||
rtb := v.(*readTrackingBody)
|
||||
rtb.r = origin
|
||||
if bufSize > cap(rtb.buf) {
|
||||
rtb.buf = make([]byte, 0, bufSize)
|
||||
}
|
||||
|
||||
return rtb
|
||||
}
|
||||
|
||||
func putReadTrackingBody(rtb *readTrackingBody) {
|
||||
if rtb.r != nil {
|
||||
_ = rtb.r.Close()
|
||||
}
|
||||
rtb.r = nil
|
||||
rtb.buf = rtb.buf[:0]
|
||||
rtb.offset = 0
|
||||
rtb.cannotRetry = false
|
||||
rtb.bufComplete = false
|
||||
|
||||
readTrackingBodyPool.Put(rtb)
|
||||
}
|
||||
|
||||
@@ -2,520 +2,18 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
)
|
||||
|
||||
func TestRequestHandler(t *testing.T) {
|
||||
f := func(cfgStr, requestURL string, backendHandler http.HandlerFunc, responseExpected string) {
|
||||
func TestReadTrackingBodyRetrySuccess(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
ts := httptest.NewServer(backendHandler)
|
||||
defer ts.Close()
|
||||
|
||||
cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
|
||||
responseExpected = strings.ReplaceAll(responseExpected, "{BACKEND}", ts.URL)
|
||||
|
||||
cfgOrigP := authConfigData.Load()
|
||||
if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
|
||||
t.Fatalf("cannot load config data: %s", err)
|
||||
rtb := &readTrackingBody{
|
||||
r: io.NopCloser(bytes.NewBufferString(s)),
|
||||
}
|
||||
defer func() {
|
||||
cfgOrig := []byte("unauthorized_user:\n url_prefix: http://foo/bar")
|
||||
if cfgOrigP != nil {
|
||||
cfgOrig = *cfgOrigP
|
||||
}
|
||||
_, err := reloadAuthConfigData(cfgOrig)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot load the original config: %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
r, err := http.NewRequest(http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot initialize http request: %s", err)
|
||||
}
|
||||
|
||||
r.RequestURI = r.URL.RequestURI()
|
||||
r.RemoteAddr = "42.2.3.84:6789"
|
||||
r.Header.Set("X-Forwarded-For", "12.34.56.78")
|
||||
r.Header.Set("Connection", "Some-Header,Other-Header")
|
||||
r.Header.Set("Some-Header", "foobar")
|
||||
r.Header.Set("Pass-Header", "abc")
|
||||
|
||||
w := &fakeResponseWriter{}
|
||||
if !requestHandler(w, r) {
|
||||
t.Fatalf("unexpected false is returned from requestHandler")
|
||||
}
|
||||
|
||||
response := w.getResponse()
|
||||
response = strings.ReplaceAll(response, "\r\n", "\n")
|
||||
response = strings.TrimSpace(response)
|
||||
responseExpected = strings.TrimSpace(responseExpected)
|
||||
if response != responseExpected {
|
||||
t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, responseExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// regular url_prefix
|
||||
cfgStr := `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/foo?bar=baz`
|
||||
requestURL := "http://some-host.com/abc/def?some_arg=some_value"
|
||||
backendHandler := func(w http.ResponseWriter, r *http.Request) {
|
||||
h := w.Header()
|
||||
h.Set("Connection", "close")
|
||||
h.Set("Foo", "bar")
|
||||
|
||||
var bb bytes.Buffer
|
||||
if err := r.Header.Write(&bb); err != nil {
|
||||
panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
|
||||
}
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
|
||||
}
|
||||
responseExpected := `
|
||||
statusCode=200
|
||||
Foo: bar
|
||||
requested_url={BACKEND}/foo/abc/def?bar=baz&some_arg=some_value
|
||||
Pass-Header: abc
|
||||
User-Agent: vmauth
|
||||
X-Forwarded-For: 12.34.56.78, 42.2.3.84`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// keep_original_host
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
keep_original_host: true`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url=http://some-host.com/foo/abc/def?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// override user-agent header
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
headers:
|
||||
- "User-Agent: foobar"`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/abc/def?bar=baz
|
||||
User-Agent=foobar`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// delete user-agent header
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
headers:
|
||||
- "User-Agent:"`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/abc/def?bar=baz
|
||||
User-Agent=Go-http-client/1.1`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// override request host with non-empty host
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
headers:
|
||||
- "Host: other-host:12345"
|
||||
- "abc:"`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url=http://other-host:12345/foo/abc/def?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// override request host with empty host
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
headers:
|
||||
- "Host:"`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/abc/def?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// /-/reload handler failure
|
||||
origAuthKey := reloadAuthKey.Get()
|
||||
if err := reloadAuthKey.Set("secret"); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo"`
|
||||
requestURL = "http://some-host.com/-/reload"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
The provided authKey doesn't match -reloadAuthKey`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
if err := reloadAuthKey.Set(origAuthKey); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
// missing authorization
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: foo
|
||||
url_prefix: "{BACKEND}/bar"`
|
||||
requestURL = "http://some-host.com/a/b"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Www-Authenticate: Basic realm="Restricted"
|
||||
missing 'Authorization' request header`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// incorrect authorization
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: foo
|
||||
password: secret
|
||||
url_prefix: "{BACKEND}/bar"`
|
||||
requestURL = "http://foo:invalid-secret@some-host.com/a/b"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Unauthorized`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// incorrect authorization with logging invalid auth tokens
|
||||
origLogInvalidAuthTokens := *logInvalidAuthTokens
|
||||
*logInvalidAuthTokens = true
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: foo
|
||||
password: secret
|
||||
url_prefix: "{BACKEND}/bar"`
|
||||
requestURL = "http://foo:invalid-secret@some-host.com/a/b?c=d"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /a/b?c=d; cannot authorize request with auth tokens ["http_auth:Basic Zm9vOmludmFsaWQtc2VjcmV0"]`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
*logInvalidAuthTokens = origLogInvalidAuthTokens
|
||||
|
||||
// correct authorization
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: foo
|
||||
password: secret
|
||||
url_prefix: "{BACKEND}/bar"`
|
||||
requestURL = "http://foo:secret@some-host.com/a/b"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/bar/a/b`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify how path cleanup works
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/foo?bar=baz`
|
||||
requestURL = "http://some-host.com/../../a//.///bar/"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/a/bar/?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify how path cleanup works for url without path
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/foo?bar=baz`
|
||||
requestURL = "http://some-host.com/"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify how path cleanup works for url without path if url_prefix path ends with /
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/foo/?bar=baz`
|
||||
requestURL = "http://some-host.com/"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify how path cleanup works for url without path and the url_prefix without path prefix
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/?bar=baz`
|
||||
requestURL = "http://some-host.com/"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify routing to default_url
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/foo/.+"]
|
||||
url_prefix: {BACKEND}/x-foo/
|
||||
default_url: {BACKEND}/404.html`
|
||||
requestURL = "http://some-host.com/abc?de=fg"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/404.html?request_path=http%3A%2F%2Fsome-host.com%2Fabc%3Fde%3Dfg`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify routing to default url_prefix
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/foo/.+"]
|
||||
url_prefix: {BACKEND}/x-foo/
|
||||
url_prefix: {BACKEND}/default`
|
||||
requestURL = "http://some-host.com/abc?de=fg"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/default/abc?de=fg`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// missing default_url and default url_prefix for unauthorized user
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/foo/.+"]
|
||||
url_prefix: {BACKEND}/x-foo/`
|
||||
requestURL = "http://some-host.com/abc?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=400
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /abc?de=fg; missing route for http://some-host.com/abc?de=fg`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// missing default_url and default url_prefix for unauthorized user when there are configs for authorized users
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: some-user
|
||||
url_map:
|
||||
- src_paths: ["/foo/.+"]
|
||||
url_prefix: {BACKEND}/x-foo/
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/abc/.*"]
|
||||
url_prefix: {BACKEND}/x-bar`
|
||||
requestURL = "http://some-host.com/abc?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Www-Authenticate: Basic realm="Restricted"
|
||||
missing 'Authorization' request header`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// all the backend_urls are unavailable for unauthorized user
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/foo/.*"]
|
||||
url_prefix:
|
||||
- http://127.0.0.1:1/
|
||||
- http://127.0.0.1:2/`
|
||||
requestURL = "http://some-host.com/foo/?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// all the backend_urls are unavailable for authorized user
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: some-user
|
||||
url_map:
|
||||
- src_paths: ["/foo/.*"]
|
||||
url_prefix:
|
||||
- http://127.0.0.1:1/
|
||||
- http://127.0.0.1:2/`
|
||||
requestURL = "http://some-user@some-host.com/foo/?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "some-user" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// zero discovered backend IPs
|
||||
customResolver := &fakeResolver{
|
||||
Resolver: &net.Resolver{},
|
||||
lookupIPAddrResults: map[string][]net.IPAddr{
|
||||
"some-addr": {},
|
||||
},
|
||||
}
|
||||
origResolver := netutil.Resolver
|
||||
netutil.Resolver = customResolver
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: ['http://some-addr:1234/foo/bar']
|
||||
discover_backend_ips: true`
|
||||
requestURL = "http://abc.com/def/?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /def/?de=fg; all the 0 backends for the user "" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
netutil.Resolver = origResolver
|
||||
|
||||
// retry_status_codes failure
|
||||
var retries atomic.Int64
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
|
||||
retry_status_codes: [500, 502]`
|
||||
requestURL = "http://some-host.com/foo/?de=fg"
|
||||
backendHandler = func(w http.ResponseWriter, _ *http.Request) {
|
||||
retries.Add(1)
|
||||
w.WriteHeader(500)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
if n := retries.Load(); n != 2 {
|
||||
t.Fatalf("unexpected number of retries; got %d; want 2", n)
|
||||
}
|
||||
|
||||
// retry_status_codes success
|
||||
retries.Store(0)
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
|
||||
retry_status_codes: [500, 502]`
|
||||
requestURL = "http://some-host.com/foo/?de=fg"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
if n := retries.Add(1); n < 2 {
|
||||
w.WriteHeader(500)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/path2/foo/?de=fg`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
if n := retries.Load(); n != 2 {
|
||||
t.Fatalf("unexpected number of retries; got %d; want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeResponseWriter struct {
|
||||
h http.Header
|
||||
|
||||
bb bytes.Buffer
|
||||
}
|
||||
|
||||
func (w *fakeResponseWriter) getResponse() string {
|
||||
return w.bb.String()
|
||||
}
|
||||
|
||||
func (w *fakeResponseWriter) Header() http.Header {
|
||||
if w.h == nil {
|
||||
w.h = http.Header{}
|
||||
}
|
||||
return w.h
|
||||
}
|
||||
|
||||
func (w *fakeResponseWriter) Write(p []byte) (int, error) {
|
||||
return w.bb.Write(p)
|
||||
}
|
||||
|
||||
func (w *fakeResponseWriter) WriteHeader(statusCode int) {
|
||||
fmt.Fprintf(&w.bb, "statusCode=%d\n", statusCode)
|
||||
if w.h == nil {
|
||||
return
|
||||
}
|
||||
err := w.h.WriteSubset(&w.bb, map[string]bool{
|
||||
"Content-Length": true,
|
||||
"Content-Type": true,
|
||||
"Date": true,
|
||||
"X-Content-Type-Options": true,
|
||||
})
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("cannot marshal headers: %s", err))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadTrackingBody_RetrySuccess(t *testing.T) {
|
||||
f := func(s string, maxBodySize int) {
|
||||
t.Helper()
|
||||
|
||||
rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
|
||||
defer putReadTrackingBody(rtb)
|
||||
|
||||
if !rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return true before reading anything")
|
||||
t.Fatalf("canRetry() must return true")
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
data, err := io.ReadAll(rtb)
|
||||
@@ -534,99 +32,41 @@ func TestReadTrackingBody_RetrySuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
f("", 0)
|
||||
f("", -1)
|
||||
f("", 100)
|
||||
f("foo", 100)
|
||||
f("foobar", 100)
|
||||
f(newTestString(1000), 1000)
|
||||
f("")
|
||||
f("foo")
|
||||
f("foobar")
|
||||
f(newTestString(maxRequestBodySizeToRetry.IntN()))
|
||||
}
|
||||
|
||||
func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
|
||||
f := func(s string, maxBodySize int) {
|
||||
func TestReadTrackingBodyRetryFailure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
// Check the case with partial read
|
||||
rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
|
||||
defer putReadTrackingBody(rtb)
|
||||
|
||||
for i := 0; i < len(s); i++ {
|
||||
buf := make([]byte, i)
|
||||
n, err := io.ReadFull(rtb, buf)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error when reading %d bytes: %s", i, err)
|
||||
}
|
||||
if n != i {
|
||||
t.Fatalf("unexpected number of bytes read; got %d; want %d", n, i)
|
||||
}
|
||||
if string(buf) != s[:i] {
|
||||
t.Fatalf("unexpected data read with the length %d\ngot\n%s\nwant\n%s", i, buf, s[:i])
|
||||
}
|
||||
if err := rtb.Close(); err != nil {
|
||||
t.Fatalf("unexpected error when closing reader after reading %d bytes", i)
|
||||
}
|
||||
if !rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return true after closing the reader after reading %d bytes", i)
|
||||
}
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(rtb)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error when reading all the data: %s", err)
|
||||
}
|
||||
if string(data) != s {
|
||||
t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
|
||||
}
|
||||
if err := rtb.Close(); err != nil {
|
||||
t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
|
||||
rtb := &readTrackingBody{
|
||||
r: io.NopCloser(bytes.NewBufferString(s)),
|
||||
}
|
||||
if !rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return true after closing the reader after reading all the input")
|
||||
}
|
||||
}
|
||||
|
||||
f("", 0)
|
||||
f("", -1)
|
||||
f("", 100)
|
||||
f("foo", 100)
|
||||
f("foobar", 100)
|
||||
f(newTestString(1000), 1000)
|
||||
}
|
||||
|
||||
func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
|
||||
f := func(s string, maxBodySize int) {
|
||||
t.Helper()
|
||||
|
||||
rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
|
||||
defer putReadTrackingBody(rtb)
|
||||
|
||||
if !rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return true before reading anything")
|
||||
t.Fatalf("canRetry() must return true")
|
||||
}
|
||||
buf := make([]byte, 1)
|
||||
n, err := io.ReadFull(rtb, buf)
|
||||
n, err := rtb.Read(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error when reading a single byte: %s", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
|
||||
}
|
||||
if !rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return true after reading one byte")
|
||||
}
|
||||
data, err := io.ReadAll(rtb)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error when reading all the data: %s", err)
|
||||
}
|
||||
dataRead := string(buf) + string(data)
|
||||
if dataRead != s {
|
||||
t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", dataRead, s)
|
||||
if string(buf)+string(data) != s {
|
||||
t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", string(buf)+string(data), s)
|
||||
}
|
||||
if err := rtb.Close(); err != nil {
|
||||
t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
|
||||
}
|
||||
if rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return false after closing the reader")
|
||||
t.Fatalf("canRetry() must return false")
|
||||
}
|
||||
|
||||
data, err = io.ReadAll(rtb)
|
||||
@@ -638,55 +78,132 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
const maxBodySize = 1000
|
||||
f(newTestString(maxBodySize+1), maxBodySize)
|
||||
f(newTestString(2*maxBodySize), maxBodySize)
|
||||
f(newTestString(maxRequestBodySizeToRetry.IntN() + 1))
|
||||
f(newTestString(2 * maxRequestBodySizeToRetry.IntN()))
|
||||
}
|
||||
|
||||
func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
|
||||
f := func(s string, maxBodySize int) {
|
||||
t.Helper()
|
||||
// request body not over maxRequestBodySizeToRetry
|
||||
// 1. When writing data downstream, buf only caches part of the data because the downstream connection is disconnected.
|
||||
// 2. retry request: because buf caches some data, first read buf and then read stream when retrying
|
||||
// 3. retry request: the data has been read to buf in the second step. if the request fails, retry to read all buf later.
|
||||
func TestRetryReadSuccessAfterPartialRead(t *testing.T) {
|
||||
f := func(s string) {
|
||||
rtb := &readTrackingBody{
|
||||
r: io.NopCloser(bytes.NewBufferString(s)),
|
||||
buf: make([]byte, 0, len(s)),
|
||||
}
|
||||
|
||||
rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
|
||||
defer putReadTrackingBody(rtb)
|
||||
var data []byte
|
||||
var err error
|
||||
halfSize := len(s) / 2
|
||||
if halfSize == 0 {
|
||||
halfSize = 100
|
||||
}
|
||||
buf := make([]byte, halfSize)
|
||||
var n int
|
||||
|
||||
if !rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return true before reading anything")
|
||||
}
|
||||
data, err := io.ReadAll(rtb)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error when reading all the data: %s", err)
|
||||
}
|
||||
if string(data) != s {
|
||||
t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
|
||||
// read part of the data
|
||||
n, err = rtb.Read(buf[:])
|
||||
data = append(data, buf[:n]...)
|
||||
if err != nil && err != io.EOF {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
// request failed when output stream is closed (eg: server connection reset)
|
||||
// would close the reader
|
||||
if err := rtb.Close(); err != nil {
|
||||
t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
|
||||
}
|
||||
if !rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return true")
|
||||
}
|
||||
|
||||
if rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return false after closing the reader")
|
||||
// retry read (read buf + remaining data)
|
||||
data = data[:0]
|
||||
err = nil
|
||||
for err == nil {
|
||||
n, err = rtb.Read(buf[:])
|
||||
data = append(data, buf[:n]...)
|
||||
}
|
||||
data, err = io.ReadAll(rtb)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
if err != io.EOF {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if len(data) != 0 {
|
||||
t.Fatalf("unexpected non-empty data read: %q", data)
|
||||
if string(data) != s {
|
||||
t.Fatalf("unexpected data read; got\n%s\nwant\n%s", data, s)
|
||||
}
|
||||
// cannotRetry return false
|
||||
// because the request data is not over maxRequestBodySizeToRetry limit
|
||||
if !rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return true")
|
||||
}
|
||||
}
|
||||
|
||||
f("foobar", 0)
|
||||
f(newTestString(1000), 0)
|
||||
f("")
|
||||
f("foo")
|
||||
f("foobar")
|
||||
f(newTestString(maxRequestBodySizeToRetry.IntN()))
|
||||
}
|
||||
|
||||
f("foobar", -1)
|
||||
f(newTestString(1000), -1)
|
||||
// request body over maxRequestBodySizeToRetry
|
||||
// 1. When writing data downstream, buf only caches part of the data because the downstream connection is disconnected.
|
||||
// 2. retry request: because buf caches some data, first read buf and then read stream when retrying
|
||||
// 3. retry request: the data has been read to buf in the second step. if the request fails, retry to read all buf later.
|
||||
func TestRetryReadSuccessAfterPartialReadAndCannotRetryAgain(t *testing.T) {
|
||||
f := func(s string) {
|
||||
rtb := &readTrackingBody{
|
||||
r: io.NopCloser(bytes.NewBufferString(s)),
|
||||
buf: make([]byte, 0, len(s)),
|
||||
}
|
||||
|
||||
var data []byte
|
||||
var err error
|
||||
halfSize := len(s) / 2
|
||||
if halfSize == 0 {
|
||||
halfSize = 100
|
||||
}
|
||||
buf := make([]byte, halfSize)
|
||||
var n int
|
||||
|
||||
// read part of the data
|
||||
n, err = rtb.Read(buf[:])
|
||||
data = append(data, buf[:n]...)
|
||||
if err != nil && err != io.EOF {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
// request failed when output stream is closed (eg: server connection reset)
|
||||
if err := rtb.Close(); err != nil {
|
||||
t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
|
||||
}
|
||||
if !rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return true")
|
||||
}
|
||||
|
||||
// retry read (read buf + remaining data)
|
||||
data = data[:0]
|
||||
err = nil
|
||||
for err == nil {
|
||||
n, err = rtb.Read(buf[:])
|
||||
data = append(data, buf[:n]...)
|
||||
}
|
||||
if err != io.EOF {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if string(data) != s {
|
||||
t.Fatalf("unexpected data read; got\n%s\nwant\n%s", data, s)
|
||||
}
|
||||
|
||||
// cannotRetry returns true
|
||||
// because the request data is over maxRequestBodySizeToRetry limit
|
||||
if rtb.canRetry() {
|
||||
t.Fatalf("canRetry() must return false")
|
||||
}
|
||||
}
|
||||
|
||||
f(newTestString(maxRequestBodySizeToRetry.IntN() + 1))
|
||||
f(newTestString(2 * maxRequestBodySizeToRetry.IntN()))
|
||||
}
|
||||
|
||||
func newTestString(sLen int) string {
|
||||
data := make([]byte, sLen)
|
||||
for i := range data {
|
||||
data[i] = byte(i)
|
||||
}
|
||||
return string(data)
|
||||
return string(make([]byte, sLen))
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -335,62 +335,13 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
|
||||
t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
|
||||
}
|
||||
|
||||
f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.50:8481/select/0/prometheus/api/v1/query?query=up")
|
||||
f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up")
|
||||
f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.50:8481/select/0/prometheus/api/v1/query?query=up")
|
||||
f(ui, `/insert/0/prometheus/api/v1/write`, "http://10.6.142.52:8480/insert/0/prometheus/api/v1/write")
|
||||
// unsuccessful dns resolve
|
||||
f(ui, `/test`, "http://non-exist-dns-addr/test")
|
||||
}
|
||||
|
||||
func TestUserInfoGetBackendURL_SRVZeroBackends(t *testing.T) {
|
||||
f := func(ui *UserInfo, requestURI string) {
|
||||
t.Helper()
|
||||
|
||||
u, err := url.Parse(requestURI)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot parse %q: %s", requestURI, err)
|
||||
}
|
||||
u = normalizeURL(u)
|
||||
up, _ := ui.getURLPrefixAndHeaders(u, nil)
|
||||
if up == nil {
|
||||
t.Fatalf("cannot match available backend: %s", err)
|
||||
}
|
||||
bu := up.getBackendURL()
|
||||
if bu != nil {
|
||||
t.Fatalf("expecting nil backendURL; got %v", bu)
|
||||
}
|
||||
}
|
||||
|
||||
customResolver := &fakeResolver{
|
||||
Resolver: &net.Resolver{},
|
||||
lookupSRVResults: map[string][]*net.SRV{
|
||||
"vmselect": {},
|
||||
},
|
||||
}
|
||||
origResolver := netutil.Resolver
|
||||
netutil.Resolver = customResolver
|
||||
defer func() {
|
||||
netutil.Resolver = origResolver
|
||||
}()
|
||||
|
||||
allowed := true
|
||||
ui := &UserInfo{
|
||||
URLMaps: []URLMap{
|
||||
{
|
||||
SrcPaths: getRegexs([]string{"/select/.+"}),
|
||||
URLPrefix: mustParseURL("http://srv+vmselect"),
|
||||
},
|
||||
},
|
||||
DiscoverBackendIPs: &allowed,
|
||||
URLPrefix: mustParseURL("http://non-exist-dns-addr"),
|
||||
}
|
||||
if err := ui.initURLs(); err != nil {
|
||||
t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
|
||||
}
|
||||
|
||||
f(ui, `/select/0/prometheus/api/v1/query?query=up`)
|
||||
}
|
||||
|
||||
func TestCreateTargetURLFailure(t *testing.T) {
|
||||
f := func(ui *UserInfo, requestURI string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
ENTRYPOINT ["/vmbackup-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./vmbackup-prod
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -10,6 +10,12 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
backoffRetries = 10
|
||||
backoffFactor = 1.8
|
||||
backoffMinDuration = time.Second * 2
|
||||
)
|
||||
|
||||
// retryableFunc describes call back which will repeat on errors
|
||||
type retryableFunc func() error
|
||||
|
||||
@@ -24,22 +30,12 @@ type Backoff struct {
|
||||
}
|
||||
|
||||
// New initialize backoff object
|
||||
func New(retries int, factor float64, minDuration time.Duration) (*Backoff, error) {
|
||||
if retries <= 0 {
|
||||
return nil, fmt.Errorf("number of backoff retries must be greater than 0")
|
||||
}
|
||||
if factor <= 1 {
|
||||
return nil, fmt.Errorf("backoff retry factor must be greater than 1")
|
||||
}
|
||||
if minDuration <= 0 {
|
||||
return nil, fmt.Errorf("backoff retry minimum duration must be greater than 0")
|
||||
}
|
||||
|
||||
func New() *Backoff {
|
||||
return &Backoff{
|
||||
retries: retries,
|
||||
factor: factor,
|
||||
minDuration: minDuration,
|
||||
}, nil
|
||||
retries: backoffRetries,
|
||||
factor: backoffFactor,
|
||||
minDuration: backoffMinDuration,
|
||||
}
|
||||
}
|
||||
|
||||
// Retry process retries until all attempts are completed
|
||||
|
||||
@@ -3,7 +3,6 @@ package backoff
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -111,32 +110,3 @@ func TestBackoffRetry_Success(t *testing.T) {
|
||||
resultExpected := 1
|
||||
f(retryFunc, resultExpected)
|
||||
}
|
||||
|
||||
func TestBackoff_New(t *testing.T) {
|
||||
f := func(retries int, factor float64, minDuration time.Duration, errExpected string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := New(retries, factor, minDuration)
|
||||
if err == nil {
|
||||
if errExpected != "" {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
}
|
||||
return
|
||||
}
|
||||
if !strings.Contains(err.Error(), errExpected) {
|
||||
t.Fatalf("unexpected error: got %q; want %q", err.Error(), errExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// empty retries
|
||||
f(0, 1.1, time.Millisecond*10, "retries must be greater than 0")
|
||||
|
||||
// empty factor
|
||||
f(1, 0, time.Millisecond*10, "factor must be greater than 1")
|
||||
|
||||
// empty minDuration
|
||||
f(1, 1.1, 0, "minimum duration must be greater than 0")
|
||||
|
||||
// no errors
|
||||
f(1, 1.1, time.Millisecond*10, "")
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
ENTRYPOINT ["/vmctl-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./vmctl-prod
|
||||
|
||||
@@ -56,10 +56,6 @@ const (
|
||||
vmRateLimit = "vm-rate-limit"
|
||||
|
||||
vmInterCluster = "vm-intercluster"
|
||||
|
||||
vmBackoffRetries = "vm-backoff-retries"
|
||||
vmBackoffFactor = "vm-backoff-factor"
|
||||
vmBackoffMinDuration = "vm-backoff-min-duration"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -150,21 +146,6 @@ var (
|
||||
Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
|
||||
Value: false,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: vmBackoffRetries,
|
||||
Value: 10,
|
||||
Usage: "How many import retries to perform before giving up.",
|
||||
},
|
||||
&cli.Float64Flag{
|
||||
Name: vmBackoffFactor,
|
||||
Value: 1.8,
|
||||
Usage: "Factor to multiply the base duration after each failed import retry. Must be greater than 1.0",
|
||||
},
|
||||
&cli.DurationFlag{
|
||||
Name: vmBackoffMinDuration,
|
||||
Value: time.Second * 2,
|
||||
Usage: "Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'.",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -449,10 +430,6 @@ const (
|
||||
vmNativeDstCAFile = "vm-native-dst-ca-file"
|
||||
vmNativeDstServerName = "vm-native-dst-server-name"
|
||||
vmNativeDstInsecureSkipVerify = "vm-native-dst-insecure-skip-verify"
|
||||
|
||||
vmNativeBackoffRetries = "vm-native-backoff-retries"
|
||||
vmNativeBackoffFactor = "vm-native-backoff-factor"
|
||||
vmNativeBackoffMinDuration = "vm-native-backoff-min-duration"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -622,21 +599,6 @@ var (
|
||||
"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
|
||||
Value: false,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: vmNativeBackoffRetries,
|
||||
Value: 10,
|
||||
Usage: "How many export/import retries to perform before giving up.",
|
||||
},
|
||||
&cli.Float64Flag{
|
||||
Name: vmNativeBackoffFactor,
|
||||
Value: 1.8,
|
||||
Usage: "Factor to multiply the base duration after each failed export/import retry. Must be greater than 1.0",
|
||||
},
|
||||
&cli.DurationFlag{
|
||||
Name: vmNativeBackoffMinDuration,
|
||||
Value: time.Second * 2,
|
||||
Usage: "Minimum duration to wait before the first export/import retry. Each subsequent export/import retry will be multiplied by the '--vm-native-backoff-factor'.",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@ func main() {
|
||||
|
||||
tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create transport for -%s=%q: %s", otsdbAddr, addr, err)
|
||||
return fmt.Errorf("failed to create Transport: %s", err)
|
||||
}
|
||||
oCfg := opentsdb.Config{
|
||||
Addr: addr,
|
||||
@@ -90,7 +90,6 @@ func main() {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init VM configuration: %s", err)
|
||||
}
|
||||
|
||||
importer, err := vm.NewImporter(ctx, vmCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create VM importer: %s", err)
|
||||
@@ -144,7 +143,6 @@ func main() {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init VM configuration: %s", err)
|
||||
}
|
||||
|
||||
importer, err = vm.NewImporter(ctx, vmCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create VM importer: %s", err)
|
||||
@@ -180,7 +178,7 @@ func main() {
|
||||
|
||||
tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create transport for -%s=%q: %s", remoteReadSrcAddr, addr, err)
|
||||
return fmt.Errorf("failed to create transport: %s", err)
|
||||
}
|
||||
|
||||
rr, err := remoteread.NewClient(remoteread.Config{
|
||||
@@ -203,7 +201,6 @@ func main() {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init VM configuration: %s", err)
|
||||
}
|
||||
|
||||
importer, err := vm.NewImporter(ctx, vmCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create VM importer: %s", err)
|
||||
@@ -236,7 +233,6 @@ func main() {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init VM configuration: %s", err)
|
||||
}
|
||||
|
||||
importer, err = vm.NewImporter(ctx, vmCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create VM importer: %s", err)
|
||||
@@ -276,14 +272,6 @@ func main() {
|
||||
return fmt.Errorf("flag %q can't be empty", vmNativeFilterMatch)
|
||||
}
|
||||
|
||||
bfRetries := c.Int(vmNativeBackoffRetries)
|
||||
bfFactor := c.Float64(vmNativeBackoffFactor)
|
||||
bfMinDuration := c.Duration(vmNativeBackoffMinDuration)
|
||||
bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create backoff object: %s", err)
|
||||
}
|
||||
|
||||
disableKeepAlive := c.Bool(vmNativeDisableHTTPKeepAlive)
|
||||
|
||||
var srcExtraLabels []string
|
||||
@@ -362,7 +350,7 @@ func main() {
|
||||
ExtraLabels: dstExtraLabels,
|
||||
HTTPClient: dstHTTPClient,
|
||||
},
|
||||
backoff: bf,
|
||||
backoff: backoff.New(),
|
||||
cc: c.Int(vmConcurrency),
|
||||
disablePerMetricRequests: c.Bool(vmNativeDisablePerMetricMigration),
|
||||
isNative: !c.Bool(vmNativeDisableBinaryProtocol),
|
||||
@@ -438,15 +426,7 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
|
||||
|
||||
tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
|
||||
if err != nil {
|
||||
return vm.Config{}, fmt.Errorf("failed to create transport for -%s=%q: %s", vmAddr, addr, err)
|
||||
}
|
||||
|
||||
bfRetries := c.Int(vmBackoffRetries)
|
||||
bfFactor := c.Float64(vmBackoffFactor)
|
||||
bfMinDuration := c.Duration(vmBackoffMinDuration)
|
||||
bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
|
||||
if err != nil {
|
||||
return vm.Config{}, fmt.Errorf("failed to create backoff object: %s", err)
|
||||
return vm.Config{}, fmt.Errorf("failed to create Transport: %s", err)
|
||||
}
|
||||
|
||||
return vm.Config{
|
||||
@@ -462,6 +442,5 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
|
||||
RoundDigits: c.Int(vmRoundDigits),
|
||||
ExtraLabels: c.StringSlice(vmExtraLabel),
|
||||
RateLimit: c.Int64(vmRateLimit),
|
||||
Backoff: bf,
|
||||
}, nil
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user