Compare commits

..

5 Commits

Author SHA1 Message Date
Arkadii Yakovets
9b10e6d26b Implement SBOM generation
Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>
2024-07-16 17:28:24 -07:00
Arkadii Yakovets
042444f3ea Merge branch 'master' into ark/add-vm-release-tests 2024-07-16 17:23:36 -07:00
Arkadii Yakovets
21beb994ad Add file contents checks
Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>
2024-07-12 11:50:48 -07:00
Arkadii Yakovets
4605a72684 Fix lint error
Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>
2024-07-12 11:50:48 -07:00
Arkadii Yakovets
08de878047 Add release-victoria-metrics smoke tests
Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>
2024-07-12 11:50:48 -07:00
753 changed files with 18920 additions and 16431 deletions

59
.github/workflows/test_package.yml vendored Normal file
View File

@@ -0,0 +1,59 @@
name: Package tests
on:
push:
branches:
- cluster
- master
paths:
- '.github/workflows/test_package.yml'
- '**/Dockerfile*'
- '**/Makefile'
pull_request:
branches:
- cluster
- master
paths:
- '.github/workflows/test_package.yml'
- '**/Dockerfile*'
- '**/Makefile'
permissions:
contents: read
concurrency:
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
jobs:
test_release:
name: Test Release
runs-on: ubuntu-latest
steps:
- name: Code checkout
uses: actions/checkout@v4
- name: Setup Go
id: go
uses: actions/setup-go@v5
with:
go-version: stable
cache: false
- name: Cache Go artifacts
uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/bin
~/go/pkg/mod
key: go-artifacts-${{ runner.os }}-test-release-${{ steps.go.outputs.go-version }}-${{ hashFiles('Makefile', '**/Dockerfile*', '**/Makefile') }}
restore-keys: go-artifacts-${{ runner.os }}-test-release-
- name: Run make release
run: |
make clean
make release
- name: Run release tests
run: make test-release-victoria-metrics

1
.gitignore vendored
View File

@@ -7,7 +7,6 @@
.vscode
*.test
*.swp
/vmdocs
/gocache-for-docker
/victoria-logs-data
/victoria-metrics-data

207
Makefile
View File

@@ -13,7 +13,10 @@ PKG_TAG := $(BUILDINFO_TAG)
endif
GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
TAR_OWNERSHIP ?= --owner=1000 --group=1000
VICTORIA_LOGS_COMPONENTS = victoria-logs
VICTORIA_METRICS_COMPONENTS = victoria-metrics
VICTORIA_METRICS_UTILS_COMPONENTS = vmagent vmalert vmalert-tool vmauth vmbackup vmrestore vmctl
.PHONY: $(MAKECMDGOALS)
@@ -24,6 +27,73 @@ include deployment/*/Makefile
include dashboards/Makefile
include package/release/Makefile
define RELEASE_GOOS_GOARCH
$(eval PKG_NAME := $(1))
$(eval PKG_COMPONENTS := $(2))
# Build
$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
# Generate SBOM
mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
$(foreach pkg_component, $(PKG_COMPONENTS),
cyclonedx-gomod app -assert-licenses -json -licenses -packages \
-main app/$(pkg_component) \
-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
# Pack and compress
$(foreach pkg_component, $(PKG_COMPONENTS),
cd bin && tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar \
$(pkg_component)-$(GOOS)-$(GOARCH)-prod)
cd bin && gzip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar
cd bin && tar -czf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
# Generate checksums
cd bin && \
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.tar.gz >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
$(foreach pkg_component, $(PKG_COMPONENTS),
cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod | sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ >> \
$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
# Clean up
cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
endef
define RELEASE_WINDOWS_GOARCH
$(eval PKG_NAME := $(1))
$(eval PKG_COMPONENTS := $(2))
# Build
$(foreach pkg_component, $(PKG_COMPONENTS), $(MAKE) $(pkg_component)-$(GOOS)-$(GOARCH)-prod)
# Generate SBOM
mkdir -p "bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom"
$(foreach pkg_component, $(PKG_COMPONENTS),
cyclonedx-gomod app -assert-licenses -json -licenses -packages \
-main app/$(pkg_component) \
-output bin/$(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom/$(pkg_component)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.json)
# Pack and compress
$(foreach pkg_component, $(PKG_COMPONENTS),
cd bin && zip -u $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip \
$(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
cd bin && zip $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip -r $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
# Generate checksums
cd bin && \
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG).zip > $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt && \
sha256sum $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom.zip >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
$(foreach pkg_component, $(PKG_COMPONENTS),
cd bin && sha256sum $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe >> $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt)
# Clean up
cd bin && rm -rf $(PKG_NAME)-$(GOOS)-$(GOARCH)-$(PKG_TAG)_bom
$(foreach pkg_component, $(PKG_COMPONENTS), cd bin && rm -f $(pkg_component)-$(GOOS)-$(GOARCH)-prod.exe)
endef
all: \
victoria-metrics-prod \
victoria-logs-prod \
@@ -242,26 +312,13 @@ release-victoria-metrics-openbsd-amd64:
GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
release-victoria-metrics-windows-amd64:
GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
GOOS=windows GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
cd bin && \
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
victoria-metrics-$(GOOS)-$(GOARCH)-prod \
&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
victoria-metrics-$(GOOS)-$(GOARCH)-prod \
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
release-victoria-metrics-goos-goarch:
$(call RELEASE_GOOS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
cd bin && \
zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
victoria-metrics-windows-$(GOARCH)-prod.exe \
&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
victoria-metrics-windows-$(GOARCH)-prod.exe \
> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf \
victoria-metrics-windows-$(GOARCH)-prod.exe
release-victoria-metrics-windows-goarch:
$(call RELEASE_WINDOWS_GOARCH,victoria-metrics,$(VICTORIA_METRICS_COMPONENTS))
release-victoria-logs:
$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
@@ -303,7 +360,7 @@ release-victoria-logs-windows-amd64:
release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
cd bin && \
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
victoria-logs-$(GOOS)-$(GOARCH)-prod \
&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
victoria-logs-$(GOOS)-$(GOARCH)-prod \
@@ -356,77 +413,13 @@ release-vmutils-openbsd-amd64:
GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
release-vmutils-windows-amd64:
GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
GOOS=windows GOARCH=amd64 $(MAKE) release-vmutils-windows-goarch
release-vmutils-goos-goarch: \
vmagent-$(GOOS)-$(GOARCH)-prod \
vmalert-$(GOOS)-$(GOARCH)-prod \
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
vmauth-$(GOOS)-$(GOARCH)-prod \
vmbackup-$(GOOS)-$(GOARCH)-prod \
vmrestore-$(GOOS)-$(GOARCH)-prod \
vmctl-$(GOOS)-$(GOARCH)-prod
cd bin && \
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
vmagent-$(GOOS)-$(GOARCH)-prod \
vmalert-$(GOOS)-$(GOARCH)-prod \
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
vmauth-$(GOOS)-$(GOARCH)-prod \
vmbackup-$(GOOS)-$(GOARCH)-prod \
vmrestore-$(GOOS)-$(GOARCH)-prod \
vmctl-$(GOOS)-$(GOARCH)-prod \
&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
vmagent-$(GOOS)-$(GOARCH)-prod \
vmalert-$(GOOS)-$(GOARCH)-prod \
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
vmauth-$(GOOS)-$(GOARCH)-prod \
vmbackup-$(GOOS)-$(GOARCH)-prod \
vmrestore-$(GOOS)-$(GOARCH)-prod \
vmctl-$(GOOS)-$(GOARCH)-prod \
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf \
vmagent-$(GOOS)-$(GOARCH)-prod \
vmalert-$(GOOS)-$(GOARCH)-prod \
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
vmauth-$(GOOS)-$(GOARCH)-prod \
vmbackup-$(GOOS)-$(GOARCH)-prod \
vmrestore-$(GOOS)-$(GOARCH)-prod \
vmctl-$(GOOS)-$(GOARCH)-prod
release-vmutils-goos-goarch:
$(call RELEASE_GOOS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
release-vmutils-windows-goarch: \
vmagent-windows-$(GOARCH)-prod \
vmalert-windows-$(GOARCH)-prod \
vmalert-tool-windows-$(GOARCH)-prod \
vmauth-windows-$(GOARCH)-prod \
vmbackup-windows-$(GOARCH)-prod \
vmrestore-windows-$(GOARCH)-prod \
vmctl-windows-$(GOARCH)-prod
cd bin && \
zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
vmagent-windows-$(GOARCH)-prod.exe \
vmalert-windows-$(GOARCH)-prod.exe \
vmalert-tool-windows-$(GOARCH)-prod.exe \
vmauth-windows-$(GOARCH)-prod.exe \
vmbackup-windows-$(GOARCH)-prod.exe \
vmrestore-windows-$(GOARCH)-prod.exe \
vmctl-windows-$(GOARCH)-prod.exe \
&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
vmagent-windows-$(GOARCH)-prod.exe \
vmalert-windows-$(GOARCH)-prod.exe \
vmalert-tool-windows-$(GOARCH)-prod.exe \
vmauth-windows-$(GOARCH)-prod.exe \
vmbackup-windows-$(GOARCH)-prod.exe \
vmrestore-windows-$(GOARCH)-prod.exe \
vmctl-windows-$(GOARCH)-prod.exe \
> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
cd bin && rm -rf \
vmagent-windows-$(GOARCH)-prod.exe \
vmalert-windows-$(GOARCH)-prod.exe \
vmalert-tool-windows-$(GOARCH)-prod.exe \
vmauth-windows-$(GOARCH)-prod.exe \
vmbackup-windows-$(GOARCH)-prod.exe \
vmrestore-windows-$(GOARCH)-prod.exe \
vmctl-windows-$(GOARCH)-prod.exe
release-vmutils-windows-goarch:
$(call RELEASE_WINDOWS_GOARCH, vmutils, $(VICTORIA_METRICS_UTILS_COMPONENTS))
pprof-cpu:
go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
@@ -495,7 +488,7 @@ golangci-lint: install-golangci-lint
golangci-lint run
install-golangci-lint:
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.1
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
remove-golangci-lint:
rm -rf `which golangci-lint`
@@ -514,3 +507,37 @@ install-wwhrd:
check-licenses: install-wwhrd
wwhrd check -f .wwhrd.yml
cyclonedx-gomod-install:
which cyclonedx-gomod || go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest
copy-docs:
# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
echo "---" > ${DST}
@if [ ${ORDER} -ne 0 ]; then \
echo "sort: ${ORDER}" >> ${DST}; \
echo "weight: ${ORDER}" >> ${DST}; \
printf "menu:\n docs:\n parent: 'victoriametrics'\n weight: ${ORDER}\n" >> ${DST}; \
fi
echo "title: ${TITLE}" >> ${DST}
@if [ ${OLD_URL} ]; then \
printf "aliases:\n - ${OLD_URL}\n" >> ${DST}; \
fi
echo "---" >> ${DST}
cat ${SRC} >> ${DST}
sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"/' ${DST}
rm -rf docs/*.tmp
# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
# For ORDER=0 it adds no order tag/weight tag.
# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
# Images starting with <img src="docs/ are replaced with <img src="
# Cluster docs are supposed to be ordered as 2nd.
# The rest of docs is ordered manually.
docs-sync:
SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs

3229
README.md

File diff suppressed because it is too large Load Diff

View File

@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
| Version | Supported |
|---------|--------------------|
| [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
| other releases | :x: |
See [this page](https://victoriametrics.com/security/) for more details.
## Reporting a Vulnerability
Please report any security issues to <security@victoriametrics.com>
Please report any security issues to security@victoriametrics.com

View File

@@ -57,12 +57,6 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
fmt.Fprintf(w, `{}`)
return true
}
if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
// Return fake response for Logstash APIs requests.
// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
fmt.Fprintf(w, `{}`)
return true
}
switch path {
case "/":
switch r.Method {

View File

@@ -230,7 +230,7 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
for i := 0; i < activeStreams; i++ {
ip := toIPv4(rand.Uint32())
uuid := toUUID(rand.Uint64(), rand.Uint64())
fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
for j := 0; j < *constFieldsPerLog; j++ {

View File

@@ -1,13 +1,13 @@
{
"files": {
"main.css": "./static/css/main.c9cc37dd.css",
"main.js": "./static/js/main.7ae2e2c4.js",
"main.css": "./static/css/main.1041c3d4.css",
"main.js": "./static/js/main.62a82f4a.js",
"static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
"static/media/MetricsQL.md": "./static/media/MetricsQL.8c2e588d62b87f90dbf0.md",
"static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
"index.html": "./index.html"
},
"entrypoints": [
"static/css/main.c9cc37dd.css",
"static/js/main.7ae2e2c4.js"
"static/css/main.1041c3d4.css",
"static/js/main.62a82f4a.js"
]
}

View File

@@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.7ae2e2c4.js"></script><link href="./static/css/main.c9cc37dd.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.62a82f4a.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -1,4 +1,5 @@
---
sort: 23
weight: 23
title: MetricsQL
menu:
@@ -9,6 +10,9 @@ aliases:
- /ExtendedPromQL.html
- /MetricsQL.html
---
# MetricsQL
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -216,11 +220,9 @@ See also [descent_over_time](#descent_over_time).
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
See also [median_over_time](#median_over_time).
#### changes
@@ -260,8 +262,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
#### count_gt_over_time
@@ -272,8 +272,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
#### count_le_over_time
@@ -284,8 +282,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
#### count_ne_over_time
@@ -296,9 +292,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
See also [count_over_time](#count_over_time).
#### count_over_time
@@ -319,8 +313,6 @@ The results are calculated independently per each time series returned from the
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
#### decreases_over_time
@@ -440,8 +432,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### histogram_over_time
`histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -453,15 +443,11 @@ For example, the following query calculates median temperature by country over t
`histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### hoeffding_bound_lower
`hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [hoeffding_bound_upper](#hoeffding_bound_upper).
#### hoeffding_bound_upper
@@ -469,8 +455,6 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
`hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [hoeffding_bound_lower](#hoeffding_bound_lower).
#### holt_winters
@@ -478,9 +462,8 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
Both `sf` and `tf` must be in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
@@ -512,14 +495,13 @@ See also [deriv](#deriv).
`increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
This function is supported by PromQL.
See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -528,13 +510,12 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
`increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [increase_pure](#increase_pure) and [increase](#increase).
#### increase_pure
@@ -543,10 +524,6 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
while [increase](#increase) ignores the first value in a series if it is too big.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [increase](#increase) and [increase_prometheus](#increase_prometheus).
#### increases_over_time
`increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -563,18 +540,15 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### irate
`irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
This function is supported by PromQL.
See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -613,8 +587,6 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
#### max_over_time
@@ -622,11 +594,9 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
`max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
See also [tmax_over_time](#tmax_over_time).
#### median_over_time
@@ -634,8 +604,6 @@ See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
on the given lookbehind window `d` per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [avg_over_time](#avg_over_time).
#### min_over_time
@@ -643,11 +611,9 @@ See also [avg_over_time](#avg_over_time).
`min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
See also [tmin_over_time](#tmin_over_time).
#### mode_over_time
@@ -656,8 +622,6 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values are discrete.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### outlier_iqr_over_time
`outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -668,8 +632,6 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [outliers_iqr](#outliers_iqr).
#### predict_linear
@@ -697,8 +659,6 @@ This function is supported by PromQL.
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
The `phi` value must be in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [quantiles_over_time](#quantiles_over_time).
@@ -710,8 +670,6 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [quantile_over_time](#quantile_over_time).
#### range_over_time
@@ -722,12 +680,11 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### rate
`rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -746,8 +703,6 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
on the given lookbehind window `d`. The calculations are performed individually per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
#### resets
@@ -755,11 +710,10 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
`resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
This function is supported by PromQL.
#### rollup
@@ -771,10 +725,6 @@ These values are calculated individually per each time series returned from the
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
See also [label_match](#label_match).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [rollup_rate](#rollup_rate).
#### rollup_candlestick
`rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -786,8 +736,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
See also [label_match](#label_match).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### rollup_delta
`rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -814,8 +762,6 @@ See also [label_match](#label_match).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
#### rollup_increase
`rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -828,17 +774,12 @@ See also [label_match](#label_match).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
#### rollup_rate
`rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
when to use `rollup_rate()`.
@@ -846,12 +787,10 @@ when to use `rollup_rate()`.
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
See also [label_match](#label_match).
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
#### rollup_scrape_interval
`rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -885,8 +824,6 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
#### share_le_over_time
@@ -901,8 +838,6 @@ the share of time series values for the last 24 hours when memory usage was belo
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
#### share_eq_over_time
@@ -914,8 +849,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_eq_over_time](#count_eq_over_time).
#### stale_samples_over_time
@@ -933,8 +866,6 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [stdvar_over_time](#stdvar_over_time).
@@ -946,43 +877,35 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [stddev_over_time](#stddev_over_time).
#### sum_eq_over_time
`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values equal to `eq` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
#### sum_gt_over_time
`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values bigger than `gt` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
#### sum_le_over_time
`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values smaller or equal to `le` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
#### sum_over_time
@@ -992,8 +915,6 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
#### sum2_over_time
@@ -1003,8 +924,6 @@ values on the given lookbehind window `d` per each time series returned from the
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### timestamp
`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1082,8 +1001,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
@@ -2313,8 +2230,7 @@ Any [rollup function](#rollup-functions) for something other than [series select
Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
VictoriaMetrics performs subqueries in the following way:
@@ -2347,6 +2263,5 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
* If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.

View File

@@ -1,55 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -1,56 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -1,56 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -10,8 +10,6 @@ import (
"strings"
"time"
"github.com/VictoriaMetrics/metrics"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
@@ -44,7 +42,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
"github.com/VictoriaMetrics/metrics"
)
var (
@@ -318,10 +316,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
influxQueryRequests.Inc()
influxutils.WriteDatabaseNames(w)
return true
case "/influx/health":
influxHealthRequests.Inc()
influxutils.WriteHealthCheckResponse(w)
return true
case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
opentelemetryPushRequests.Inc()
if err := opentelemetry.InsertHandler(nil, r); err != nil {
@@ -456,7 +450,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
w.Header().Set("Content-Type", "application/json")
var bb bytesutil.ByteBuffer
promscrape.WriteConfigData(&bb)
fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
return true
case "/prometheus/-/reload", "/-/reload":
if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
@@ -568,10 +562,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
influxQueryRequests.Inc()
influxutils.WriteDatabaseNames(w)
return true
case "influx/health":
influxHealthRequests.Inc()
influxutils.WriteHealthCheckResponse(w)
return true
case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
opentelemetryPushRequests.Inc()
if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -682,8 +672,7 @@ var (
influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
influxWriteErrors = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
datadogv1WriteErrors = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

View File

@@ -34,10 +34,8 @@ var (
rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxInterval")
retryMaxTime = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
tlsHandshakeTimeout = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
@@ -92,9 +90,6 @@ type client struct {
fq *persistentqueue.FastQueue
hc *http.Client
retryMinInterval time.Duration
retryMaxTime time.Duration
sendBlock func(block []byte) bool
authCfg *promauth.Config
awsCfg *awsapi.Config
@@ -148,15 +143,13 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
Timeout: sendTimeout.GetOptionalArg(argIdx),
}
c := &client{
sanitizedURL: sanitizedURL,
remoteWriteURL: remoteWriteURL,
authCfg: authCfg,
awsCfg: awsCfg,
fq: fq,
hc: hc,
retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
retryMaxTime: retryMaxTime.GetOptionalArg(argIdx),
stopCh: make(chan struct{}),
sanitizedURL: sanitizedURL,
remoteWriteURL: remoteWriteURL,
authCfg: authCfg,
awsCfg: awsCfg,
fq: fq,
hc: hc,
stopCh: make(chan struct{}),
}
c.sendBlock = c.sendBlockHTTP
@@ -403,11 +396,11 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
// sendBlockHTTP sends the given block to c.remoteWriteURL.
//
// The function returns false only if c.stopCh is closed.
// Otherwise, it tries sending the block to remote storage indefinitely.
// Otherwise it tries sending the block to remote storage indefinitely.
func (c *client) sendBlockHTTP(block []byte) bool {
c.rl.Register(len(block))
maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
retryDuration := timeutil.AddJitterToDuration(time.Second)
retriesCount := 0
again:

View File

@@ -441,7 +441,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
var rctx *relabelCtx
rcs := allRelabelConfigs.Load()
pcsGlobal := rcs.global
if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
if pcsGlobal.Len() > 0 {
rctx = getRelabelCtx()
defer putRelabelCtx(rctx)
}

View File

@@ -50,7 +50,7 @@ var (
streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
@@ -133,7 +133,7 @@ func initStreamAggrConfigGlobal() {
} else {
dedupInterval := streamAggrGlobalDedupInterval.Duration()
if dedupInterval > 0 {
deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrDropInputLabels, "dedup-global")
}
}
}
@@ -202,7 +202,6 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
DropInputLabels: *streamAggrGlobalDropInputLabels,
IgnoreOldSamples: *streamAggrGlobalIgnoreOldSamples,
IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
KeepInput: *streamAggrGlobalKeepInput,
}
sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
@@ -230,8 +229,7 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag
DedupInterval: streamAggrDedupInterval.GetOptionalArg(idx),
DropInputLabels: *streamAggrDropInputLabels,
IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(idx),
IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
KeepInput: streamAggrKeepInput.GetOptionalArg(idx),
IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
}
sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)

View File

@@ -41,19 +41,9 @@ Examples:
Usage: "disable adding group's Name as label to generated alerts and time series.",
Required: false,
},
&cli.StringSliceFlag{
Name: "external.label",
Usage: `Optional label in the form 'name=value' to add to all generated recording rules and alerts. Supports an array of values separated by comma or specified via multiple flags.`,
Required: false,
},
&cli.StringFlag{
Name: "external.url",
Usage: `Optional external URL to template in rule's labels or annotations.`,
Required: false,
},
},
Action: func(c *cli.Context) error {
if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url")); failed {
if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
return fmt.Errorf("unittest failed")
}
return nil

View File

@@ -13,7 +13,7 @@ tests:
- expr: suquery_interval_test
eval_time: 4m
exp_samples:
- labels: '{__name__="suquery_interval_test", instance="localhost:9090", job="vmagent2"}'
- labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
value: 1
alert_rule_test:
@@ -24,16 +24,20 @@ tests:
job: vmagent2
severity: page
instance: localhost:9090
datacenter: dc-123
exp_annotations:
summary: "Instance localhost:9090 down"
description: "localhost:9090 of job vmagent2 in cluster has been down for more than 5 minutes."
dashboard: "/d/dashboard?orgId=1"
description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
- eval_time: 0
alertname: AlwaysFiring
exp_alerts:
- {}
- exp_labels:
datacenter: dc-123
- eval_time: 0
alertname: InstanceDown
exp_alerts: []
external_labels:
datacenter: dc-123

View File

@@ -8,8 +8,7 @@ groups:
severity: page
annotations:
summary: "Instance {{ $labels.instance }} down"
description: "{{ $labels.instance }} of job {{ $labels.job }} in cluster {{ $externalLabels.cluster }} has been down for more than 5 minutes."
dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
- alert: AlwaysFiring
expr: 1
- alert: SameAlertNameWithDifferentGroup

View File

@@ -16,8 +16,7 @@ tests:
groupname: group1
alertname: SameAlertNameWithDifferentGroup
exp_alerts:
- exp_labels:
cluster: prod
- {}
- eval_time: 1m
groupname: group2
alertname: SameAlertNameWithDifferentGroup
@@ -26,8 +25,7 @@ tests:
groupname: group1
alertname: SameAlertNameWithDifferentGroup
exp_alerts:
- exp_labels:
cluster: prod
- {}
- eval_time: 6m
groupname: group1
alertname: SameAlertNameWithDifferentGroup
@@ -63,18 +61,18 @@ tests:
eval_time: 4m
exp_samples:
- value: 4
labels: '{__name__="t1", cluster="prod"}'
labels: '{__name__="t1", datacenter="dc-123"}'
- expr: t2
eval_time: 4m
exp_samples:
- value: 4
labels: '{__name__="t2", cluster="prod"}'
labels: '{__name__="t2", datacenter="dc-123"}'
- expr: t3
eval_time: 4m
exp_samples:
# t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
- value: 3
labels: '{__name__="t3", cluster="prod"}'
labels: '{__name__="t3", datacenter="dc-123"}'
alert_rule_test:
- eval_time: 10m
@@ -85,21 +83,22 @@ tests:
job: vmagent1
severity: page
instance: localhost:9090
cluster: prod
datacenter: dc-123
exp_annotations:
summary: "Instance localhost:9090 down"
description: "localhost:9090 of job vmagent1 in cluster prod has been down for more than 5 minutes."
dashboard: "http://grafana:3000/d/dashboard?orgId=1"
description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
- eval_time: 0
groupname: group1
alertname: AlwaysFiring
exp_alerts:
- exp_labels:
cluster: prod
datacenter: dc-123
- eval_time: 0
groupname: alerts
alertname: InstanceDown
exp_alerts: []
external_labels:
datacenter: dc-123

View File

@@ -13,7 +13,7 @@ tests:
- expr: suquery_interval_test
eval_time: 4m
exp_samples:
- labels: '{__name__="suquery_interval_test", cluster="prod", instance="localhost:9090", job="vmagent2"}'
- labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
value: 1
alert_rule_test:
@@ -25,21 +25,22 @@ tests:
job: vmagent2
severity: page
instance: localhost:9090
cluster: prod
datacenter: dc-123
exp_annotations:
summary: "Instance localhost:9090 down"
description: "localhost:9090 of job vmagent2 in cluster prod has been down for more than 5 minutes."
dashboard: "http://grafana:3000/d/dashboard?orgId=1"
description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
- eval_time: 0
groupname: group1
alertname: AlwaysFiring
exp_alerts:
- exp_labels:
cluster: prod
datacenter: dc-123
- eval_time: 0
groupname: group1
alertname: InstanceDown
exp_alerts: []
external_labels:
datacenter: dc-123

View File

@@ -55,7 +55,7 @@ const (
)
// UnitTest runs unittest for files
func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL string) bool {
func UnitTest(files []string, disableGroupLabel bool) bool {
if err := templates.Load([]string{}, true); err != nil {
logger.Fatalf("failed to load template: %v", err)
}
@@ -71,34 +71,14 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
testfiles, err := config.ReadFromFS(files)
if err != nil {
logger.Fatalf("failed to load test files %q: %v", files, err)
fmt.Println(" FAILED")
fmt.Printf("\nfailed to read test files: \n%v", err)
}
if len(testfiles) == 0 {
fmt.Println("no test file found")
return false
}
labels := make(map[string]string)
for _, s := range externalLabels {
if len(s) == 0 {
continue
}
n := strings.IndexByte(s, '=')
if n < 0 {
logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
}
labels[s[:n]] = s[n+1:]
}
_, err = notifier.Init(nil, labels, externalURL)
if err != nil {
logger.Fatalf("failed to init notifier: %v", err)
}
var failed bool
for fileName, file := range testfiles {
if err := ruleUnitTest(fileName, file, labels); err != nil {
if err := ruleUnitTest(fileName, file); err != nil {
fmt.Println(" FAILED")
fmt.Printf("\nfailed to run unit test for file %q: \n%v", fileName, err)
fmt.Printf("\nfailed to run unit test for file %q: \n%v", file, err)
failed = true
} else {
fmt.Println(" SUCCESS")
@@ -108,7 +88,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
return failed
}
func ruleUnitTest(filename string, content []byte, externalLabels map[string]string) []error {
func ruleUnitTest(filename string, content []byte) []error {
fmt.Println("\nUnit Testing: ", filename)
var unitTestInp unitTestFile
if err := yaml.UnmarshalStrict(content, &unitTestInp); err != nil {
@@ -146,7 +126,7 @@ func ruleUnitTest(filename string, content []byte, externalLabels map[string]str
errs = append(errs, err)
continue
}
testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups, externalLabels)
testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups)
errs = append(errs, testErrs...)
}
@@ -183,10 +163,6 @@ func verifyTestGroup(group testGroup) error {
return fmt.Errorf("\n%s missing required field \"eval_time\"", testGroupName)
}
}
if group.ExternalLabels != nil {
fmt.Printf("\n%s warning: filed `external_labels` will be deprecated soon, please use `-external.label` cmd-line flag instead. "+
"Check https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6735 for details.\n", testGroupName)
}
return nil
}
@@ -203,7 +179,6 @@ func processFlags() {
{flag: "retentionPeriod", value: "100y"},
{flag: "datasource.url", value: testDataSourcePath},
{flag: "remoteWrite.url", value: testRemoteWritePath},
{flag: "notifier.blackhole", value: "true"},
} {
// panics if flag doesn't exist
if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
@@ -264,7 +239,7 @@ func tearDown() {
fs.MustRemoveAll(storagePath)
}
func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group, externalLabels map[string]string) (checkErrs []error) {
func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
// set up vmstorage and http server for ingest and read queries
setUp()
// tear down vmstorage and clean the data dir
@@ -313,14 +288,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
// create groups with given rule
var groups []*rule.Group
for _, group := range testGroups {
mergedExternalLabels := make(map[string]string)
for k, v := range tg.ExternalLabels {
mergedExternalLabels[k] = v
}
for k, v := range externalLabels {
mergedExternalLabels[k] = v
}
ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels)
groups = append(groups, ng)
}

View File

@@ -18,7 +18,7 @@ func TestUnitTest_Failure(t *testing.T) {
f := func(files []string) {
t.Helper()
failed := UnitTest(files, false, nil, "")
failed := UnitTest(files, false)
if !failed {
t.Fatalf("expecting failed test")
}
@@ -29,19 +29,18 @@ func TestUnitTest_Failure(t *testing.T) {
}
func TestUnitTest_Success(t *testing.T) {
f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL string) {
f := func(disableGroupLabel bool, files []string) {
t.Helper()
failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL)
failed := UnitTest(files, disableGroupLabel)
if failed {
t.Fatalf("unexpected failed test")
}
}
// run multi files
f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000")
f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"})
// disable group label
// template with null external values
f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "")
f(true, []string{"./testdata/disable-group-label.yaml"})
}

View File

@@ -17,7 +17,7 @@ import (
var (
addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
"See also -remoteRead.disablePathAppend and -datasource.showURL")
appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
@@ -99,7 +99,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
if err != nil {
return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
return nil, fmt.Errorf("failed to create transport: %w", err)
}
tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
tr.DisableKeepAlives = *disableKeepAlive

File diff suppressed because one or more lines are too long

View File

@@ -57,23 +57,20 @@ var jsonParserPool fastjson.ParserPool
// [{"metric":{"__name__":"up","job":"prometheus"},value": [ 1435781451.781,"1"]},
// {"metric":{"__name__":"up","job":"node"},value": [ 1435781451.781,"0"]}]
func (pi *promInstant) Unmarshal(b []byte) error {
var metrics []json.RawMessage
// metrics slice could be large, so parsing it with fastjson could consume a lot of memory.
// We parse the slice with standard lib to keep mem usage low.
// And each metric object will be parsed with fastjson to reduce allocations.
if err := json.Unmarshal(b, &metrics); err != nil {
return fmt.Errorf("cannot unmarshal metrics: %w", err)
}
p := jsonParserPool.Get()
defer jsonParserPool.Put(p)
pi.ms = make([]Metric, len(metrics))
for i, data := range metrics {
row, err := p.ParseBytes(data)
if err != nil {
return fmt.Errorf("cannot parse metric object: %w", err)
}
v, err := p.ParseBytes(b)
if err != nil {
return err
}
rows, err := v.Array()
if err != nil {
return fmt.Errorf("cannot find the top-level array of result objects: %w", err)
}
pi.ms = make([]Metric, len(rows))
for i, row := range rows {
metric := row.Get("metric")
if metric == nil {
return fmt.Errorf("can't find `metric` object in %q", row)

View File

@@ -1,24 +1,43 @@
package datasource
import (
"bytes"
"io"
"net/http"
"os"
"testing"
)
func BenchmarkPromInstantUnmarshal(b *testing.B) {
func BenchmarkMetrics(b *testing.B) {
payload := []byte(`[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests", "foo":"bar", "baz": "qux"},"value":[1583786140,"2000"]}]`)
var pi promInstant
if err := pi.Unmarshal(payload); err != nil {
b.Fatalf(err.Error())
}
b.Run("Instant", func(b *testing.B) {
for i := 0; i < b.N; i++ {
_, _ = pi.metrics()
}
})
}
func BenchmarkParsePrometheusResponse(b *testing.B) {
req, _ := http.NewRequest("GET", "", nil)
resp := &http.Response{StatusCode: http.StatusOK}
data, err := os.ReadFile("testdata/instant_response.json")
if err != nil {
b.Fatalf("error while reading file: %s", err)
}
resp.Body = io.NopCloser(bytes.NewReader(data))
// BenchmarkParsePrometheusResponse/Instant_std+fastjson-10 1760 668959 ns/op 280147 B/op 5781 allocs/op
b.Run("Instant std+fastjson", func(b *testing.B) {
b.Run("Instant", func(b *testing.B) {
for i := 0; i < b.N; i++ {
var pi promInstant
err = pi.Unmarshal(data)
_, err := parsePrometheusResponse(req, resp)
if err != nil {
b.Fatalf("unexpected parse err: %s", err)
}
resp.Body = io.NopCloser(bytes.NewReader(data))
}
})
}

View File

@@ -76,6 +76,9 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
return err
}
req.Header.Set("Content-Type", "application/json")
for key, value := range headers {
req.Header.Set(key, value)
}
if am.timeout > 0 {
var cancel context.CancelFunc
@@ -91,11 +94,6 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
return err
}
}
// external headers have higher priority
for key, value := range headers {
req.Header.Set(key, value)
}
resp, err := am.client.Do(req)
if err != nil {
return err
@@ -132,8 +130,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
}
tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
if err != nil {
return nil, fmt.Errorf("failed to create transport for alertmanager URL=%q: %w", alertManagerURL, err)
return nil, fmt.Errorf("failed to create transport: %w", err)
}
ba := new(promauth.BasicAuthConfig)
@@ -148,9 +145,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
aCfg, err := utils.AuthConfig(
utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
utils.WithHeaders(strings.Join(authCfg.Headers, "^^")),
)
utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
if err != nil {
return nil, fmt.Errorf("failed to configure auth: %w", err)
}

View File

@@ -3,7 +3,6 @@ package notifier
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
@@ -49,9 +48,6 @@ func TestAlertManager_Send(t *testing.T) {
conn, _, _ := w.(http.Hijacker).Hijack()
_ = conn.Close()
case 1:
if r.Header.Get(headerKey) != headerValue {
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
}
w.WriteHeader(500)
case 2:
var a []struct {
@@ -76,9 +72,6 @@ func TestAlertManager_Send(t *testing.T) {
if a[0].EndAt.IsZero() {
t.Fatalf("expected non-zero end time")
}
if r.Header.Get(headerKey) != "bar" {
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
}
case 3:
if r.Header.Get(headerKey) != headerValue {
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
@@ -93,7 +86,6 @@ func TestAlertManager_Send(t *testing.T) {
Username: baUser,
Password: promauth.NewSecret(baPass),
},
Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
}
am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
@@ -113,7 +105,7 @@ func TestAlertManager_Send(t *testing.T) {
Start: time.Now().UTC(),
End: time.Now().UTC(),
Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
}}, map[string]string{headerKey: "bar"}); err != nil {
}}, nil); err != nil {
t.Fatalf("unexpected error %s", err)
}
if c != 2 {

View File

@@ -25,9 +25,6 @@ var (
"Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). "+
"-notifier.url, -notifier.config and -notifier.blackhole are mutually exclusive.")
headers = flagutil.NewArrayString("notifier.headers", "Optional HTTP headers to send with each request to the corresponding -notifier.url. "+
"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
basicAuthUsername = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
basicAuthPassword = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
@@ -174,7 +171,6 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
Scopes: strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
TokenURL: oauth2TokenURL.GetOptionalArg(i),
},
Headers: []string{headers.GetOptionalArg(i)},
}
addr = strings.TrimSuffix(addr, "/")

View File

@@ -17,7 +17,7 @@ var (
addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
"Remote read is used to restore alerts state."+
"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
@@ -68,7 +68,7 @@ func Init() (datasource.QuerierBuilder, error) {
}
tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
if err != nil {
return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
return nil, fmt.Errorf("failed to create transport: %w", err)
}
tr.IdleConnTimeout = *idleConnectionTimeout
tr.DialContext = netutil.NewStatDialFunc("vmalert_remoteread")

View File

@@ -32,7 +32,7 @@ func NewDebugClient() (*DebugClient, error) {
t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
if err != nil {
return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
return nil, fmt.Errorf("failed to create transport: %w", err)
}
c := &DebugClient{
c: &http.Client{

View File

@@ -15,7 +15,7 @@ import (
var (
addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
"and recording rules results in form of timeseries. "+
"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -72,7 +72,7 @@ func Init(ctx context.Context) (*Client, error) {
t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
if err != nil {
return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
return nil, fmt.Errorf("failed to create transport: %w", err)
}
t.IdleConnTimeout = *idleConnectionTimeout
t.DialContext = netutil.NewStatDialFunc("vmalert_remotewrite")

View File

@@ -441,6 +441,9 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
}
a.Value = m.Values[0]
a.Annotations = annotations
if err != nil {
return nil, err
}
a.KeepFiringSince = time.Time{}
continue
}

View File

@@ -83,6 +83,7 @@ type UserInfo struct {
concurrencyLimitCh chan struct{}
concurrencyLimitReached *metrics.Counter
overrideHostHeader bool
rt http.RoundTripper
@@ -93,9 +94,8 @@ type UserInfo struct {
// HeadersConf represents config for request and response headers.
type HeadersConf struct {
RequestHeaders []*Header `yaml:"headers,omitempty"`
ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
KeepOriginalHost *bool `yaml:"keep_original_host,omitempty"`
RequestHeaders []*Header `yaml:"headers,omitempty"`
ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
}
func (ui *UserInfo) beginConcurrencyLimit() error {
@@ -150,6 +150,15 @@ func (h *Header) MarshalYAML() (any, error) {
return h.sOriginal, nil
}
func overrideHostHeader(headers []*Header) bool {
for _, h := range headers {
if h.Name == "Host" && h.Value == "" {
return true
}
}
return false
}
// URLMap is a mapping from source paths to target urls.
type URLMap struct {
// SrcPaths is an optional list of regular expressions, which must match the request path.
@@ -301,18 +310,12 @@ func (up *URLPrefix) getBackendsCount() int {
// getBackendURL returns the backendURL depending on the load balance policy.
//
// It can return nil if there are no backend urls available at the moment.
//
// backendURL.put() must be called on the returned backendURL after the request is complete.
func (up *URLPrefix) getBackendURL() *backendURL {
up.discoverBackendAddrsIfNeeded()
pbus := up.bus.Load()
bus := *pbus
if len(bus) == 0 {
return nil
}
if up.loadBalancingPolicy == "first_available" {
return getFirstAvailableBackendURL(bus)
}
@@ -394,7 +397,7 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
buCopy.Host = addr
if port != "" {
if n := strings.IndexByte(buCopy.Host, ':'); n >= 0 {
// Drop the discovered port and substitute it the port specified in bu.
// Drop the discovered port and substitute it the the port specified in bu.
buCopy.Host = buCopy.Host[:n]
}
buCopy.Host += ":" + port
@@ -461,7 +464,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
}
// Slow path - select other backend urls.
n := atomicCounter.Add(1) - 1
n := atomicCounter.Add(1)
for i := uint32(0); i < uint32(len(bus)); i++ {
idx := (n + i) % uint32(len(bus))
@@ -597,7 +600,7 @@ func initAuthConfig() {
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
sighupCh := procutil.NewSighupChan()
_, err := reloadAuthConfig()
_, err := loadAuthConfig()
if err != nil {
logger.Fatalf("cannot load auth config: %s", err)
}
@@ -629,7 +632,7 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
updateFn := func() {
configReloads.Inc()
updated, err := reloadAuthConfig()
updated, err := loadAuthConfig()
if err != nil {
logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
configSuccess.Set(0)
@@ -655,45 +658,27 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
}
}
// authConfigData stores the yaml definition for this config.
// authConfigData needs to be updated each time authConfig is updated.
var authConfigData atomic.Pointer[[]byte]
var (
// authConfigData stores the yaml definition for this config.
// authConfigData needs to be updated each time authConfig is updated.
authConfigData atomic.Pointer[[]byte]
// authConfig contains the currently loaded auth config
authConfig atomic.Pointer[AuthConfig]
// authUsers contains the currently loaded auth users
authUsers atomic.Pointer[map[string]*UserInfo]
authConfig atomic.Pointer[AuthConfig]
authUsers atomic.Pointer[map[string]*UserInfo]
authConfigWG sync.WaitGroup
stopCh chan struct{}
)
// reloadAuthConfig loads and applies the config from *authConfigPath.
// loadAuthConfig loads and applies the config from *authConfigPath.
// It returns bool value to identify if new config was applied.
// The config can be not applied if there is a parsing error
// or if there are no changes to the current authConfig.
func reloadAuthConfig() (bool, error) {
func loadAuthConfig() (bool, error) {
data, err := fscore.ReadFileOrHTTP(*authConfigPath)
if err != nil {
return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
}
ok, err := reloadAuthConfigData(data)
if err != nil {
return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
}
if !ok {
return false, nil
}
mp := authUsers.Load()
logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
return true, nil
}
func reloadAuthConfigData(data []byte) (bool, error) {
oldData := authConfigData.Load()
if oldData != nil && bytes.Equal(data, *oldData) {
// there are no updates in the config - skip reloading.
@@ -702,13 +687,14 @@ func reloadAuthConfigData(data []byte) (bool, error) {
ac, err := parseAuthConfig(data)
if err != nil {
return false, fmt.Errorf("failed to parse auth config: %w", err)
return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
}
m, err := parseAuthConfigUsers(ac)
if err != nil {
return false, fmt.Errorf("failed to parse users from auth config: %w", err)
return false, fmt.Errorf("failed to parse users from -auth.config=%q: %w", *authConfigPath, err)
}
logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
acPrev := authConfig.Load()
if acPrev != nil {
@@ -755,6 +741,7 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
if err := ui.initURLs(); err != nil {
return nil, err
}
ui.overrideHostHeader = overrideHostHeader(ui.HeadersConf.RequestHeaders)
metricLabels, err := ui.getMetricLabels()
if err != nil {
@@ -819,6 +806,7 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
return float64(len(ui.concurrencyLimitCh))
})
ui.overrideHostHeader = overrideHostHeader(ui.HeadersConf.RequestHeaders)
rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
if err != nil {
@@ -1009,14 +997,6 @@ func getAuthTokensFromRequest(r *http.Request) []string {
}
}
// Authorization via http://user:pass@hosname/path
if u := r.URL.User; u != nil && u.Username() != "" {
username := u.Username()
password, _ := u.Password()
at := getHTTPAuthBasicToken(username, password)
ats = append(ats, at)
}
return ats
}
@@ -1042,6 +1022,10 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
}
func sanitizeURLPrefix(urlPrefix *url.URL) (*url.URL, error) {
// Remove trailing '/' from urlPrefix
for strings.HasSuffix(urlPrefix.Path, "/") {
urlPrefix.Path = urlPrefix.Path[:len(urlPrefix.Path)-1]
}
// Validate urlPrefix
if urlPrefix.Scheme != "http" && urlPrefix.Scheme != "https" {
return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, urlPrefix.Scheme)

View File

@@ -82,14 +82,6 @@ users:
headers: foobar
`)
// Invalid keep_original_host value
f(`
users:
- username: foo
url_prefix: http://foo.bar
keep_original_host: foobar
`)
// empty url_prefix
f(`
users:
@@ -418,7 +410,7 @@ users:
HeadersConf: HeadersConf{
RequestHeaders: []*Header{
mustNewHeader("'foo: bar'"),
mustNewHeader("'xxx:'"),
mustNewHeader("'xxx: y'"),
},
},
},
@@ -437,7 +429,7 @@ users:
url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
headers:
- "foo: bar"
- "xxx:"
- "xxx: y"
`, map[string]*UserInfo{
getHTTPAuthBearerToken("foo"): sharedUserInfo,
getHTTPAuthBasicToken("foo", ""): sharedUserInfo,
@@ -466,7 +458,6 @@ users:
})
// with default url
keepOriginalHost := true
f(`
users:
- bearer_token: foo
@@ -478,7 +469,6 @@ users:
headers:
- "foo: bar"
- "xxx: y"
keep_original_host: true
default_url:
- http://default1/select/0/prometheus
- http://default2/select/0/prometheus
@@ -501,7 +491,6 @@ users:
mustNewHeader("'foo: bar'"),
mustNewHeader("'xxx: y'"),
},
KeepOriginalHost: &keepOriginalHost,
},
},
},
@@ -528,7 +517,6 @@ users:
mustNewHeader("'foo: bar'"),
mustNewHeader("'xxx: y'"),
},
KeepOriginalHost: &keepOriginalHost,
},
},
},
@@ -548,17 +536,12 @@ users:
metric_labels:
dc: eu
team: dev
keep_original_host: true
- username: foo-same
password: bar
url_prefix: https://bar/x
metric_labels:
backend_env: test
team: accounting
headers:
- "foo: bar"
response_headers:
- "Abc: def"
`, map[string]*UserInfo{
getHTTPAuthBasicToken("foo-same", "baz"): {
Username: "foo-same",
@@ -568,9 +551,6 @@ users:
"dc": "eu",
"team": "dev",
},
HeadersConf: HeadersConf{
KeepOriginalHost: &keepOriginalHost,
},
},
getHTTPAuthBasicToken("foo-same", "bar"): {
Username: "foo-same",
@@ -580,14 +560,6 @@ users:
"backend_env": "test",
"team": "accounting",
},
HeadersConf: HeadersConf{
RequestHeaders: []*Header{
mustNewHeader("'foo: bar'"),
},
ResponseHeaders: []*Header{
mustNewHeader("'Abc: def'"),
},
},
},
})
}
@@ -723,9 +695,9 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
fn := func(ns ...int) {
t.Helper()
pbus := up.bus.Load()
bus := *pbus
for i, b := range bus {
bus := up.bus.Load()
pbus := *bus
for i, b := range pbus {
got := int(b.concurrentRequests.Load())
exp := ns[i]
if got != exp {
@@ -735,39 +707,39 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
}
up.getBackendURL()
fn(1, 0, 0)
fn(0, 1, 0)
up.getBackendURL()
fn(1, 1, 0)
fn(0, 1, 1)
up.getBackendURL()
fn(1, 1, 1)
up.getBackendURL()
up.getBackendURL()
fn(2, 2, 1)
fn(1, 2, 2)
bus := up.bus.Load()
pbus := *bus
pbus[0].concurrentRequests.Add(2)
pbus[2].concurrentRequests.Add(5)
fn(4, 2, 6)
fn(3, 2, 7)
up.getBackendURL()
fn(4, 3, 6)
fn(3, 3, 7)
up.getBackendURL()
fn(4, 4, 6)
fn(3, 4, 7)
up.getBackendURL()
fn(4, 5, 6)
fn(4, 4, 7)
up.getBackendURL()
fn(5, 5, 6)
fn(5, 4, 7)
up.getBackendURL()
fn(6, 5, 6)
fn(5, 5, 7)
up.getBackendURL()
fn(6, 6, 6)
fn(6, 5, 7)
up.getBackendURL()
fn(6, 6, 7)

View File

@@ -50,7 +50,7 @@ var (
`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
"Bigger values may require more memory. Negative or zero values disable request body caching and retries.")
backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
@@ -117,7 +117,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
return true
}
handleMissingAuthorizationError(w)
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
return true
}
@@ -188,27 +189,26 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
// to a route that is not in the configuration for unauthorized user.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5236
if ui.BearerToken == "" && ui.Username == "" && len(*authUsers.Load()) > 0 {
handleMissingAuthorizationError(w)
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
return
}
missingRouteRequests.Inc()
httpserver.Errorf(w, r, "missing route for %s", u.String())
httpserver.Errorf(w, r, "missing route for %q", u.String())
return
}
up, hc = ui.DefaultURL, ui.HeadersConf
isDefault = true
}
rtb := getReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
defer putReadTrackingBody(rtb)
r.Body = rtb
// caching makes sense only for positive non zero size
if maxRequestBodySizeToRetry.IntN() > 0 {
rtb := getReadTrackingBody(r.Body, int(r.ContentLength))
defer putReadTrackingBody(rtb)
r.Body = rtb
}
maxAttempts := up.getBackendsCount()
for i := 0; i < maxAttempts; i++ {
bu := up.getBackendURL()
if bu == nil {
break
}
targetURL := bu.url
// Don't change path and add request_path query param for default route.
if isDefault {
@@ -218,15 +218,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
} else { // Update path for regular routes.
targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
}
wasLocalRetry := false
again:
ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
if needLocalRetry && !wasLocalRetry {
wasLocalRetry = true
goto again
}
ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
bu.put()
if ok {
return
@@ -234,28 +226,29 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
bu.setBroken()
}
err := &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
StatusCode: http.StatusBadGateway,
Err: fmt.Errorf("all the backends for the user %q are unavailable", ui.name()),
StatusCode: http.StatusServiceUnavailable,
}
httpserver.Errorf(w, r, "%s", err)
ui.backendErrors.Inc()
}
func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
// This code has been copied from net/http/httputil/reverseproxy.go
req := sanitizeRequestHeaders(r)
req.URL = targetURL
req.Header.Set("User-Agent", "vmauth")
updateHeadersByConfig(req.Header, hc.RequestHeaders)
if hc.KeepOriginalHost == nil || !*hc.KeepOriginalHost {
if host := getHostHeader(hc.RequestHeaders); host != "" {
req.Host = host
} else {
req.Host = targetURL.Host
}
}
if req.URL.Scheme == "https" || ui.overrideHostHeader {
// Override req.Host only for https requests, since https server verifies hostnames during TLS handshake,
// so it expects the targetURL.Host in the request.
// There is no need in overriding the req.Host for http requests, since it is expected that backend server
// may properly process queries with the original req.Host.
req.Host = targetURL.Host
}
updateHeadersByConfig(req.Header, hc.RequestHeaders)
var trivialRetries int
rtb, rtbOK := req.Body.(*readTrackingBody)
again:
res, err := ui.rt.RoundTrip(req)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -267,7 +260,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
// Timed out request must be counted as errors, since this usually means that the backend is slow.
ui.backendErrors.Inc()
}
return true, false
return true
}
if !rtbOK || !rtb.canRetry() {
// Request body cannot be re-sent to another backend. Return the error to the client then.
@@ -277,19 +270,19 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
}
httpserver.Errorf(w, r, "%s", err)
ui.backendErrors.Inc()
return true, false
return true
}
if netutil.IsTrivialNetworkError(err) {
// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
return false, true
// one time retry trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
if (netutil.IsTrivialNetworkError(err) || errors.Is(err, io.EOF)) && trivialRetries < 1 {
trivialRetries++
goto again
}
// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
// NOTE: do not use httpserver.GetRequestURI
// it explicitly reads request body, which may fail retries.
logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
return false, false
return false
}
if slices.Contains(retryStatusCodes, res.StatusCode) {
_ = res.Body.Close()
@@ -303,7 +296,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
}
httpserver.Errorf(w, r, "%s", err)
ui.backendErrors.Inc()
return true, false
return true
}
// Retry requests at other backends if it matches retryStatusCodes.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
@@ -312,7 +305,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
// it explicitly reads request body, which may fail retries.
logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
return false, false
return false
}
removeHopHeaders(res.Header)
copyHeader(w.Header(), res.Header)
@@ -328,9 +321,9 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
requestURI := httpserver.GetRequestURI(r)
logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
return true, false
return true
}
return true, false
return true
}
var copyBufPool bytesutil.ByteBufferPool
@@ -343,21 +336,12 @@ func copyHeader(dst, src http.Header) {
}
}
func getHostHeader(headers []*Header) string {
for _, h := range headers {
if h.Name == "Host" {
return h.Value
}
}
return ""
}
func updateHeadersByConfig(dst http.Header, src []*Header) {
for _, h := range src {
func updateHeadersByConfig(headers http.Header, config []*Header) {
for _, h := range config {
if h.Value == "" {
dst.Del(h.Name)
headers.Del(h.Name)
} else {
dst.Set(h.Name, h.Value)
headers.Set(h.Name, h.Value)
}
}
}
@@ -497,11 +481,6 @@ See the docs at https://docs.victoriametrics.com/vmauth/ .
flagutil.Usage(s)
}
func handleMissingAuthorizationError(w http.ResponseWriter) {
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(w, "missing 'Authorization' request header", http.StatusUnauthorized)
}
func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
w.Header().Add("Retry-After", "10")
err = &httpserver.ErrorWithStatusCode{
@@ -511,124 +490,85 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
httpserver.Errorf(w, r, "%s", err)
}
// readTrackingBody must be obtained via getReadTrackingBody()
type readTrackingBody struct {
// maxBodySize is the maximum body size to cache in buf.
//
// Bigger bodies cannot be retried.
maxBodySize int
// r contains reader for initial data reading
r io.ReadCloser
// buf is a buffer for data read from r. Buf size is limited by maxBodySize.
// If more than maxBodySize is read from r, then cannotRetry is set to true.
// buf is a buffer for data read from r. Buf size is limited by maxRequestBodySizeToRetry.
// If more than maxRequestBodySizeToRetry is read from r, then cannotRetry is set to true.
buf []byte
// readBuf points to the cached data at buf, which must be read in the next call to Read().
readBuf []byte
// cannotRetry is set to true when more than maxBodySize bytes are read from r.
// cannotRetry is set to true when more than maxRequestBodySizeToRetry are read from r.
// In this case the read data cannot fit buf, so it cannot be re-read from buf.
cannotRetry bool
// bufComplete is set to true when buf contains complete request body read from r.
bufComplete bool
// offset is an offset at buf for the next data read if needReadBuf is set to true.
offset int
}
func (rtb *readTrackingBody) reset() {
rtb.maxBodySize = 0
rtb.r = nil
rtb.buf = rtb.buf[:0]
rtb.readBuf = nil
rtb.cannotRetry = false
rtb.bufComplete = false
}
func getReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
v := readTrackingBodyPool.Get()
if v == nil {
v = &readTrackingBody{}
}
rtb := v.(*readTrackingBody)
if maxBodySize < 0 {
maxBodySize = 0
}
rtb.maxBodySize = maxBodySize
if r == nil {
// This is GET request without request body
r = (*zeroReader)(nil)
}
rtb.r = r
return rtb
}
type zeroReader struct{}
func (r *zeroReader) Read(_ []byte) (int, error) {
return 0, io.EOF
}
func (r *zeroReader) Close() error {
return nil
}
func putReadTrackingBody(rtb *readTrackingBody) {
rtb.reset()
readTrackingBodyPool.Put(rtb)
}
var readTrackingBodyPool sync.Pool
// Read implements io.Reader interface.
// Read implements io.Reader interface
// tracks body reading requests
func (rtb *readTrackingBody) Read(p []byte) (int, error) {
if len(rtb.readBuf) > 0 {
n := copy(p, rtb.readBuf)
rtb.readBuf = rtb.readBuf[n:]
return n, nil
}
if rtb.r == nil {
if rtb.bufComplete {
return 0, io.EOF
if rtb.offset < len(rtb.buf) {
if rtb.cannotRetry {
return 0, fmt.Errorf("cannot retry reading data from buf")
}
return 0, fmt.Errorf("cannot read client request body after closing client reader")
nb := copy(p, rtb.buf[rtb.offset:])
rtb.offset += nb
if rtb.bufComplete {
if rtb.offset == len(rtb.buf) {
return nb, io.EOF
}
return nb, nil
}
if nb < len(p) {
nr, err := rtb.readFromStream(p[nb:])
return nb + nr, err
}
return nb, nil
}
if rtb.bufComplete {
return 0, io.EOF
}
return rtb.readFromStream(p)
}
func (rtb *readTrackingBody) readFromStream(p []byte) (int, error) {
if rtb.r == nil {
return 0, fmt.Errorf("cannot read data after closing the reader")
}
n, err := rtb.r.Read(p)
if rtb.cannotRetry {
return n, err
}
if len(rtb.buf)+n > rtb.maxBodySize {
if rtb.offset+n > maxRequestBodySizeToRetry.IntN() {
rtb.cannotRetry = true
}
if n > 0 {
rtb.offset += n
rtb.buf = append(rtb.buf, p[:n]...)
}
if err != nil {
if err == io.EOF {
rtb.bufComplete = true
return n, err
}
rtb.cannotRetry = true
return n, err
}
rtb.buf = append(rtb.buf, p[:n]...)
if err == io.EOF {
rtb.bufComplete = true
}
return n, err
return n, nil
}
func (rtb *readTrackingBody) canRetry() bool {
if rtb.cannotRetry {
return false
}
if rtb.bufComplete {
return true
}
return rtb.r != nil
return !rtb.cannotRetry
}
// Close implements io.Closer interface.
func (rtb *readTrackingBody) Close() error {
if !rtb.cannotRetry {
rtb.readBuf = rtb.buf
} else {
rtb.readBuf = nil
}
rtb.offset = 0
// Close rtb.r only if the request body is completely read or if it is too big.
// http.Roundtrip performs body.Close call even without any Read calls,
@@ -644,3 +584,38 @@ func (rtb *readTrackingBody) Close() error {
return nil
}
var readTrackingBodyPool sync.Pool
func getReadTrackingBody(origin io.ReadCloser, b int) *readTrackingBody {
bufSize := 1024
if b > 0 && b < maxRequestBodySizeToRetry.IntN() {
bufSize = b
}
v := readTrackingBodyPool.Get()
if v == nil {
v = &readTrackingBody{
buf: make([]byte, 0, bufSize),
}
}
rtb := v.(*readTrackingBody)
rtb.r = origin
if bufSize > cap(rtb.buf) {
rtb.buf = make([]byte, 0, bufSize)
}
return rtb
}
func putReadTrackingBody(rtb *readTrackingBody) {
if rtb.r != nil {
_ = rtb.r.Close()
}
rtb.r = nil
rtb.buf = rtb.buf[:0]
rtb.offset = 0
rtb.cannotRetry = false
rtb.bufComplete = false
readTrackingBodyPool.Put(rtb)
}

View File

@@ -2,520 +2,18 @@ package main
import (
"bytes"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
)
func TestRequestHandler(t *testing.T) {
f := func(cfgStr, requestURL string, backendHandler http.HandlerFunc, responseExpected string) {
func TestReadTrackingBodyRetrySuccess(t *testing.T) {
f := func(s string) {
t.Helper()
ts := httptest.NewServer(backendHandler)
defer ts.Close()
cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
responseExpected = strings.ReplaceAll(responseExpected, "{BACKEND}", ts.URL)
cfgOrigP := authConfigData.Load()
if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
t.Fatalf("cannot load config data: %s", err)
rtb := &readTrackingBody{
r: io.NopCloser(bytes.NewBufferString(s)),
}
defer func() {
cfgOrig := []byte("unauthorized_user:\n url_prefix: http://foo/bar")
if cfgOrigP != nil {
cfgOrig = *cfgOrigP
}
_, err := reloadAuthConfigData(cfgOrig)
if err != nil {
t.Fatalf("cannot load the original config: %s", err)
}
}()
r, err := http.NewRequest(http.MethodGet, requestURL, nil)
if err != nil {
t.Fatalf("cannot initialize http request: %s", err)
}
r.RequestURI = r.URL.RequestURI()
r.RemoteAddr = "42.2.3.84:6789"
r.Header.Set("X-Forwarded-For", "12.34.56.78")
r.Header.Set("Connection", "Some-Header,Other-Header")
r.Header.Set("Some-Header", "foobar")
r.Header.Set("Pass-Header", "abc")
w := &fakeResponseWriter{}
if !requestHandler(w, r) {
t.Fatalf("unexpected false is returned from requestHandler")
}
response := w.getResponse()
response = strings.ReplaceAll(response, "\r\n", "\n")
response = strings.TrimSpace(response)
responseExpected = strings.TrimSpace(responseExpected)
if response != responseExpected {
t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, responseExpected)
}
}
// regular url_prefix
cfgStr := `
unauthorized_user:
url_prefix: {BACKEND}/foo?bar=baz`
requestURL := "http://some-host.com/abc/def?some_arg=some_value"
backendHandler := func(w http.ResponseWriter, r *http.Request) {
h := w.Header()
h.Set("Connection", "close")
h.Set("Foo", "bar")
var bb bytes.Buffer
if err := r.Header.Write(&bb); err != nil {
panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
}
fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
}
responseExpected := `
statusCode=200
Foo: bar
requested_url={BACKEND}/foo/abc/def?bar=baz&some_arg=some_value
Pass-Header: abc
User-Agent: vmauth
X-Forwarded-For: 12.34.56.78, 42.2.3.84`
f(cfgStr, requestURL, backendHandler, responseExpected)
// keep_original_host
cfgStr = `
unauthorized_user:
url_prefix: "{BACKEND}/foo?bar=baz"
keep_original_host: true`
requestURL = "http://some-host.com/abc/def"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url=http://some-host.com/foo/abc/def?bar=baz`
f(cfgStr, requestURL, backendHandler, responseExpected)
// override user-agent header
cfgStr = `
unauthorized_user:
url_prefix: "{BACKEND}/foo?bar=baz"
headers:
- "User-Agent: foobar"`
requestURL = "http://some-host.com/abc/def"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/foo/abc/def?bar=baz
User-Agent=foobar`
f(cfgStr, requestURL, backendHandler, responseExpected)
// delete user-agent header
cfgStr = `
unauthorized_user:
url_prefix: "{BACKEND}/foo?bar=baz"
headers:
- "User-Agent:"`
requestURL = "http://some-host.com/abc/def"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/foo/abc/def?bar=baz
User-Agent=Go-http-client/1.1`
f(cfgStr, requestURL, backendHandler, responseExpected)
// override request host with non-empty host
cfgStr = `
unauthorized_user:
url_prefix: "{BACKEND}/foo?bar=baz"
headers:
- "Host: other-host:12345"
- "abc:"`
requestURL = "http://some-host.com/abc/def"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url=http://other-host:12345/foo/abc/def?bar=baz`
f(cfgStr, requestURL, backendHandler, responseExpected)
// override request host with empty host
cfgStr = `
unauthorized_user:
url_prefix: "{BACKEND}/foo?bar=baz"
headers:
- "Host:"`
requestURL = "http://some-host.com/abc/def"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/foo/abc/def?bar=baz`
f(cfgStr, requestURL, backendHandler, responseExpected)
// /-/reload handler failure
origAuthKey := reloadAuthKey.Get()
if err := reloadAuthKey.Set("secret"); err != nil {
t.Fatalf("unexpected error: %s", err)
}
cfgStr = `
unauthorized_user:
url_prefix: "{BACKEND}/foo"`
requestURL = "http://some-host.com/-/reload"
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
panic(fmt.Errorf("backend handler shouldn't be called"))
}
responseExpected = `
statusCode=401
The provided authKey doesn't match -reloadAuthKey`
f(cfgStr, requestURL, backendHandler, responseExpected)
if err := reloadAuthKey.Set(origAuthKey); err != nil {
t.Fatalf("unexpected error: %s", err)
}
// missing authorization
cfgStr = `
users:
- username: foo
url_prefix: "{BACKEND}/bar"`
requestURL = "http://some-host.com/a/b"
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
panic(fmt.Errorf("backend handler shouldn't be called"))
}
responseExpected = `
statusCode=401
Www-Authenticate: Basic realm="Restricted"
missing 'Authorization' request header`
f(cfgStr, requestURL, backendHandler, responseExpected)
// incorrect authorization
cfgStr = `
users:
- username: foo
password: secret
url_prefix: "{BACKEND}/bar"`
requestURL = "http://foo:invalid-secret@some-host.com/a/b"
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
panic(fmt.Errorf("backend handler shouldn't be called"))
}
responseExpected = `
statusCode=401
Unauthorized`
f(cfgStr, requestURL, backendHandler, responseExpected)
// incorrect authorization with logging invalid auth tokens
origLogInvalidAuthTokens := *logInvalidAuthTokens
*logInvalidAuthTokens = true
cfgStr = `
users:
- username: foo
password: secret
url_prefix: "{BACKEND}/bar"`
requestURL = "http://foo:invalid-secret@some-host.com/a/b?c=d"
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
panic(fmt.Errorf("backend handler shouldn't be called"))
}
responseExpected = `
statusCode=401
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /a/b?c=d; cannot authorize request with auth tokens ["http_auth:Basic Zm9vOmludmFsaWQtc2VjcmV0"]`
f(cfgStr, requestURL, backendHandler, responseExpected)
*logInvalidAuthTokens = origLogInvalidAuthTokens
// correct authorization
cfgStr = `
users:
- username: foo
password: secret
url_prefix: "{BACKEND}/bar"`
requestURL = "http://foo:secret@some-host.com/a/b"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/bar/a/b`
f(cfgStr, requestURL, backendHandler, responseExpected)
// verify how path cleanup works
cfgStr = `
unauthorized_user:
url_prefix: {BACKEND}/foo?bar=baz`
requestURL = "http://some-host.com/../../a//.///bar/"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/foo/a/bar/?bar=baz`
f(cfgStr, requestURL, backendHandler, responseExpected)
// verify how path cleanup works for url without path
cfgStr = `
unauthorized_user:
url_prefix: {BACKEND}/foo?bar=baz`
requestURL = "http://some-host.com/"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/foo?bar=baz`
f(cfgStr, requestURL, backendHandler, responseExpected)
// verify how path cleanup works for url without path if url_prefix path ends with /
cfgStr = `
unauthorized_user:
url_prefix: {BACKEND}/foo/?bar=baz`
requestURL = "http://some-host.com/"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/foo/?bar=baz`
f(cfgStr, requestURL, backendHandler, responseExpected)
// verify how path cleanup works for url without path and the url_prefix without path prefix
cfgStr = `
unauthorized_user:
url_prefix: {BACKEND}/?bar=baz`
requestURL = "http://some-host.com/"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/?bar=baz`
f(cfgStr, requestURL, backendHandler, responseExpected)
// verify routing to default_url
cfgStr = `
unauthorized_user:
url_map:
- src_paths: ["/foo/.+"]
url_prefix: {BACKEND}/x-foo/
default_url: {BACKEND}/404.html`
requestURL = "http://some-host.com/abc?de=fg"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/404.html?request_path=http%3A%2F%2Fsome-host.com%2Fabc%3Fde%3Dfg`
f(cfgStr, requestURL, backendHandler, responseExpected)
// verify routing to default url_prefix
cfgStr = `
unauthorized_user:
url_map:
- src_paths: ["/foo/.+"]
url_prefix: {BACKEND}/x-foo/
url_prefix: {BACKEND}/default`
requestURL = "http://some-host.com/abc?de=fg"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/default/abc?de=fg`
f(cfgStr, requestURL, backendHandler, responseExpected)
// missing default_url and default url_prefix for unauthorized user
cfgStr = `
unauthorized_user:
url_map:
- src_paths: ["/foo/.+"]
url_prefix: {BACKEND}/x-foo/`
requestURL = "http://some-host.com/abc?de=fg"
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
panic(fmt.Errorf("backend handler shouldn't be called"))
}
responseExpected = `
statusCode=400
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /abc?de=fg; missing route for http://some-host.com/abc?de=fg`
f(cfgStr, requestURL, backendHandler, responseExpected)
// missing default_url and default url_prefix for unauthorized user when there are configs for authorized users
cfgStr = `
users:
- username: some-user
url_map:
- src_paths: ["/foo/.+"]
url_prefix: {BACKEND}/x-foo/
unauthorized_user:
url_map:
- src_paths: ["/abc/.*"]
url_prefix: {BACKEND}/x-bar`
requestURL = "http://some-host.com/abc?de=fg"
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
panic(fmt.Errorf("backend handler shouldn't be called"))
}
responseExpected = `
statusCode=401
Www-Authenticate: Basic realm="Restricted"
missing 'Authorization' request header`
f(cfgStr, requestURL, backendHandler, responseExpected)
// all the backend_urls are unavailable for unauthorized user
cfgStr = `
unauthorized_user:
url_map:
- src_paths: ["/foo/.*"]
url_prefix:
- http://127.0.0.1:1/
- http://127.0.0.1:2/`
requestURL = "http://some-host.com/foo/?de=fg"
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
panic(fmt.Errorf("backend handler shouldn't be called"))
}
responseExpected = `
statusCode=502
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
f(cfgStr, requestURL, backendHandler, responseExpected)
// all the backend_urls are unavailable for authorized user
cfgStr = `
users:
- username: some-user
url_map:
- src_paths: ["/foo/.*"]
url_prefix:
- http://127.0.0.1:1/
- http://127.0.0.1:2/`
requestURL = "http://some-user@some-host.com/foo/?de=fg"
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
panic(fmt.Errorf("backend handler shouldn't be called"))
}
responseExpected = `
statusCode=502
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "some-user" are unavailable`
f(cfgStr, requestURL, backendHandler, responseExpected)
// zero discovered backend IPs
customResolver := &fakeResolver{
Resolver: &net.Resolver{},
lookupIPAddrResults: map[string][]net.IPAddr{
"some-addr": {},
},
}
origResolver := netutil.Resolver
netutil.Resolver = customResolver
cfgStr = `
unauthorized_user:
url_prefix: ['http://some-addr:1234/foo/bar']
discover_backend_ips: true`
requestURL = "http://abc.com/def/?de=fg"
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
panic(fmt.Errorf("backend handler shouldn't be called"))
}
responseExpected = `
statusCode=502
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /def/?de=fg; all the 0 backends for the user "" are unavailable`
f(cfgStr, requestURL, backendHandler, responseExpected)
netutil.Resolver = origResolver
// retry_status_codes failure
var retries atomic.Int64
cfgStr = `
unauthorized_user:
url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
retry_status_codes: [500, 502]`
requestURL = "http://some-host.com/foo/?de=fg"
backendHandler = func(w http.ResponseWriter, _ *http.Request) {
retries.Add(1)
w.WriteHeader(500)
}
responseExpected = `
statusCode=502
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
f(cfgStr, requestURL, backendHandler, responseExpected)
if n := retries.Load(); n != 2 {
t.Fatalf("unexpected number of retries; got %d; want 2", n)
}
// retry_status_codes success
retries.Store(0)
cfgStr = `
unauthorized_user:
url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
retry_status_codes: [500, 502]`
requestURL = "http://some-host.com/foo/?de=fg"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
if n := retries.Add(1); n < 2 {
w.WriteHeader(500)
return
}
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=200
requested_url={BACKEND}/path2/foo/?de=fg`
f(cfgStr, requestURL, backendHandler, responseExpected)
if n := retries.Load(); n != 2 {
t.Fatalf("unexpected number of retries; got %d; want 2", n)
}
}
type fakeResponseWriter struct {
h http.Header
bb bytes.Buffer
}
func (w *fakeResponseWriter) getResponse() string {
return w.bb.String()
}
func (w *fakeResponseWriter) Header() http.Header {
if w.h == nil {
w.h = http.Header{}
}
return w.h
}
func (w *fakeResponseWriter) Write(p []byte) (int, error) {
return w.bb.Write(p)
}
func (w *fakeResponseWriter) WriteHeader(statusCode int) {
fmt.Fprintf(&w.bb, "statusCode=%d\n", statusCode)
if w.h == nil {
return
}
err := w.h.WriteSubset(&w.bb, map[string]bool{
"Content-Length": true,
"Content-Type": true,
"Date": true,
"X-Content-Type-Options": true,
})
if err != nil {
panic(fmt.Errorf("cannot marshal headers: %s", err))
}
}
func TestReadTrackingBody_RetrySuccess(t *testing.T) {
f := func(s string, maxBodySize int) {
t.Helper()
rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
defer putReadTrackingBody(rtb)
if !rtb.canRetry() {
t.Fatalf("canRetry() must return true before reading anything")
t.Fatalf("canRetry() must return true")
}
for i := 0; i < 5; i++ {
data, err := io.ReadAll(rtb)
@@ -534,99 +32,41 @@ func TestReadTrackingBody_RetrySuccess(t *testing.T) {
}
}
f("", 0)
f("", -1)
f("", 100)
f("foo", 100)
f("foobar", 100)
f(newTestString(1000), 1000)
f("")
f("foo")
f("foobar")
f(newTestString(maxRequestBodySizeToRetry.IntN()))
}
func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
f := func(s string, maxBodySize int) {
func TestReadTrackingBodyRetryFailure(t *testing.T) {
f := func(s string) {
t.Helper()
// Check the case with partial read
rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
defer putReadTrackingBody(rtb)
for i := 0; i < len(s); i++ {
buf := make([]byte, i)
n, err := io.ReadFull(rtb, buf)
if err != nil {
t.Fatalf("unexpected error when reading %d bytes: %s", i, err)
}
if n != i {
t.Fatalf("unexpected number of bytes read; got %d; want %d", n, i)
}
if string(buf) != s[:i] {
t.Fatalf("unexpected data read with the length %d\ngot\n%s\nwant\n%s", i, buf, s[:i])
}
if err := rtb.Close(); err != nil {
t.Fatalf("unexpected error when closing reader after reading %d bytes", i)
}
if !rtb.canRetry() {
t.Fatalf("canRetry() must return true after closing the reader after reading %d bytes", i)
}
}
data, err := io.ReadAll(rtb)
if err != nil {
t.Fatalf("unexpected error when reading all the data: %s", err)
}
if string(data) != s {
t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
}
if err := rtb.Close(); err != nil {
t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
rtb := &readTrackingBody{
r: io.NopCloser(bytes.NewBufferString(s)),
}
if !rtb.canRetry() {
t.Fatalf("canRetry() must return true after closing the reader after reading all the input")
}
}
f("", 0)
f("", -1)
f("", 100)
f("foo", 100)
f("foobar", 100)
f(newTestString(1000), 1000)
}
func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
f := func(s string, maxBodySize int) {
t.Helper()
rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
defer putReadTrackingBody(rtb)
if !rtb.canRetry() {
t.Fatalf("canRetry() must return true before reading anything")
t.Fatalf("canRetry() must return true")
}
buf := make([]byte, 1)
n, err := io.ReadFull(rtb, buf)
n, err := rtb.Read(buf)
if err != nil {
t.Fatalf("unexpected error when reading a single byte: %s", err)
}
if n != 1 {
t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
}
if !rtb.canRetry() {
t.Fatalf("canRetry() must return true after reading one byte")
}
data, err := io.ReadAll(rtb)
if err != nil {
t.Fatalf("unexpected error when reading all the data: %s", err)
}
dataRead := string(buf) + string(data)
if dataRead != s {
t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", dataRead, s)
if string(buf)+string(data) != s {
t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", string(buf)+string(data), s)
}
if err := rtb.Close(); err != nil {
t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
}
if rtb.canRetry() {
t.Fatalf("canRetry() must return false after closing the reader")
t.Fatalf("canRetry() must return false")
}
data, err = io.ReadAll(rtb)
@@ -638,55 +78,132 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
}
}
const maxBodySize = 1000
f(newTestString(maxBodySize+1), maxBodySize)
f(newTestString(2*maxBodySize), maxBodySize)
f(newTestString(maxRequestBodySizeToRetry.IntN() + 1))
f(newTestString(2 * maxRequestBodySizeToRetry.IntN()))
}
func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
f := func(s string, maxBodySize int) {
t.Helper()
// request body not over maxRequestBodySizeToRetry
// 1. When writing data downstream, buf only caches part of the data because the downstream connection is disconnected.
// 2. retry request: because buf caches some data, first read buf and then read stream when retrying
// 3. retry request: the data has been read to buf in the second step. if the request fails, retry to read all buf later.
func TestRetryReadSuccessAfterPartialRead(t *testing.T) {
f := func(s string) {
rtb := &readTrackingBody{
r: io.NopCloser(bytes.NewBufferString(s)),
buf: make([]byte, 0, len(s)),
}
rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
defer putReadTrackingBody(rtb)
var data []byte
var err error
halfSize := len(s) / 2
if halfSize == 0 {
halfSize = 100
}
buf := make([]byte, halfSize)
var n int
if !rtb.canRetry() {
t.Fatalf("canRetry() must return true before reading anything")
}
data, err := io.ReadAll(rtb)
if err != nil {
t.Fatalf("unexpected error when reading all the data: %s", err)
}
if string(data) != s {
t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
// read part of the data
n, err = rtb.Read(buf[:])
data = append(data, buf[:n]...)
if err != nil && err != io.EOF {
t.Fatalf("unexpected error: %s", err)
}
// request failed when output stream is closed (eg: server connection reset)
// would close the reader
if err := rtb.Close(); err != nil {
t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
}
if !rtb.canRetry() {
t.Fatalf("canRetry() must return true")
}
if rtb.canRetry() {
t.Fatalf("canRetry() must return false after closing the reader")
// retry read (read buf + remaining data)
data = data[:0]
err = nil
for err == nil {
n, err = rtb.Read(buf[:])
data = append(data, buf[:n]...)
}
data, err = io.ReadAll(rtb)
if err == nil {
t.Fatalf("expecting non-nil error")
if err != io.EOF {
t.Fatalf("unexpected error: %s", err)
}
if len(data) != 0 {
t.Fatalf("unexpected non-empty data read: %q", data)
if string(data) != s {
t.Fatalf("unexpected data read; got\n%s\nwant\n%s", data, s)
}
// cannotRetry return false
// because the request data is not over maxRequestBodySizeToRetry limit
if !rtb.canRetry() {
t.Fatalf("canRetry() must return true")
}
}
f("foobar", 0)
f(newTestString(1000), 0)
f("")
f("foo")
f("foobar")
f(newTestString(maxRequestBodySizeToRetry.IntN()))
}
f("foobar", -1)
f(newTestString(1000), -1)
// request body over maxRequestBodySizeToRetry
// 1. When writing data downstream, buf only caches part of the data because the downstream connection is disconnected.
// 2. retry request: because buf caches some data, first read buf and then read stream when retrying
// 3. retry request: the data has been read to buf in the second step. if the request fails, retry to read all buf later.
func TestRetryReadSuccessAfterPartialReadAndCannotRetryAgain(t *testing.T) {
f := func(s string) {
rtb := &readTrackingBody{
r: io.NopCloser(bytes.NewBufferString(s)),
buf: make([]byte, 0, len(s)),
}
var data []byte
var err error
halfSize := len(s) / 2
if halfSize == 0 {
halfSize = 100
}
buf := make([]byte, halfSize)
var n int
// read part of the data
n, err = rtb.Read(buf[:])
data = append(data, buf[:n]...)
if err != nil && err != io.EOF {
t.Fatalf("unexpected error: %s", err)
}
// request failed when output stream is closed (eg: server connection reset)
if err := rtb.Close(); err != nil {
t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
}
if !rtb.canRetry() {
t.Fatalf("canRetry() must return true")
}
// retry read (read buf + remaining data)
data = data[:0]
err = nil
for err == nil {
n, err = rtb.Read(buf[:])
data = append(data, buf[:n]...)
}
if err != io.EOF {
t.Fatalf("unexpected error: %s", err)
}
if string(data) != s {
t.Fatalf("unexpected data read; got\n%s\nwant\n%s", data, s)
}
// cannotRetry returns true
// because the request data is over maxRequestBodySizeToRetry limit
if rtb.canRetry() {
t.Fatalf("canRetry() must return false")
}
}
f(newTestString(maxRequestBodySizeToRetry.IntN() + 1))
f(newTestString(2 * maxRequestBodySizeToRetry.IntN()))
}
func newTestString(sLen int) string {
data := make([]byte, sLen)
for i := range data {
data[i] = byte(i)
}
return string(data)
return string(make([]byte, sLen))
}

View File

@@ -335,62 +335,13 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
}
f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.50:8481/select/0/prometheus/api/v1/query?query=up")
f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up")
f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.50:8481/select/0/prometheus/api/v1/query?query=up")
f(ui, `/insert/0/prometheus/api/v1/write`, "http://10.6.142.52:8480/insert/0/prometheus/api/v1/write")
// unsuccessful dns resolve
f(ui, `/test`, "http://non-exist-dns-addr/test")
}
func TestUserInfoGetBackendURL_SRVZeroBackends(t *testing.T) {
f := func(ui *UserInfo, requestURI string) {
t.Helper()
u, err := url.Parse(requestURI)
if err != nil {
t.Fatalf("cannot parse %q: %s", requestURI, err)
}
u = normalizeURL(u)
up, _ := ui.getURLPrefixAndHeaders(u, nil)
if up == nil {
t.Fatalf("cannot match available backend: %s", err)
}
bu := up.getBackendURL()
if bu != nil {
t.Fatalf("expecting nil backendURL; got %v", bu)
}
}
customResolver := &fakeResolver{
Resolver: &net.Resolver{},
lookupSRVResults: map[string][]*net.SRV{
"vmselect": {},
},
}
origResolver := netutil.Resolver
netutil.Resolver = customResolver
defer func() {
netutil.Resolver = origResolver
}()
allowed := true
ui := &UserInfo{
URLMaps: []URLMap{
{
SrcPaths: getRegexs([]string{"/select/.+"}),
URLPrefix: mustParseURL("http://srv+vmselect"),
},
},
DiscoverBackendIPs: &allowed,
URLPrefix: mustParseURL("http://non-exist-dns-addr"),
}
if err := ui.initURLs(); err != nil {
t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
}
f(ui, `/select/0/prometheus/api/v1/query?query=up`)
}
func TestCreateTargetURLFailure(t *testing.T) {
f := func(ui *UserInfo, requestURI string) {
t.Helper()

View File

@@ -10,6 +10,12 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
const (
backoffRetries = 10
backoffFactor = 1.8
backoffMinDuration = time.Second * 2
)
// retryableFunc describes call back which will repeat on errors
type retryableFunc func() error
@@ -24,22 +30,12 @@ type Backoff struct {
}
// New initialize backoff object
func New(retries int, factor float64, minDuration time.Duration) (*Backoff, error) {
if retries <= 0 {
return nil, fmt.Errorf("number of backoff retries must be greater than 0")
}
if factor <= 1 {
return nil, fmt.Errorf("backoff retry factor must be greater than 1")
}
if minDuration <= 0 {
return nil, fmt.Errorf("backoff retry minimum duration must be greater than 0")
}
func New() *Backoff {
return &Backoff{
retries: retries,
factor: factor,
minDuration: minDuration,
}, nil
retries: backoffRetries,
factor: backoffFactor,
minDuration: backoffMinDuration,
}
}
// Retry process retries until all attempts are completed

View File

@@ -3,7 +3,6 @@ package backoff
import (
"context"
"fmt"
"strings"
"testing"
"time"
)
@@ -111,32 +110,3 @@ func TestBackoffRetry_Success(t *testing.T) {
resultExpected := 1
f(retryFunc, resultExpected)
}
func TestBackoff_New(t *testing.T) {
f := func(retries int, factor float64, minDuration time.Duration, errExpected string) {
t.Helper()
_, err := New(retries, factor, minDuration)
if err == nil {
if errExpected != "" {
t.Fatalf("expecting non-nil error")
}
return
}
if !strings.Contains(err.Error(), errExpected) {
t.Fatalf("unexpected error: got %q; want %q", err.Error(), errExpected)
}
}
// empty retries
f(0, 1.1, time.Millisecond*10, "retries must be greater than 0")
// empty factor
f(1, 0, time.Millisecond*10, "factor must be greater than 1")
// empty minDuration
f(1, 1.1, 0, "minimum duration must be greater than 0")
// no errors
f(1, 1.1, time.Millisecond*10, "")
}

View File

@@ -56,10 +56,6 @@ const (
vmRateLimit = "vm-rate-limit"
vmInterCluster = "vm-intercluster"
vmBackoffRetries = "vm-backoff-retries"
vmBackoffFactor = "vm-backoff-factor"
vmBackoffMinDuration = "vm-backoff-min-duration"
)
var (
@@ -150,21 +146,6 @@ var (
Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
Value: false,
},
&cli.IntFlag{
Name: vmBackoffRetries,
Value: 10,
Usage: "How many import retries to perform before giving up.",
},
&cli.Float64Flag{
Name: vmBackoffFactor,
Value: 1.8,
Usage: "Factor to multiply the base duration after each failed import retry. Must be greater than 1.0",
},
&cli.DurationFlag{
Name: vmBackoffMinDuration,
Value: time.Second * 2,
Usage: "Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'.",
},
}
)
@@ -449,10 +430,6 @@ const (
vmNativeDstCAFile = "vm-native-dst-ca-file"
vmNativeDstServerName = "vm-native-dst-server-name"
vmNativeDstInsecureSkipVerify = "vm-native-dst-insecure-skip-verify"
vmNativeBackoffRetries = "vm-native-backoff-retries"
vmNativeBackoffFactor = "vm-native-backoff-factor"
vmNativeBackoffMinDuration = "vm-native-backoff-min-duration"
)
var (
@@ -622,21 +599,6 @@ var (
"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
Value: false,
},
&cli.IntFlag{
Name: vmNativeBackoffRetries,
Value: 10,
Usage: "How many export/import retries to perform before giving up.",
},
&cli.Float64Flag{
Name: vmNativeBackoffFactor,
Value: 1.8,
Usage: "Factor to multiply the base duration after each failed export/import retry. Must be greater than 1.0",
},
&cli.DurationFlag{
Name: vmNativeBackoffMinDuration,
Value: time.Second * 2,
Usage: "Minimum duration to wait before the first export/import retry. Each subsequent export/import retry will be multiplied by the '--vm-native-backoff-factor'.",
},
}
)

View File

@@ -68,7 +68,7 @@ func main() {
tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
if err != nil {
return fmt.Errorf("failed to create transport for -%s=%q: %s", otsdbAddr, addr, err)
return fmt.Errorf("failed to create Transport: %s", err)
}
oCfg := opentsdb.Config{
Addr: addr,
@@ -90,7 +90,6 @@ func main() {
if err != nil {
return fmt.Errorf("failed to init VM configuration: %s", err)
}
importer, err := vm.NewImporter(ctx, vmCfg)
if err != nil {
return fmt.Errorf("failed to create VM importer: %s", err)
@@ -144,7 +143,6 @@ func main() {
if err != nil {
return fmt.Errorf("failed to init VM configuration: %s", err)
}
importer, err = vm.NewImporter(ctx, vmCfg)
if err != nil {
return fmt.Errorf("failed to create VM importer: %s", err)
@@ -180,7 +178,7 @@ func main() {
tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
if err != nil {
return fmt.Errorf("failed to create transport for -%s=%q: %s", remoteReadSrcAddr, addr, err)
return fmt.Errorf("failed to create transport: %s", err)
}
rr, err := remoteread.NewClient(remoteread.Config{
@@ -203,7 +201,6 @@ func main() {
if err != nil {
return fmt.Errorf("failed to init VM configuration: %s", err)
}
importer, err := vm.NewImporter(ctx, vmCfg)
if err != nil {
return fmt.Errorf("failed to create VM importer: %s", err)
@@ -236,7 +233,6 @@ func main() {
if err != nil {
return fmt.Errorf("failed to init VM configuration: %s", err)
}
importer, err = vm.NewImporter(ctx, vmCfg)
if err != nil {
return fmt.Errorf("failed to create VM importer: %s", err)
@@ -276,14 +272,6 @@ func main() {
return fmt.Errorf("flag %q can't be empty", vmNativeFilterMatch)
}
bfRetries := c.Int(vmNativeBackoffRetries)
bfFactor := c.Float64(vmNativeBackoffFactor)
bfMinDuration := c.Duration(vmNativeBackoffMinDuration)
bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
if err != nil {
return fmt.Errorf("failed to create backoff object: %s", err)
}
disableKeepAlive := c.Bool(vmNativeDisableHTTPKeepAlive)
var srcExtraLabels []string
@@ -362,7 +350,7 @@ func main() {
ExtraLabels: dstExtraLabels,
HTTPClient: dstHTTPClient,
},
backoff: bf,
backoff: backoff.New(),
cc: c.Int(vmConcurrency),
disablePerMetricRequests: c.Bool(vmNativeDisablePerMetricMigration),
isNative: !c.Bool(vmNativeDisableBinaryProtocol),
@@ -438,15 +426,7 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
if err != nil {
return vm.Config{}, fmt.Errorf("failed to create transport for -%s=%q: %s", vmAddr, addr, err)
}
bfRetries := c.Int(vmBackoffRetries)
bfFactor := c.Float64(vmBackoffFactor)
bfMinDuration := c.Duration(vmBackoffMinDuration)
bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
if err != nil {
return vm.Config{}, fmt.Errorf("failed to create backoff object: %s", err)
return vm.Config{}, fmt.Errorf("failed to create Transport: %s", err)
}
return vm.Config{
@@ -462,6 +442,5 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
RoundDigits: c.Int(vmRoundDigits),
ExtraLabels: c.StringSlice(vmExtraLabel),
RateLimit: c.Int64(vmRateLimit),
Backoff: bf,
}, nil
}

View File

@@ -8,7 +8,6 @@ import (
"github.com/prometheus/prometheus/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
@@ -150,12 +149,6 @@ func TestRemoteRead(t *testing.T) {
tt.vmCfg.Addr = remoteWriteServer.URL()
b, err := backoff.New(10, 1.8, time.Second*2)
if err != nil {
t.Fatalf("failed to create backoff: %s", err)
}
tt.vmCfg.Backoff = b
importer, err := vm.NewImporter(ctx, tt.vmCfg)
if err != nil {
t.Fatalf("failed to create VM importer: %s", err)
@@ -315,12 +308,6 @@ func TestSteamRemoteRead(t *testing.T) {
tt.vmCfg.Addr = remoteWriteServer.URL()
b, err := backoff.New(10, 1.8, time.Second*2)
if err != nil {
t.Fatalf("failed to create backoff: %s", err)
}
tt.vmCfg.Backoff = b
importer, err := vm.NewImporter(ctx, tt.vmCfg)
if err != nil {
t.Fatalf("failed to create VM importer: %s", err)

View File

@@ -54,8 +54,6 @@ type Config struct {
// RateLimit defines a data transfer speed in bytes per second.
// Is applied to each worker (see Concurrency) independently.
RateLimit int64
// Backoff defines backoff policy for retries
Backoff *backoff.Backoff
}
// Importer performs insertion of timeseries
@@ -146,7 +144,7 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
close: make(chan struct{}),
input: make(chan *TimeSeries, cfg.Concurrency*4),
errors: make(chan *ImportError, cfg.Concurrency),
backoff: cfg.Backoff,
backoff: backoff.New(),
}
if err := im.Ping(); err != nil {
return nil, fmt.Errorf("ping to %q failed: %s", addr, err)

View File

@@ -81,16 +81,11 @@ func TestVMNativeProcessorRun(t *testing.T) {
isSilent = true
defer func() { isSilent = false }()
bf, err := backoff.New(10, 1.8, time.Second*2)
if err != nil {
t.Fatalf("cannot create backoff: %s", err)
}
p := &vmNativeProcessor{
filter: filter,
dst: dstClient,
src: srcClient,
backoff: bf,
backoff: backoff.New(),
cc: 1,
isNative: true,
}

View File

@@ -2,23 +2,16 @@ package common
import (
"sync"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
)
// GetInsertCtx returns InsertCtx from the pool.
//
// Call PutInsertCtx for returning it to the pool.
func GetInsertCtx() *InsertCtx {
select {
case ctx := <-insertCtxPoolCh:
return ctx
default:
if v := insertCtxPool.Get(); v != nil {
return v.(*InsertCtx)
}
return &InsertCtx{}
if v := insertCtxPool.Get(); v != nil {
return v.(*InsertCtx)
}
return &InsertCtx{}
}
// PutInsertCtx returns ctx to the pool.
@@ -26,14 +19,7 @@ func GetInsertCtx() *InsertCtx {
// ctx cannot be used after the call.
func PutInsertCtx(ctx *InsertCtx) {
ctx.Reset(0)
select {
case insertCtxPoolCh <- ctx:
default:
insertCtxPool.Put(ctx)
}
insertCtxPool.Put(ctx)
}
var (
insertCtxPool sync.Pool
insertCtxPoolCh = make(chan *InsertCtx, cgroup.AvailableCPUs())
)
var insertCtxPool sync.Pool

View File

@@ -42,7 +42,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
)
var (
@@ -218,10 +217,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
addInfluxResponseHeaders(w)
influxutils.WriteDatabaseNames(w)
return true
case "/influx/health":
influxHealthRequests.Inc()
influxutils.WriteHealthCheckResponse(w)
return true
case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
opentelemetryPushRequests.Inc()
if err := opentelemetry.InsertHandler(r); err != nil {
@@ -348,7 +343,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
w.Header().Set("Content-Type", "application/json")
var bb bytesutil.ByteBuffer
promscrape.WriteConfigData(&bb)
fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
return true
case "/prometheus/-/reload", "/-/reload":
if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
@@ -401,8 +396,7 @@ var (
influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/influx/write", protocol="influx"}`)
influxWriteErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/influx/write", protocol="influx"}`)
influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/influx/query", protocol="influx"}`)
influxHealthRequests = metrics.NewCounter(`vm_http_requests_total{path="/influx/health", protocol="influx"}`)
influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/influx/query", protocol="influx"}`)
datadogv1WriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
datadogv1WriteErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

View File

@@ -115,7 +115,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
startTime := time.Now()
defer requestDuration.UpdateDuration(startTime)
tracerEnabled := httputils.GetBool(r, "trace")
qt := querytracer.New(tracerEnabled, "%s", r.URL.Path)
qt := querytracer.New(tracerEnabled, r.URL.Path)
// Limit the number of concurrent queries.
select {

View File

@@ -7,7 +7,6 @@ import (
"sync"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
@@ -15,10 +14,6 @@ import (
"github.com/VictoriaMetrics/metrics"
)
var tmpBufSize = flagutil.NewBytes("search.inmemoryBufSizeBytes", 0, "Size for in-memory data blocks used during processing search requests. "+
"By default, the size is automatically calculated based on available memory. "+
"Adjust this flag value if you observe that vm_tmp_blocks_max_inmemory_file_size_bytes metric constantly shows much higher values than vm_tmp_blocks_inmemory_file_size_bytes. See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6851")
// InitTmpBlocksDir initializes directory to store temporary search results.
//
// It stores data in system-defined temporary directory if tmpDirPath is empty.
@@ -34,9 +29,6 @@ func InitTmpBlocksDir(tmpDirPath string) {
var tmpBlocksDir string
func maxInmemoryTmpBlocksFile() int {
if tmpBufSize.IntN() > 0 {
return tmpBufSize.IntN()
}
mem := memory.Allowed()
maxLen := mem / 1024
if maxLen < 64*1024 {
@@ -48,12 +40,9 @@ func maxInmemoryTmpBlocksFile() int {
return maxLen
}
var (
_ = metrics.NewGauge(`vm_tmp_blocks_max_inmemory_file_size_bytes`, func() float64 {
return float64(maxInmemoryTmpBlocksFile())
})
tmpBufSizeSummary = metrics.NewSummary(`vm_tmp_blocks_inmemory_file_size_bytes`)
)
var _ = metrics.NewGauge(`vm_tmp_blocks_max_inmemory_file_size_bytes`, func() float64 {
return float64(maxInmemoryTmpBlocksFile())
})
type tmpBlocksFile struct {
buf []byte
@@ -76,8 +65,6 @@ func getTmpBlocksFile() *tmpBlocksFile {
func putTmpBlocksFile(tbf *tmpBlocksFile) {
tbf.MustClose()
bufLen := tbf.Len()
tmpBufSizeSummary.Update(float64(bufLen))
tbf.buf = tbf.buf[:0]
tbf.f = nil
tbf.r = nil
@@ -151,7 +138,7 @@ func (tbf *tmpBlocksFile) Finalize() error {
tbf.buf = tbf.buf[:0]
r := fs.NewReaderAt(tbf.f)
// Hint the OS that the file is read almost sequentially.
// Hint the OS that the file is read almost sequentiallly.
// This should reduce the number of disk seeks, which is important
// for HDDs.
r.MustFadviseSequentialRead(true)

View File

@@ -7,8 +7,6 @@ import (
"sync"
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
)
// ActiveQueriesHandler returns response to /api/v1/status/active_queries
@@ -25,8 +23,8 @@ func ActiveQueriesHandler(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintf(w, `{"status":"ok","data":[`)
for i, aqe := range aqes {
d := now.Sub(aqe.startTime)
fmt.Fprintf(w, `{"duration":"%.3fs","id":"%016X","remote_addr":%s,"query":%s,"start":%d,"end":%d,"step":%d}`,
d.Seconds(), aqe.qid, aqe.quotedRemoteAddr, stringsutil.JSONString(aqe.q), aqe.start, aqe.end, aqe.step)
fmt.Fprintf(w, `{"duration":"%.3fs","id":"%016X","remote_addr":%s,"query":%q,"start":%d,"end":%d,"step":%d}`,
d.Seconds(), aqe.qid, aqe.quotedRemoteAddr, aqe.q, aqe.start, aqe.end, aqe.step)
if i+1 < len(aqes) {
fmt.Fprintf(w, `,`)
}

View File

@@ -1052,7 +1052,7 @@ func fixBrokenBuckets(i int, xss []leTimeseries) {
// Substitute upper bucket values with lower bucket values if the upper values are NaN
// or are bigger than the lower bucket values.
vNext := xss[0].ts.Values[i]
vNext := xss[0].ts.Values[0]
for j := 1; j < len(xss); j++ {
v := xss[j].ts.Values[i]
if math.IsNaN(v) || vNext > v {

View File

@@ -39,30 +39,6 @@ func TestFixBrokenBuckets(t *testing.T) {
f([]float64{5, 10, 4, 3}, []float64{5, 10, 10, 10})
}
func TestFixBrokenBucketsMultipleValues(t *testing.T) {
f := func(values, expectedResult [][]float64) {
t.Helper()
xss := make([]leTimeseries, len(values))
for i, v := range values {
xss[i].ts = &timeseries{
Values: v,
}
}
for i := range len(values) - 1 {
fixBrokenBuckets(i, xss)
}
result := make([][]float64, len(values))
for i, xs := range xss {
result[i] = xs.ts.Values
}
if !reflect.DeepEqual(result, expectedResult) {
t.Fatalf("unexpected result for values=%v\ngot\n%v\nwant\n%v", values, result, expectedResult)
}
}
f([][]float64{{10, 1}, {11, 2}, {13, 3}}, [][]float64{{10, 1}, {11, 2}, {13, 3}})
}
func TestVmrangeBucketsToLE(t *testing.T) {
f := func(buckets, bucketsExpected string) {
t.Helper()

View File

@@ -9,7 +9,6 @@ import (
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
)
var (
@@ -75,13 +74,13 @@ func initQueryStats() {
}
func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, maxLifetime time.Duration) {
fmt.Fprintf(w, `{"topN":"%d","maxLifetime":"%s",`, topN, maxLifetime)
fmt.Fprintf(w, `{"topN":"%d","maxLifetime":%q,`, topN, maxLifetime)
fmt.Fprintf(w, `"search.queryStats.lastQueriesCount":%d,`, *lastQueriesCount)
fmt.Fprintf(w, `"search.queryStats.minQueryDuration":"%s",`, *minQueryDuration)
fmt.Fprintf(w, `"search.queryStats.minQueryDuration":%q,`, *minQueryDuration)
fmt.Fprintf(w, `"topByCount":[`)
topByCount := qst.getTopByCount(topN, maxLifetime)
for i, r := range topByCount {
fmt.Fprintf(w, `{"query":%s,"timeRangeSeconds":%d,"count":%d}`, stringsutil.JSONString(r.query), r.timeRangeSecs, r.count)
fmt.Fprintf(w, `{"query":%q,"timeRangeSeconds":%d,"count":%d}`, r.query, r.timeRangeSecs, r.count)
if i+1 < len(topByCount) {
fmt.Fprintf(w, `,`)
}
@@ -89,7 +88,7 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, maxLife
fmt.Fprintf(w, `],"topByAvgDuration":[`)
topByAvgDuration := qst.getTopByAvgDuration(topN, maxLifetime)
for i, r := range topByAvgDuration {
fmt.Fprintf(w, `{"query":%s,"timeRangeSeconds":%d,"avgDurationSeconds":%.3f,"count":%d}`, stringsutil.JSONString(r.query), r.timeRangeSecs, r.duration.Seconds(), r.count)
fmt.Fprintf(w, `{"query":%q,"timeRangeSeconds":%d,"avgDurationSeconds":%.3f,"count":%d}`, r.query, r.timeRangeSecs, r.duration.Seconds(), r.count)
if i+1 < len(topByAvgDuration) {
fmt.Fprintf(w, `,`)
}
@@ -97,7 +96,7 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, maxLife
fmt.Fprintf(w, `],"topBySumDuration":[`)
topBySumDuration := qst.getTopBySumDuration(topN, maxLifetime)
for i, r := range topBySumDuration {
fmt.Fprintf(w, `{"query":%s,"timeRangeSeconds":%d,"sumDurationSeconds":%.3f,"count":%d}`, stringsutil.JSONString(r.query), r.timeRangeSecs, r.duration.Seconds(), r.count)
fmt.Fprintf(w, `{"query":%q,"timeRangeSeconds":%d,"sumDurationSeconds":%.3f,"count":%d}`, r.query, r.timeRangeSecs, r.duration.Seconds(), r.count)
if i+1 < len(topBySumDuration) {
fmt.Fprintf(w, `,`)
}

View File

@@ -1,13 +1,13 @@
{
"files": {
"main.css": "./static/css/main.81c6ec3a.css",
"main.js": "./static/js/main.9ed27532.js",
"main.css": "./static/css/main.fce049bf.css",
"main.js": "./static/js/main.4b213512.js",
"static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
"static/media/MetricsQL.md": "./static/media/MetricsQL.8c2e588d62b87f90dbf0.md",
"static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
"index.html": "./index.html"
},
"entrypoints": [
"static/css/main.81c6ec3a.css",
"static/js/main.9ed27532.js"
"static/css/main.fce049bf.css",
"static/js/main.4b213512.js"
]
}

View File

@@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.9ed27532.js"></script><link href="./static/css/main.81c6ec3a.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.4b213512.js"></script><link href="./static/css/main.fce049bf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -5,7 +5,7 @@
*/
/**
* @remix-run/router v1.17.0
* @remix-run/router v1.15.1
*
* Copyright (c) Remix Software Inc.
*
@@ -16,7 +16,7 @@
*/
/**
* React Router DOM v6.24.0
* React Router DOM v6.22.1
*
* Copyright (c) Remix Software Inc.
*
@@ -27,7 +27,7 @@
*/
/**
* React Router v6.24.0
* React Router v6.22.1
*
* Copyright (c) Remix Software Inc.
*

File diff suppressed because one or more lines are too long

View File

@@ -1,4 +1,5 @@
---
sort: 23
weight: 23
title: MetricsQL
menu:
@@ -9,6 +10,9 @@ aliases:
- /ExtendedPromQL.html
- /MetricsQL.html
---
# MetricsQL
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -216,11 +220,9 @@ See also [descent_over_time](#descent_over_time).
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
See also [median_over_time](#median_over_time).
#### changes
@@ -260,8 +262,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
#### count_gt_over_time
@@ -272,8 +272,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
#### count_le_over_time
@@ -284,8 +282,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
#### count_ne_over_time
@@ -296,9 +292,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
See also [count_over_time](#count_over_time).
#### count_over_time
@@ -319,8 +313,6 @@ The results are calculated independently per each time series returned from the
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
#### decreases_over_time
@@ -440,8 +432,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### histogram_over_time
`histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -453,15 +443,11 @@ For example, the following query calculates median temperature by country over t
`histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### hoeffding_bound_lower
`hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [hoeffding_bound_upper](#hoeffding_bound_upper).
#### hoeffding_bound_upper
@@ -469,8 +455,6 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
`hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [hoeffding_bound_lower](#hoeffding_bound_lower).
#### holt_winters
@@ -478,9 +462,8 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
Both `sf` and `tf` must be in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
@@ -512,14 +495,13 @@ See also [deriv](#deriv).
`increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
This function is supported by PromQL.
See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -528,13 +510,12 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
`increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [increase_pure](#increase_pure) and [increase](#increase).
#### increase_pure
@@ -543,10 +524,6 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
while [increase](#increase) ignores the first value in a series if it is too big.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [increase](#increase) and [increase_prometheus](#increase_prometheus).
#### increases_over_time
`increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -563,18 +540,15 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### irate
`irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
This function is supported by PromQL.
See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -613,8 +587,6 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
#### max_over_time
@@ -622,11 +594,9 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
`max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
See also [tmax_over_time](#tmax_over_time).
#### median_over_time
@@ -634,8 +604,6 @@ See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
on the given lookbehind window `d` per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [avg_over_time](#avg_over_time).
#### min_over_time
@@ -643,11 +611,9 @@ See also [avg_over_time](#avg_over_time).
`min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
See also [tmin_over_time](#tmin_over_time).
#### mode_over_time
@@ -656,8 +622,6 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values are discrete.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### outlier_iqr_over_time
`outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -668,8 +632,6 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [outliers_iqr](#outliers_iqr).
#### predict_linear
@@ -697,8 +659,6 @@ This function is supported by PromQL.
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
The `phi` value must be in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [quantiles_over_time](#quantiles_over_time).
@@ -710,8 +670,6 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [quantile_over_time](#quantile_over_time).
#### range_over_time
@@ -722,12 +680,11 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### rate
`rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -746,8 +703,6 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
on the given lookbehind window `d`. The calculations are performed individually per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
#### resets
@@ -755,11 +710,10 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
`resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
This function is supported by PromQL.
#### rollup
@@ -771,10 +725,6 @@ These values are calculated individually per each time series returned from the
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
See also [label_match](#label_match).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [rollup_rate](#rollup_rate).
#### rollup_candlestick
`rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -786,8 +736,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
See also [label_match](#label_match).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### rollup_delta
`rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -814,8 +762,6 @@ See also [label_match](#label_match).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
#### rollup_increase
`rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -828,17 +774,12 @@ See also [label_match](#label_match).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
#### rollup_rate
`rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
when to use `rollup_rate()`.
@@ -846,12 +787,10 @@ when to use `rollup_rate()`.
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
See also [label_match](#label_match).
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
#### rollup_scrape_interval
`rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -885,8 +824,6 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
#### share_le_over_time
@@ -901,8 +838,6 @@ the share of time series values for the last 24 hours when memory usage was belo
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
#### share_eq_over_time
@@ -914,8 +849,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_eq_over_time](#count_eq_over_time).
#### stale_samples_over_time
@@ -933,8 +866,6 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [stdvar_over_time](#stdvar_over_time).
@@ -946,43 +877,35 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [stddev_over_time](#stddev_over_time).
#### sum_eq_over_time
`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values equal to `eq` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
#### sum_gt_over_time
`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values bigger than `gt` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
#### sum_le_over_time
`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values smaller or equal to `le` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
#### sum_over_time
@@ -992,8 +915,6 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
#### sum2_over_time
@@ -1003,8 +924,6 @@ values on the given lookbehind window `d` per each time series returned from the
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### timestamp
`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1082,8 +1001,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
@@ -2313,8 +2230,7 @@ Any [rollup function](#rollup-functions) for something other than [series select
Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
VictoriaMetrics performs subqueries in the following way:
@@ -2347,6 +2263,5 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
* If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.

View File

@@ -1,55 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -1,56 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -1,56 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -21,7 +21,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/mergeset"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
)
@@ -154,9 +153,9 @@ func AddRows(mrs []storage.MetricRow) error {
}
resetResponseCacheIfNeeded(mrs)
WG.Add(1)
Storage.AddRows(mrs, uint8(*precisionBits))
err := Storage.AddRows(mrs, uint8(*precisionBits))
WG.Done()
return nil
return err
}
var errReadOnly = errors.New("the storage is in read-only mode; check -storage.minFreeDiskSpaceBytes command-line flag value")
@@ -309,9 +308,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
return true
}
if prometheusCompatibleResponse {
fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotPath))
fmt.Fprintf(w, `{"status":"success","data":{"name":%q}}`, snapshotPath)
} else {
fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotPath))
fmt.Fprintf(w, `{"status":"ok","snapshot":%q}`, snapshotPath)
}
return true
case "/list":
@@ -637,6 +636,5 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
func jsonResponseError(w http.ResponseWriter, err error) {
logger.Errorf("%s", err)
w.WriteHeader(http.StatusInternalServerError)
errStr := err.Error()
fmt.Fprintf(w, `{"status":"error","msg":%s}`, stringsutil.JSONString(errStr))
fmt.Fprintf(w, `{"status":"error","msg":%q}`, err)
}

View File

@@ -1,4 +1,4 @@
FROM golang:1.23.0 AS build-web-stage
FROM golang:1.22.5 AS build-web-stage
COPY build /build
WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
FROM alpine:3.20.2
FROM alpine:3.20.1
USER root
COPY --from=build-web-stage /build/web-amd64 /app/web

View File

@@ -1,22 +1,6 @@
/* eslint-disable */
const { override, addExternalBabelPlugin, addWebpackAlias, addWebpackPlugin } = require("customize-cra");
const webpack = require("webpack");
const fs = require('fs');
const path = require('path');
// This will replace the default check
const pathIndexHTML = (() => {
switch (process.env.REACT_APP_TYPE) {
case 'logs':
return 'public/victorialogs.html';
case 'anomaly':
return 'public/vmanomaly.html';
default:
return 'public/victoriametrics.html';
}
})();
const fileContent = fs.readFileSync(path.resolve(__dirname, pathIndexHTML), 'utf8');
fs.writeFileSync(path.resolve(__dirname, 'public/index.html'), fileContent);
module.exports = override(
addExternalBabelPlugin("@babel/plugin-proposal-nullish-coalescing-operator"),

View File

@@ -0,0 +1,56 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5" />
<meta name="theme-color" content="#000000" />
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png" />
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -1,55 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -1,56 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -1,56 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
<meta name="theme-color" content="#000000"/>
<meta
name="description"
content="UI for VictoriaMetrics"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>VM UI</title>
<script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
<meta name="twitter:title" content="UI for VictoriaMetrics">
<meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta name="twitter:site" content="@VictoriaMetrics">
<meta property="og:title" content="Metric explorer for VictoriaMetrics">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
<meta property="og:type" content="website">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -7,7 +7,7 @@ import ExploreAnomaly from "./pages/ExploreAnomaly/ExploreAnomaly";
import router from "./router";
import CustomPanel from "./pages/CustomPanel";
const AppAnomaly: FC = () => {
const AppLogs: FC = () => {
const [loadedTheme, setLoadedTheme] = useState(false);
return <>
@@ -38,4 +38,4 @@ const AppAnomaly: FC = () => {
</>;
};
export default AppAnomaly;
export default AppLogs;

View File

@@ -38,7 +38,6 @@ export interface Logs {
export interface LogHits {
timestamps: string[];
values: number[];
total?: number;
fields: {
[key: string]: string;
};

View File

@@ -1,4 +1,5 @@
---
sort: 23
weight: 23
title: MetricsQL
menu:
@@ -9,6 +10,9 @@ aliases:
- /ExtendedPromQL.html
- /MetricsQL.html
---
# MetricsQL
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -216,11 +220,9 @@ See also [descent_over_time](#descent_over_time).
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
See also [median_over_time](#median_over_time).
#### changes
@@ -260,8 +262,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
#### count_gt_over_time
@@ -272,8 +272,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
#### count_le_over_time
@@ -284,8 +282,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
#### count_ne_over_time
@@ -296,9 +292,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
See also [count_over_time](#count_over_time).
#### count_over_time
@@ -319,8 +313,6 @@ The results are calculated independently per each time series returned from the
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
#### decreases_over_time
@@ -440,8 +432,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### histogram_over_time
`histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -453,15 +443,11 @@ For example, the following query calculates median temperature by country over t
`histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### hoeffding_bound_lower
`hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [hoeffding_bound_upper](#hoeffding_bound_upper).
#### hoeffding_bound_upper
@@ -469,8 +455,6 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
`hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [hoeffding_bound_lower](#hoeffding_bound_lower).
#### holt_winters
@@ -478,9 +462,8 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
Both `sf` and `tf` must be in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
@@ -512,14 +495,13 @@ See also [deriv](#deriv).
`increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
This function is supported by PromQL.
See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -528,13 +510,12 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
`increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [increase_pure](#increase_pure) and [increase](#increase).
#### increase_pure
@@ -543,10 +524,6 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
while [increase](#increase) ignores the first value in a series if it is too big.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [increase](#increase) and [increase_prometheus](#increase_prometheus).
#### increases_over_time
`increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -563,18 +540,15 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### irate
`irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
This function is supported by PromQL.
See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -613,8 +587,6 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
#### max_over_time
@@ -622,11 +594,9 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
`max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
See also [tmax_over_time](#tmax_over_time).
#### median_over_time
@@ -634,8 +604,6 @@ See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
on the given lookbehind window `d` per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [avg_over_time](#avg_over_time).
#### min_over_time
@@ -643,11 +611,9 @@ See also [avg_over_time](#avg_over_time).
`min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
See also [tmin_over_time](#tmin_over_time).
#### mode_over_time
@@ -656,8 +622,6 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values are discrete.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### outlier_iqr_over_time
`outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -668,8 +632,6 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [outliers_iqr](#outliers_iqr).
#### predict_linear
@@ -697,8 +659,6 @@ This function is supported by PromQL.
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
The `phi` value must be in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [quantiles_over_time](#quantiles_over_time).
@@ -710,8 +670,6 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [quantile_over_time](#quantile_over_time).
#### range_over_time
@@ -722,12 +680,11 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### rate
`rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -746,8 +703,6 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
on the given lookbehind window `d`. The calculations are performed individually per each time series returned
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
#### resets
@@ -755,11 +710,10 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
`resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
This function is supported by PromQL.
#### rollup
@@ -771,10 +725,6 @@ These values are calculated individually per each time series returned from the
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
See also [label_match](#label_match).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [rollup_rate](#rollup_rate).
#### rollup_candlestick
`rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -786,8 +736,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
See also [label_match](#label_match).
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### rollup_delta
`rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -814,8 +762,6 @@ See also [label_match](#label_match).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
#### rollup_increase
`rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -828,17 +774,12 @@ See also [label_match](#label_match).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
#### rollup_rate
`rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
when to use `rollup_rate()`.
@@ -846,12 +787,10 @@ when to use `rollup_rate()`.
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
See also [label_match](#label_match).
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
#### rollup_scrape_interval
`rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -885,8 +824,6 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
#### share_le_over_time
@@ -901,8 +838,6 @@ the share of time series values for the last 24 hours when memory usage was belo
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
#### share_eq_over_time
@@ -914,8 +849,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [count_eq_over_time](#count_eq_over_time).
#### stale_samples_over_time
@@ -933,8 +866,6 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [stdvar_over_time](#stdvar_over_time).
@@ -946,43 +877,35 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
See also [stddev_over_time](#stddev_over_time).
#### sum_eq_over_time
`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values equal to `eq` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
#### sum_gt_over_time
`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values bigger than `gt` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
#### sum_le_over_time
`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
values smaller or equal to `le` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
#### sum_over_time
@@ -992,8 +915,6 @@ on the given lookbehind window `d` per each time series returned from the given
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
This function is supported by PromQL.
#### sum2_over_time
@@ -1003,8 +924,6 @@ values on the given lookbehind window `d` per each time series returned from the
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
#### timestamp
`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1082,8 +1001,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
@@ -2313,8 +2230,7 @@ Any [rollup function](#rollup-functions) for something other than [series select
Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
VictoriaMetrics performs subqueries in the following way:
@@ -2347,6 +2263,5 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
* If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.

View File

@@ -1,66 +1,32 @@
import React, { FC, useMemo, useRef, useState } from "preact/compat";
import React, { FC, useRef, useState } from "preact/compat";
import "./style.scss";
import "uplot/dist/uPlot.min.css";
import useElementSize from "../../../hooks/useElementSize";
import uPlot, { AlignedData } from "uplot";
import { useEffect } from "react";
import useBarHitsOptions from "./hooks/useBarHitsOptions";
import BarHitsTooltip from "./BarHitsTooltip/BarHitsTooltip";
import TooltipBarHitsChart from "./TooltipBarHitsChart";
import { TimeParams } from "../../../types";
import usePlotScale from "../../../hooks/uplot/usePlotScale";
import useReadyChart from "../../../hooks/uplot/useReadyChart";
import useZoomChart from "../../../hooks/uplot/useZoomChart";
import classNames from "classnames";
import { LogHits } from "../../../api/types";
import { addSeries, delSeries, setBand } from "../../../utils/uplot";
import { GraphOptions, GRAPH_STYLES } from "./types";
import BarHitsOptions from "./BarHitsOptions/BarHitsOptions";
import stack from "../../../utils/uplot/stack";
import BarHitsLegend from "./BarHitsLegend/BarHitsLegend";
interface Props {
logHits: LogHits[];
data: AlignedData;
period: TimeParams;
setPeriod: ({ from, to }: { from: Date, to: Date }) => void;
onApplyFilter: (value: string) => void;
}
const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onApplyFilter }) => {
const BarHitsChart: FC<Props> = ({ data, period, setPeriod }) => {
const [containerRef, containerSize] = useElementSize();
const uPlotRef = useRef<HTMLDivElement>(null);
const [uPlotInst, setUPlotInst] = useState<uPlot>();
const [graphOptions, setGraphOptions] = useState<GraphOptions>({
graphStyle: GRAPH_STYLES.LINE_STEPPED,
stacked: false,
fill: false,
});
const { xRange, setPlotScale } = usePlotScale({ period, setPeriod });
const { onReadyChart, isPanning } = useReadyChart(setPlotScale);
useZoomChart({ uPlotInst, xRange, setPlotScale });
const { data, bands } = useMemo(() => {
return graphOptions.stacked ? stack(_data, () => false) : { data: _data, bands: [] };
}, [graphOptions, _data]);
const { options, series, focusDataIdx } = useBarHitsOptions({
data,
logHits,
bands,
xRange,
containerSize,
onReadyChart,
setPlotScale,
graphOptions
});
useEffect(() => {
if (!uPlotInst) return;
delSeries(uPlotInst);
addSeries(uPlotInst, series, true);
setBand(uPlotInst, series);
uPlotInst.redraw();
}, [series]);
const { options, focusDataIdx } = useBarHitsOptions({ xRange, containerSize, onReadyChart, setPlotScale });
useEffect(() => {
if (!uPlotRef.current) return;
@@ -88,31 +54,21 @@ const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onAp
}, [data]);
return (
<div className="vm-bar-hits-chart__wrapper">
<div
className={classNames({
"vm-bar-hits-chart": true,
"vm-bar-hits-chart_panning": isPanning
})}
ref={containerRef}
>
<div
className={classNames({
"vm-bar-hits-chart": true,
"vm-bar-hits-chart_panning": isPanning
})}
ref={containerRef}
>
<div
className="vm-line-chart__u-plot"
ref={uPlotRef}
/>
<BarHitsTooltip
uPlotInst={uPlotInst}
data={_data}
focusDataIdx={focusDataIdx}
/>
</div>
<BarHitsOptions onChange={setGraphOptions}/>
{uPlotInst && (
<BarHitsLegend
uPlotInst={uPlotInst}
onApplyFilter={onApplyFilter}
/>
)}
className="vm-line-chart__u-plot"
ref={uPlotRef}
/>
<TooltipBarHitsChart
uPlotInst={uPlotInst}
focusDataIdx={focusDataIdx}
/>
</div>
);
};

View File

@@ -1,68 +0,0 @@
import React, { FC, useCallback, useEffect, useState } from "preact/compat";
import uPlot, { Series } from "uplot";
import "./style.scss";
import "../../Line/Legend/style.scss";
import classNames from "classnames";
import { MouseEvent } from "react";
import { isMacOs } from "../../../../utils/detect-device";
import Tooltip from "../../../Main/Tooltip/Tooltip";
interface Props {
uPlotInst: uPlot;
onApplyFilter: (value: string) => void;
}
const BarHitsLegend: FC<Props> = ({ uPlotInst, onApplyFilter }) => {
const [series, setSeries] = useState<Series[]>([]);
const updateSeries = useCallback(() => {
const series = uPlotInst.series.filter(s => s.scale !== "x");
setSeries(series);
}, [uPlotInst]);
const handleClick = (target: Series) => (e: MouseEvent<HTMLDivElement>) => {
const metaKey = e.metaKey || e.ctrlKey;
if (!metaKey) {
target.show = !target.show;
} else {
onApplyFilter(target.label || "");
}
updateSeries();
uPlotInst.redraw();
};
useEffect(updateSeries, [uPlotInst]);
return (
<div className="vm-bar-hits-legend">
{series.map(s => (
<Tooltip
key={s.label}
title={(
<ul className="vm-bar-hits-legend-info">
<li>Click to {s.show ? "hide" : "show"} the _stream.</li>
<li>{isMacOs() ? "Cmd" : "Ctrl"} + Click to filter by the _stream.</li>
</ul>
)}
>
<div
className={classNames({
"vm-bar-hits-legend-item": true,
"vm-bar-hits-legend-item_hide": !s.show,
})}
onClick={handleClick(s)}
>
<div
className="vm-bar-hits-legend-item__marker"
style={{ backgroundColor: `${(s?.stroke as () => string)?.()}` }}
/>
<div>{s.label}</div>
</div>
</Tooltip>
))}
</div>
);
};
export default BarHitsLegend;

View File

@@ -1,35 +0,0 @@
@use "src/styles/variables" as *;
.vm-bar-hits-legend {
display: flex;
flex-wrap: wrap;
gap: 0;
padding: 0 $padding-small $padding-small;
&-item {
display: grid;
grid-template-columns: auto 1fr;
align-items: center;
gap: 4px;
font-size: 12px;
padding: $padding-small;
border-radius: $border-radius-small;
cursor: pointer;
transition: 0.2s;
&:hover {
background-color: rgba(0, 0, 0, 0.05);
}
&_hide {
text-decoration: line-through;
opacity: 0.5;
}
&__marker {
width: 14px;
height: 14px;
border: $color-background-block;
}
}
}

View File

@@ -1,116 +0,0 @@
import React, { FC, useEffect, useMemo, useRef } from "preact/compat";
import { GraphOptions, GRAPH_STYLES } from "../types";
import Switch from "../../../Main/Switch/Switch";
import "./style.scss";
import useStateSearchParams from "../../../../hooks/useStateSearchParams";
import { useSearchParams } from "react-router-dom";
import Button from "../../../Main/Button/Button";
import classNames from "classnames";
import { SettingsIcon } from "../../../Main/Icons";
import Tooltip from "../../../Main/Tooltip/Tooltip";
import Popper from "../../../Main/Popper/Popper";
import useBoolean from "../../../../hooks/useBoolean";
interface Props {
onChange: (options: GraphOptions) => void;
}
const BarHitsOptions: FC<Props> = ({ onChange }) => {
const [searchParams, setSearchParams] = useSearchParams();
const optionsButtonRef = useRef<HTMLDivElement>(null);
const {
value: openOptions,
toggle: toggleOpenOptions,
setFalse: handleCloseOptions,
} = useBoolean(false);
const [graphStyle, setGraphStyle] = useStateSearchParams(GRAPH_STYLES.LINE_STEPPED, "graph");
const [stacked, setStacked] = useStateSearchParams(false, "stacked");
const [fill, setFill] = useStateSearchParams(false, "fill");
const options: GraphOptions = useMemo(() => ({
graphStyle,
stacked,
fill,
}), [graphStyle, stacked, fill]);
const handleChangeGraphStyle = (val: string) => () => {
setGraphStyle(val as GRAPH_STYLES);
searchParams.set("graph", val);
setSearchParams(searchParams);
};
const handleChangeFill = (val: boolean) => {
setFill(val);
val ? searchParams.set("fill", "true") : searchParams.delete("fill");
setSearchParams(searchParams);
};
const handleChangeStacked = (val: boolean) => {
setStacked(val);
val ? searchParams.set("stacked", "true") : searchParams.delete("stacked");
setSearchParams(searchParams);
};
useEffect(() => {
onChange(options);
}, [options]);
return (
<div
className="vm-bar-hits-options"
ref={optionsButtonRef}
>
<Tooltip title="Graph settings">
<Button
variant="text"
color="primary"
startIcon={<SettingsIcon/>}
onClick={toggleOpenOptions}
ariaLabel="settings"
/>
</Tooltip>
<Popper
open={openOptions}
placement="bottom-right"
onClose={handleCloseOptions}
buttonRef={optionsButtonRef}
title={"Graph settings"}
>
<div className="vm-bar-hits-options-settings">
<div className="vm-bar-hits-options-settings-item vm-bar-hits-options-settings-item_list">
<p className="vm-bar-hits-options-settings-item__title">Graph style:</p>
{Object.values(GRAPH_STYLES).map(style => (
<div
key={style}
className={classNames({
"vm-list-item": true,
"vm-list-item_active": graphStyle === style,
})}
onClick={handleChangeGraphStyle(style)}
>
{style}
</div>
))}
</div>
<div className="vm-bar-hits-options-settings-item">
<Switch
label={"Stacked"}
value={stacked}
onChange={handleChangeStacked}
/>
</div>
<div className="vm-bar-hits-options-settings-item">
<Switch
label={"Fill"}
value={fill}
onChange={handleChangeFill}
/>
</div>
</div>
</Popper>
</div>
);
};
export default BarHitsOptions;

View File

@@ -1,35 +0,0 @@
@use "src/styles/variables" as *;
.vm-bar-hits-options {
position: absolute;
top: $padding-small;
right: $padding-small;
z-index: 2;
&-settings {
display: grid;
align-items: flex-start;
gap: $padding-global;
min-width: 200px;
&-item {
border-bottom: $border-divider;
padding: 0 $padding-global $padding-global;
&_list {
padding: 0;
}
&__title {
font-size: $font-size-small;
color: $color-text-secondary;
padding: 0 $padding-small $padding-small;
}
&:last-child {
border-bottom: none;
}
}
}
}

View File

@@ -1,125 +0,0 @@
import React, { FC, useMemo, useRef } from "preact/compat";
import uPlot, { AlignedData } from "uplot";
import dayjs from "dayjs";
import { DATE_TIME_FORMAT } from "../../../../constants/date";
import classNames from "classnames";
import "./style.scss";
import "../../ChartTooltip/style.scss";
interface Props {
data: AlignedData;
uPlotInst?: uPlot;
focusDataIdx: number;
}
const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
const tooltipRef = useRef<HTMLDivElement>(null);
const tooltipData = useMemo(() => {
const series = uPlotInst?.series || [];
const [time, ...values] = data.map((d) => d[focusDataIdx] || 0);
const tooltipItems = values.map((value, i) => {
const targetSeries = series[i + 1];
const stroke = (targetSeries?.stroke as () => string)?.();
const label = targetSeries?.label || "other";
const show = targetSeries?.show;
return {
label,
stroke,
value,
show
};
}).filter(item => item.value > 0 && item.show).sort((a, b) => b.value - a.value);
const point = {
top: tooltipItems[0] ? uPlotInst?.valToPos?.(tooltipItems[0].value, "y") || 0 : 0,
left: uPlotInst?.valToPos?.(time, "x") || 0,
};
return {
point,
values: tooltipItems,
total: tooltipItems.reduce((acc, item) => acc + item.value, 0),
timestamp: dayjs(time * 1000).tz().format(DATE_TIME_FORMAT),
};
}, [focusDataIdx, uPlotInst, data]);
const tooltipPosition = useMemo(() => {
if (!uPlotInst || !tooltipData.total || !tooltipRef.current) return;
const { top, left } = tooltipData.point;
const uPlotPosition = {
left: parseFloat(uPlotInst.over.style.left),
top: parseFloat(uPlotInst.over.style.top)
};
const {
width: uPlotWidth,
height: uPlotHeight
} = uPlotInst.over.getBoundingClientRect();
const {
width: tooltipWidth,
height: tooltipHeight
} = tooltipRef.current.getBoundingClientRect();
const margin = 50;
const overflowX = left + tooltipWidth >= uPlotWidth ? tooltipWidth + (2 * margin) : 0;
const overflowY = top + tooltipHeight >= uPlotHeight ? tooltipHeight + (2 * margin) : 0;
const position = {
top: top + uPlotPosition.top + margin - overflowY,
left: left + uPlotPosition.left + margin - overflowX
};
if (position.left < 0) position.left = 20;
if (position.top < 0) position.top = 20;
return position;
}, [tooltipData, uPlotInst, tooltipRef.current]);
return (
<div
className={classNames({
"vm-chart-tooltip": true,
"vm-chart-tooltip_hits": true,
"vm-bar-hits-tooltip": true,
"vm-bar-hits-tooltip_visible": focusDataIdx !== -1 && tooltipData.values.length
})}
ref={tooltipRef}
style={tooltipPosition}
>
<div>
{tooltipData.values.map((item, i) => (
<div
className="vm-chart-tooltip-data"
key={i}
>
<span
className="vm-chart-tooltip-data__marker"
style={{ background: item.stroke }}
/>
<p>
{item.label}: <b>{item.value}</b>
</p>
</div>
))}
</div>
{tooltipData.values.length > 1 && (
<div className="vm-chart-tooltip-data">
<p>
Total records: <b>{tooltipData.total}</b>
</p>
</div>
)}
<div className="vm-chart-tooltip-header">
<div className="vm-chart-tooltip-header__title">
{tooltipData.timestamp}
</div>
</div>
</div>
);
};
export default BarHitsTooltip;

View File

@@ -1,12 +0,0 @@
@use "src/styles/variables" as *;
.vm-bar-hits-tooltip {
opacity: 0;
pointer-events: none;
gap: $padding-small;
&_visible {
opacity: 1;
pointer-events: auto;
}
}

View File

@@ -0,0 +1,89 @@
import React, { FC, useMemo, useRef } from "preact/compat";
import uPlot from "uplot";
import dayjs from "dayjs";
import { DATE_TIME_FORMAT } from "../../../constants/date";
import classNames from "classnames";
import "./style.scss";
import "../../../components/Chart/ChartTooltip/style.scss";
interface Props {
uPlotInst?: uPlot;
focusDataIdx: number
}
const TooltipBarHitsChart: FC<Props> = ({ focusDataIdx, uPlotInst }) => {
const tooltipRef = useRef<HTMLDivElement>(null);
const tooltipData = useMemo(() => {
const value = uPlotInst?.data?.[1]?.[focusDataIdx];
const timestamp = uPlotInst?.data?.[0]?.[focusDataIdx] || 0;
const top = uPlotInst?.valToPos?.((value || 0), "y") || 0;
const left = uPlotInst?.valToPos?.(timestamp, "x") || 0;
return {
point: { top, left },
value,
timestamp: dayjs(timestamp * 1000).tz().format(DATE_TIME_FORMAT),
};
}, [focusDataIdx, uPlotInst]);
const tooltipPosition = useMemo(() => {
if (!uPlotInst || !tooltipData.value || !tooltipRef.current) return;
const { top, left } = tooltipData.point;
const uPlotPosition = {
left: parseFloat(uPlotInst.over.style.left),
top: parseFloat(uPlotInst.over.style.top)
};
const {
width: uPlotWidth,
height: uPlotHeight
} = uPlotInst.over.getBoundingClientRect();
const {
width: tooltipWidth,
height: tooltipHeight
} = tooltipRef.current.getBoundingClientRect();
const margin = 10;
const overflowX = left + tooltipWidth >= uPlotWidth ? tooltipWidth + (2 * margin) : 0;
const overflowY = top + tooltipHeight >= uPlotHeight ? tooltipHeight + (2 * margin) : 0;
const position = {
top: top + uPlotPosition.top + margin - overflowY,
left: left + uPlotPosition.left + margin - overflowX
};
if (position.left < 0) position.left = 20;
if (position.top < 0) position.top = 20;
return position;
}, [tooltipData, uPlotInst, tooltipRef.current]);
return (
<div
className={classNames({
"vm-chart-tooltip": true,
"vm-bar-hits-chart-tooltip": true,
"vm-bar-hits-chart-tooltip_visible": focusDataIdx !== -1
})}
ref={tooltipRef}
style={tooltipPosition}
>
<div className="vm-chart-tooltip-data">
Count of records:
<p className="vm-chart-tooltip-data__value">
<b>{tooltipData.value}</b>
</p>
</div>
<div className="vm-chart-tooltip-header">
<div className="vm-chart-tooltip-header__title">
{tooltipData.timestamp}
</div>
</div>
</div>
);
};
export default TooltipBarHitsChart;

View File

@@ -2,81 +2,42 @@ import { useMemo, useState } from "preact/compat";
import { getAxes, handleDestroy, setSelect } from "../../../../utils/uplot";
import dayjs from "dayjs";
import { dateFromSeconds, formatDateForNativeInput } from "../../../../utils/time";
import uPlot, { AlignedData, Band, Options, Series } from "uplot";
import uPlot, { Options } from "uplot";
import { getCssVariable } from "../../../../utils/theme";
import { barPaths } from "../../../../utils/uplot/bars";
import { useAppState } from "../../../../state/common/StateContext";
import { MinMax, SetMinMax } from "../../../../types";
import { LogHits } from "../../../../api/types";
import getSeriesPaths from "../../../../utils/uplot/paths";
import { GraphOptions, GRAPH_STYLES } from "../types";
const seriesColors = [
"color-log-hits-bar-1",
"color-log-hits-bar-2",
"color-log-hits-bar-3",
"color-log-hits-bar-4",
"color-log-hits-bar-5",
];
const strokeWidth = {
[GRAPH_STYLES.BAR]: 0.8,
[GRAPH_STYLES.LINE_STEPPED]: 1.2,
[GRAPH_STYLES.LINE]: 1.2,
[GRAPH_STYLES.POINTS]: 0,
};
interface UseGetBarHitsOptionsArgs {
data: AlignedData;
logHits: LogHits[];
xRange: MinMax;
bands?: Band[];
containerSize: { width: number, height: number };
setPlotScale: SetMinMax;
onReadyChart: (u: uPlot) => void;
graphOptions: GraphOptions;
}
const useBarHitsOptions = ({
data,
logHits,
xRange,
bands,
containerSize,
onReadyChart,
setPlotScale,
graphOptions
}: UseGetBarHitsOptionsArgs) => {
const useBarHitsOptions = ({ xRange, containerSize, onReadyChart, setPlotScale }: UseGetBarHitsOptionsArgs) => {
const { isDarkTheme } = useAppState();
const [focusDataIdx, setFocusDataIdx] = useState(-1);
const series = useMemo(() => [
{},
{
label: "y",
width: 1,
stroke: getCssVariable("color-log-hits-bar"),
fill: getCssVariable("color-log-hits-bar"),
paths: barPaths,
}
], [isDarkTheme]);
const setCursor = (u: uPlot) => {
const dataIdx = u.cursor.idx ?? -1;
setFocusDataIdx(dataIdx);
};
const series: Series[] = useMemo(() => {
let colorN = 0;
return data.map((_d, i) => {
if (i === 0) return {}; // 0 index is xAxis(timestamps)
const fields = Object.values(logHits?.[i - 1]?.fields || {});
const label = fields.map((value) => value || "\"\"").join(", ");
const color = getCssVariable(label ? seriesColors[colorN] : "color-log-hits-bar-0");
if (label) colorN++;
return {
label: label || "other",
width: strokeWidth[graphOptions.graphStyle],
spanGaps: true,
stroke: color,
fill: graphOptions.fill ? color + "80" : "",
paths: getSeriesPaths(graphOptions.graphStyle),
};
});
}, [isDarkTheme, data, graphOptions]);
const options: Options = useMemo(() => ({
series,
bands,
width: containerSize.width || (window.innerWidth / 2),
height: containerSize.height || 200,
cursor: {
@@ -94,7 +55,6 @@ const useBarHitsOptions = ({
}
},
hooks: {
drawSeries: [],
ready: [onReadyChart],
setCursor: [setCursor],
setSelect: [setSelect(setPlotScale)],
@@ -103,11 +63,10 @@ const useBarHitsOptions = ({
legend: { show: false },
axes: getAxes([{}, { scale: "y" }]),
tzDate: ts => dayjs(formatDateForNativeInput(dateFromSeconds(ts))).local().toDate(),
}), [isDarkTheme, series, bands]);
}), [isDarkTheme]);
return {
options,
series,
focusDataIdx,
};
};

View File

@@ -1,18 +1,22 @@
@use "src/styles/variables" as *;
.vm-bar-hits-chart {
position: relative;
height: 100%;
width: 100%;
height: 200px;
&__wrapper {
display: flex;
flex-direction: column;
width: 100%;
height: 100%;
}
&_panning {
pointer-events: none;
}
&-tooltip {
opacity: 0;
pointer-events: none;
width: 240px;
gap: $padding-small;
&_visible {
opacity: 1;
pointer-events: auto;
}
}
}

View File

@@ -1,12 +0,0 @@
export enum GRAPH_STYLES {
BAR = "Bars",
LINE = "Lines",
LINE_STEPPED = "Stepped lines",
POINTS = "Points",
}
export interface GraphOptions {
graphStyle: GRAPH_STYLES;
stacked: boolean;
fill: boolean;
}

View File

@@ -25,12 +25,6 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
user-select: text;
pointer-events: none;
&_hits {
white-space: pre-wrap;
word-break: break-all;
width: auto;
}
&_sticky {
pointer-events: auto;
z-index: 99;
@@ -80,22 +74,10 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
justify-content: flex-start;
gap: $padding-small;
&_margin-bottom {
margin-bottom: $padding-global;
}
&_margin-top {
margin-top: $padding-global;
}
&__marker {
width: $font-size;
height: $font-size;
border: 1px solid rgba($color-white, 0.5);
&_tranparent {
opacity: 0;
}
}
&__value {

View File

@@ -13,7 +13,6 @@ import ThemeControl from "../ThemeControl/ThemeControl";
import useDeviceDetect from "../../../hooks/useDeviceDetect";
import useBoolean from "../../../hooks/useBoolean";
import { AppType } from "../../../types/appType";
import SwitchMarkdownParsing from "../LogsSettings/MarkdownParsing/SwitchMarkdownParsing";
const title = "Settings";
@@ -61,10 +60,6 @@ const GlobalSettings: FC = () => {
onClose={handleClose}
/>
},
{
show: isLogsApp,
component: <SwitchMarkdownParsing/>
},
{
show: true,
component: <Timezones ref={timezoneSettingRef}/>

View File

@@ -1,150 +0,0 @@
import React, { FC, useRef } from "preact/compat";
import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
import { ArrowDownIcon, QuestionIcon, StorageIcon } from "../../../Main/Icons";
import Button from "../../../Main/Button/Button";
import "./style.scss";
import "../../TimeRangeSettings/ExecutionControls/style.scss";
import classNames from "classnames";
import Popper from "../../../Main/Popper/Popper";
import { getAppModeEnable } from "../../../../utils/app-mode";
import Tooltip from "../../../Main/Tooltip/Tooltip";
import useDeviceDetect from "../../../../hooks/useDeviceDetect";
import TextField from "../../../Main/TextField/TextField";
import useBoolean from "../../../../hooks/useBoolean";
import useStateSearchParams from "../../../../hooks/useStateSearchParams";
import { useSearchParams } from "react-router-dom";
import { useEffect } from "react";
const TenantsFields: FC = () => {
const appModeEnable = getAppModeEnable();
const { isMobile } = useDeviceDetect();
const timeDispatch = useTimeDispatch();
const [searchParams, setSearchParams] = useSearchParams();
const [accountID, setAccountID] = useStateSearchParams("0", "accountID");
const [projectID, setProjectID] = useStateSearchParams("0", "projectID");
const formattedTenant = `${accountID}:${projectID}`;
const buttonRef = useRef<HTMLDivElement>(null);
const {
value: openPopup,
toggle: toggleOpenPopup,
setFalse: handleClosePopup,
} = useBoolean(false);
const applyChanges = () => {
searchParams.set("accountID", accountID);
searchParams.set("projectID", projectID);
setSearchParams(searchParams);
handleClosePopup();
timeDispatch({ type: "RUN_QUERY" });
};
const handleReset = () => {
setAccountID(searchParams.get("accountID") || "0");
setProjectID(searchParams.get("projectID") || "0");
};
useEffect(() => {
if (openPopup) return;
handleReset();
}, [openPopup]);
return (
<div className="vm-tenant-input">
<Tooltip title="Define Tenant ID if you need request to another storage">
<div ref={buttonRef}>
{isMobile ? (
<div
className="vm-mobile-option"
onClick={toggleOpenPopup}
>
<span className="vm-mobile-option__icon"><StorageIcon/></span>
<div className="vm-mobile-option-text">
<span className="vm-mobile-option-text__label">Tenant ID</span>
<span className="vm-mobile-option-text__value">{formattedTenant}</span>
</div>
<span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
</div>
) : (
<Button
className={appModeEnable ? "" : "vm-header-button"}
variant="contained"
color="primary"
fullWidth
startIcon={<StorageIcon/>}
endIcon={(
<div
className={classNames({
"vm-execution-controls-buttons__arrow": true,
"vm-execution-controls-buttons__arrow_open": openPopup,
})}
>
<ArrowDownIcon/>
</div>
)}
onClick={toggleOpenPopup}
>
{formattedTenant}
</Button>
)}
</div>
</Tooltip>
<Popper
open={openPopup}
placement="bottom-right"
onClose={handleClosePopup}
buttonRef={buttonRef}
title={isMobile ? "Define Tenant ID" : undefined}
>
<div
className={classNames({
"vm-list vm-tenant-input-list": true,
"vm-list vm-tenant-input-list_mobile": isMobile,
"vm-tenant-input-list_inline": true,
})}
>
<TextField
autofocus
label="accountID"
value={accountID}
onChange={setAccountID}
type="number"
/>
<TextField
autofocus
label="projectID"
value={projectID}
onChange={setProjectID}
type="number"
/>
<div className="vm-tenant-input-list__buttons">
<Tooltip title="Multitenancy in VictoriaLogs documentation">
<a
href="https://docs.victoriametrics.com/victorialogs/#multitenancy"
target="_blank"
rel="help noreferrer"
>
<Button
variant="text"
color="gray"
startIcon={<QuestionIcon/>}
/>
</a>
</Tooltip>
<Button
variant="contained"
color="primary"
onClick={applyChanges}
>
Apply
</Button>
</div>
</div>
</Popper>
</div>
);
};
export default TenantsFields;

View File

@@ -17,23 +17,11 @@
padding: 0 $padding-global $padding-small;
}
&_inline {
display: grid;
gap: calc($padding-small/2);
padding: $padding-global;
}
&__search {
position: sticky;
top: 0;
padding: $padding-small $padding-global;
background-color: $color-background-block;
}
&__buttons {
display: flex;
justify-content: space-between;
gap: $padding-small;
}
}
}

View File

@@ -4,7 +4,7 @@
display: flex;
flex-direction: column;
align-items: center;
gap: $padding-large;
gap: $padding-medium;
width: 600px;
padding-bottom: $padding-medium;
@@ -39,13 +39,6 @@
margin-bottom: $padding-global;
}
&__info {
padding-top: $padding-small;
font-size: $font-size-small;
color: $color-text-secondary;
line-height: 130%;
}
&-url {
display: flex;
align-items: flex-start;

View File

@@ -1,35 +0,0 @@
import React, { FC } from "preact/compat";
import Switch from "../../../Main/Switch/Switch";
import useDeviceDetect from "../../../../hooks/useDeviceDetect";
import { useLogsDispatch, useLogsState } from "../../../../state/logsPanel/LogsStateContext";
const SwitchMarkdownParsing: FC = () => {
const { isMobile } = useDeviceDetect();
const { markdownParsing } = useLogsState();
const dispatch = useLogsDispatch();
const handleChangeMarkdownParsing = (val: boolean) => {
dispatch({ type: "SET_MARKDOWN_PARSING", payload: val });
};
return (
<div>
<div className="vm-server-configurator__title">
Markdown Parsing for Logs
</div>
<Switch
label={markdownParsing ? "Disable markdown parsing" : "Enable markdown parsing"}
value={markdownParsing}
onChange={handleChangeMarkdownParsing}
fullWidth={isMobile}
/>
<div className="vm-server-configurator__info">
Toggle this switch to enable or disable the Markdown formatting for log entries.
Enabling this will parse log texts to Markdown.
</div>
</div>
);
};
export default SwitchMarkdownParsing;

View File

@@ -34,25 +34,14 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
}, [value, caretPosition]);
const exprLastPart = useMemo(() => {
const regexpSplit = /\s(or|and|unless|default|ifnot|if|group_left|group_right)\s|}|\+|\|-|\*|\/|\^/i;
const parts = values.beforeCursor.split(regexpSplit);
const parts = values.beforeCursor.split("}");
return parts[parts.length - 1];
}, [values]);
const metric = useMemo(() => {
const regex1 = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
const matchAlt = [...exprLastPart.matchAll(regex1)];
if (matchAlt.length > 0 && matchAlt[0].groups && matchAlt[0].groups.metricName) {
return matchAlt[0].groups.metricName;
}
const regex2 = /^\s*\b(?<metricName>[^{}(),\s]+)(?={|$)/g;
const match = [...exprLastPart.matchAll(regex2)];
if (match.length > 0 && match[0].groups && match[0].groups.metricName) {
return match[0].groups.metricName;
}
return "";
const regexp = /\b[^{}(),\s]+(?={|$)/g;
const match = exprLastPart.match(regexp);
return match ? match[0] : "";
}, [exprLastPart]);
const label = useMemo(() => {
@@ -62,7 +51,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
}, [exprLastPart]);
const shouldSuppressAutoSuggestion = (value: string) => {
const pattern = /([{(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right|by|without|on|ignoring)\b)/i;
const pattern = /([{(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right)\b)/;
const parts = value.split(/\s+/);
const partsCount = parts.length;
const lastPart = parts[partsCount - 1];
@@ -74,16 +63,12 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
};
const context = useMemo(() => {
const valueBeforeCursor = values.beforeCursor.trim();
const endOfClosedBrackets = ["}", ")"].some(char => valueBeforeCursor.endsWith(char));
const endOfClosedQuotes = !hasUnclosedQuotes(valueBeforeCursor) && ["`", "'", "\""].some(char => valueBeforeCursor.endsWith(char));
if (!values.beforeCursor || endOfClosedBrackets || endOfClosedQuotes || shouldSuppressAutoSuggestion(values.beforeCursor)) {
if (!values.beforeCursor || values.beforeCursor.endsWith("}") || shouldSuppressAutoSuggestion(values.beforeCursor)) {
return QueryContextType.empty;
}
const labelRegexp = /(?:by|without|on|ignoring)\s*\(\s*[^)]*$|\{[^}]*$/i;
const patternLabelValue = `(${escapeRegexp(metric)})?{?.+${escapeRegexp(label)}(=|!=|=~|!~)"?([^"]*)$`;
const labelValueRegexp = new RegExp(patternLabelValue, "g");
const labelRegexp = /\{[^}]*$/;
const labelValueRegexp = new RegExp(`(${escapeRegexp(metric)})?{?.+${escapeRegexp(label)}(=|!=|=~|!~)"?([^"]*)$`, "g");
switch (true) {
case labelValueRegexp.test(values.beforeCursor):
@@ -96,7 +81,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
}, [values, metric, label]);
const valueByContext = useMemo(() => {
const wordMatch = values.beforeCursor.match(/([\w_.:]+(?![},]))$/);
const wordMatch = values.beforeCursor.match(/([\w_\-.:/]+(?![},]))$/);
return wordMatch ? wordMatch[0] : "";
}, [values.beforeCursor]);
@@ -134,10 +119,9 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
// Add quotes around the value if the context is labelValue
if (context === QueryContextType.labelValue) {
const quote = "\"";
const needsQuote = /(?:=|!=|=~|!~)$/.test(beforeValueByContext);
valueAfterCursor = valueAfterCursor.replace(/^[^\s"|},]*/, "");
const needsOpenQuote = /(?:=|!=|=~|!~)$/.test(beforeValueByContext);
const needsCloseQuote = valueAfterCursor.trim()[0] !== "\"";
insert = `${needsOpenQuote ? quote : ""}${insert}${needsCloseQuote ? quote : ""}`;
insert = `${needsQuote ? quote : ""}${insert}`;
}
if (context === QueryContextType.label) {

View File

@@ -63,7 +63,7 @@ const AnomalyConfig: FC = () => {
URL.revokeObjectURL(downloadUrl);
setTextConfig("");
setDownloadUrl("");
return fetchConfig();
fetchConfig();
};
return (

Some files were not shown because too many files have changed in this diff Show More