mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-05-18 09:16:35 +03:00
Compare commits
1 Commits
test-tmp
...
per-aggreg
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c71a9478f0 |
4
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
4
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
@@ -60,8 +60,8 @@ body:
|
||||
|
||||
For VictoriaMetrics health-state issues please provide full-length screenshots
|
||||
of Grafana dashboards if possible:
|
||||
* [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229)
|
||||
* [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
|
||||
* [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
|
||||
* [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
|
||||
|
||||
See how to setup monitoring here:
|
||||
* [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
|
||||
|
||||
33
.github/workflows/main.yml
vendored
33
.github/workflows/main.yml
vendored
@@ -85,38 +85,9 @@ jobs:
|
||||
restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
|
||||
|
||||
- name: Run tests
|
||||
run: GOGC=10 make ${{ matrix.scenario}}
|
||||
run: make ${{ matrix.scenario}}
|
||||
|
||||
- name: Publish coverage
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
file: ./coverage.txt
|
||||
|
||||
integration-test:
|
||||
name: integration-test
|
||||
needs: [lint, test]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Code checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Go
|
||||
id: go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
cache: false
|
||||
go-version: stable
|
||||
|
||||
- name: Cache Go artifacts
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/bin
|
||||
~/go/pkg/mod
|
||||
key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
|
||||
restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
|
||||
|
||||
- name: Run integration tests
|
||||
run: make integration-test
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -7,7 +7,6 @@
|
||||
.vscode
|
||||
*.test
|
||||
*.swp
|
||||
/vmdocs
|
||||
/gocache-for-docker
|
||||
/victoria-logs-data
|
||||
/victoria-metrics-data
|
||||
@@ -26,4 +25,3 @@ _site
|
||||
/docs/.jekyll-metadata
|
||||
coverage.txt
|
||||
cspell.json
|
||||
*~
|
||||
|
||||
@@ -4,4 +4,3 @@ allowlist:
|
||||
- BSD-3-Clause
|
||||
- BSD-2-Clause
|
||||
- ISC
|
||||
- MPL-2.0
|
||||
|
||||
2
LICENSE
2
LICENSE
@@ -175,7 +175,7 @@
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2019-2025 VictoriaMetrics, Inc.
|
||||
Copyright 2019-2024 VictoriaMetrics, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
120
Makefile
120
Makefile
@@ -13,7 +13,6 @@ PKG_TAG := $(BUILDINFO_TAG)
|
||||
endif
|
||||
|
||||
GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
|
||||
TAR_OWNERSHIP ?= --owner=1000 --group=1000
|
||||
|
||||
.PHONY: $(MAKECMDGOALS)
|
||||
|
||||
@@ -27,7 +26,6 @@ include package/release/Makefile
|
||||
all: \
|
||||
victoria-metrics-prod \
|
||||
victoria-logs-prod \
|
||||
vlogscli-prod \
|
||||
vmagent-prod \
|
||||
vmalert-prod \
|
||||
vmalert-tool-prod \
|
||||
@@ -52,7 +50,6 @@ publish: \
|
||||
package: \
|
||||
package-victoria-metrics \
|
||||
package-victoria-logs \
|
||||
package-vlogscli \
|
||||
package-vmagent \
|
||||
package-vmalert \
|
||||
package-vmalert-tool \
|
||||
@@ -248,7 +245,7 @@ release-victoria-metrics-windows-amd64:
|
||||
|
||||
release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
victoria-metrics-$(GOOS)-$(GOARCH)-prod \
|
||||
&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
victoria-metrics-$(GOOS)-$(GOARCH)-prod \
|
||||
@@ -265,14 +262,6 @@ release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
|
||||
cd bin && rm -rf \
|
||||
victoria-metrics-windows-$(GOARCH)-prod.exe
|
||||
|
||||
release-victoria-logs-bundle: \
|
||||
release-victoria-logs \
|
||||
release-vlogscli
|
||||
|
||||
publish-victoria-logs-bundle: \
|
||||
publish-victoria-logs \
|
||||
publish-vlogscli
|
||||
|
||||
release-victoria-logs:
|
||||
$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
|
||||
release-victoria-logs-linux-amd64 \
|
||||
@@ -313,7 +302,7 @@ release-victoria-logs-windows-amd64:
|
||||
|
||||
release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
victoria-logs-$(GOOS)-$(GOARCH)-prod \
|
||||
&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
victoria-logs-$(GOOS)-$(GOARCH)-prod \
|
||||
@@ -330,63 +319,6 @@ release-victoria-logs-windows-goarch: victoria-logs-windows-$(GOARCH)-prod
|
||||
cd bin && rm -rf \
|
||||
victoria-logs-windows-$(GOARCH)-prod.exe
|
||||
|
||||
release-vlogscli:
|
||||
$(MAKE_PARALLEL) release-vlogscli-linux-386 \
|
||||
release-vlogscli-linux-amd64 \
|
||||
release-vlogscli-linux-arm \
|
||||
release-vlogscli-linux-arm64 \
|
||||
release-vlogscli-darwin-amd64 \
|
||||
release-vlogscli-darwin-arm64 \
|
||||
release-vlogscli-freebsd-amd64 \
|
||||
release-vlogscli-openbsd-amd64 \
|
||||
release-vlogscli-windows-amd64
|
||||
|
||||
release-vlogscli-linux-386:
|
||||
GOOS=linux GOARCH=386 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-linux-amd64:
|
||||
GOOS=linux GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-linux-arm:
|
||||
GOOS=linux GOARCH=arm $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-linux-arm64:
|
||||
GOOS=linux GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-darwin-amd64:
|
||||
GOOS=darwin GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-darwin-arm64:
|
||||
GOOS=darwin GOARCH=arm64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-freebsd-amd64:
|
||||
GOOS=freebsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-openbsd-amd64:
|
||||
GOOS=openbsd GOARCH=amd64 $(MAKE) release-vlogscli-goos-goarch
|
||||
|
||||
release-vlogscli-windows-amd64:
|
||||
GOARCH=amd64 $(MAKE) release-vlogscli-windows-goarch
|
||||
|
||||
release-vlogscli-goos-goarch: vlogscli-$(GOOS)-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
vlogscli-$(GOOS)-$(GOARCH)-prod \
|
||||
&& sha256sum vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
vlogscli-$(GOOS)-$(GOARCH)-prod \
|
||||
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > vlogscli-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
cd bin && rm -rf vlogscli-$(GOOS)-$(GOARCH)-prod
|
||||
|
||||
release-vlogscli-windows-goarch: vlogscli-windows-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
zip vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
|
||||
vlogscli-windows-$(GOARCH)-prod.exe \
|
||||
&& sha256sum vlogscli-windows-$(GOARCH)-$(PKG_TAG).zip \
|
||||
vlogscli-windows-$(GOARCH)-prod.exe \
|
||||
> vlogscli-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
|
||||
cd bin && rm -rf \
|
||||
vlogscli-windows-$(GOARCH)-prod.exe
|
||||
|
||||
release-vmutils: \
|
||||
release-vmutils-linux-386 \
|
||||
release-vmutils-linux-amd64 \
|
||||
@@ -400,7 +332,7 @@ release-vmutils: \
|
||||
|
||||
release-vmutils-linux-386:
|
||||
GOOS=linux GOARCH=386 $(MAKE) release-vmutils-goos-goarch
|
||||
|
||||
|
||||
release-vmutils-linux-amd64:
|
||||
GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
|
||||
|
||||
@@ -434,7 +366,7 @@ release-vmutils-goos-goarch: \
|
||||
vmrestore-$(GOOS)-$(GOARCH)-prod \
|
||||
vmctl-$(GOOS)-$(GOARCH)-prod
|
||||
cd bin && \
|
||||
tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
|
||||
vmagent-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-$(GOOS)-$(GOARCH)-prod \
|
||||
vmalert-tool-$(GOOS)-$(GOARCH)-prod \
|
||||
@@ -501,12 +433,10 @@ pprof-cpu:
|
||||
fmt:
|
||||
gofmt -l -w -s ./lib
|
||||
gofmt -l -w -s ./app
|
||||
gofmt -l -w -s ./apptest
|
||||
|
||||
vet:
|
||||
go vet ./lib/...
|
||||
go vet ./app/...
|
||||
go vet ./apptest/...
|
||||
|
||||
check-all: fmt vet golangci-lint govulncheck
|
||||
|
||||
@@ -527,9 +457,6 @@ test-full:
|
||||
test-full-386:
|
||||
GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
|
||||
|
||||
integration-test: victoria-metrics vmagent vmalert vmauth
|
||||
go test ./apptest/... -skip="^TestCluster.*"
|
||||
|
||||
benchmark:
|
||||
go test -bench=. ./lib/...
|
||||
go test -bench=. ./app/...
|
||||
@@ -539,9 +466,9 @@ benchmark-pure:
|
||||
CGO_ENABLED=0 go test -bench=. ./app/...
|
||||
|
||||
vendor-update:
|
||||
go get -u ./lib/...
|
||||
go get -u ./app/...
|
||||
go mod tidy -compat=1.23
|
||||
go get -u -d ./lib/...
|
||||
go get -u -d ./app/...
|
||||
go mod tidy -compat=1.22
|
||||
go mod vendor
|
||||
|
||||
app-local:
|
||||
@@ -567,7 +494,7 @@ golangci-lint: install-golangci-lint
|
||||
golangci-lint run
|
||||
|
||||
install-golangci-lint:
|
||||
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.63.4
|
||||
which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
|
||||
|
||||
remove-golangci-lint:
|
||||
rm -rf `which golangci-lint`
|
||||
@@ -586,3 +513,34 @@ install-wwhrd:
|
||||
|
||||
check-licenses: install-wwhrd
|
||||
wwhrd check -f .wwhrd.yml
|
||||
|
||||
copy-docs:
|
||||
# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
|
||||
# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
|
||||
echo "---" > ${DST}
|
||||
@if [ ${ORDER} -ne 0 ]; then \
|
||||
echo "sort: ${ORDER}" >> ${DST}; \
|
||||
echo "weight: ${ORDER}" >> ${DST}; \
|
||||
printf "menu:\n docs:\n parent: 'victoriametrics'\n weight: ${ORDER}\n" >> ${DST}; \
|
||||
fi
|
||||
|
||||
echo "title: ${TITLE}" >> ${DST}
|
||||
@if [ ${OLD_URL} ]; then \
|
||||
printf "aliases:\n - ${OLD_URL}\n" >> ${DST}; \
|
||||
fi
|
||||
echo "---" >> ${DST}
|
||||
cat ${SRC} >> ${DST}
|
||||
sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
|
||||
sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"/' ${DST}
|
||||
rm -rf docs/*.tmp
|
||||
|
||||
# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
|
||||
# For ORDER=0 it adds no order tag/weight tag.
|
||||
# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
|
||||
# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
|
||||
# Images starting with <img src="docs/ are replaced with <img src="
|
||||
# Cluster docs are supposed to be ordered as 2nd.
|
||||
# The rest of docs is ordered manually.
|
||||
docs-sync:
|
||||
SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
|
||||
SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs
|
||||
|
||||
@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
|
||||
| Version | Supported |
|
||||
|---------|--------------------|
|
||||
| [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
|
||||
| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
|
||||
| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
|
||||
| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
|
||||
| other releases | :x: |
|
||||
|
||||
See [this page](https://victoriametrics.com/security/) for more details.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please report any security issues to <security@victoriametrics.com>
|
||||
Please report any security issues to security@victoriametrics.com
|
||||
|
||||
BIN
VM_logo.zip
BIN
VM_logo.zip
Binary file not shown.
@@ -1,8 +1,8 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 9428
|
||||
|
||||
ENTRYPOINT ["/victoria-logs-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./victoria-logs-prod
|
||||
|
||||
@@ -92,9 +92,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
if vlselect.RequestHandler(w, r) {
|
||||
return true
|
||||
}
|
||||
if vlstorage.RequestHandler(w, r) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 8428
|
||||
|
||||
ENTRYPOINT ["/victoria-metrics-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./victoria-metrics-prod
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
@@ -40,24 +39,9 @@ var (
|
||||
"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
|
||||
"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
|
||||
"Smaller intervals increase disk IO load. Minimum supported value is 1s")
|
||||
maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmsingle can receive per second. Data ingestion is paused when the limit is exceeded. "+
|
||||
"By default there are no limits on samples ingestion rate.")
|
||||
finalDedupScheduleInterval = flag.Duration("storage.finalDedupScheduleCheckInterval", time.Hour, "The interval for checking when final deduplication process should be started."+
|
||||
"Storage unconditionally adds 25% jitter to the interval value on each check evaluation."+
|
||||
" Changing the interval to the bigger values may delay downsampling, deduplication for historical data."+
|
||||
" See also https://docs.victoriametrics.com/#deduplication")
|
||||
)
|
||||
|
||||
func main() {
|
||||
// VictoriaMetrics is optimized for reduced memory allocations,
|
||||
// so it can run with the reduced GOGC in order to reduce the used memory,
|
||||
// while keeping CPU usage spent in GC at low levels.
|
||||
//
|
||||
// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
|
||||
// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
|
||||
// exceeds 0.05 for extended periods of time.
|
||||
cgroup.SetGOGC(30)
|
||||
|
||||
// Write flags and help message to stdout, since it is easier to grep or pipe.
|
||||
flag.CommandLine.SetOutput(os.Stdout)
|
||||
flag.Usage = usage
|
||||
@@ -90,13 +74,8 @@ func main() {
|
||||
startTime := time.Now()
|
||||
storage.SetDedupInterval(*minScrapeInterval)
|
||||
storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
|
||||
if *finalDedupScheduleInterval < time.Hour {
|
||||
logger.Fatalf("-dedup.finalDedupScheduleCheckInterval cannot be smaller than 1 hour; got %s", *finalDedupScheduleInterval)
|
||||
}
|
||||
storage.SetFinalDedupScheduleInterval(*finalDedupScheduleInterval)
|
||||
vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
|
||||
vmselect.Init()
|
||||
vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
|
||||
vminsert.Init()
|
||||
|
||||
startSelfScraper()
|
||||
@@ -118,7 +97,6 @@ func main() {
|
||||
}
|
||||
logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
|
||||
vminsert.Stop()
|
||||
vminsertcommon.StopIngestionRateLimiter()
|
||||
|
||||
vmstorage.Stop()
|
||||
vmselect.Stop()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -10,10 +10,9 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -49,7 +48,7 @@ func selfScraper(scrapeInterval time.Duration) {
|
||||
var bb bytesutil.ByteBuffer
|
||||
var rows prometheus.Rows
|
||||
var mrs []storage.MetricRow
|
||||
var labels []prompbmarshal.Label
|
||||
var labels []prompb.Label
|
||||
t := time.NewTicker(scrapeInterval)
|
||||
f := func(currentTime time.Time, sendStaleMarkers bool) {
|
||||
currentTimestamp := currentTime.UnixNano() / 1e6
|
||||
@@ -69,10 +68,6 @@ func selfScraper(scrapeInterval time.Duration) {
|
||||
t := &r.Tags[j]
|
||||
labels = addLabel(labels, t.Key, t.Value)
|
||||
}
|
||||
if timeserieslimits.IsExceeding(labels) {
|
||||
// Skip metric with exceeding labels.
|
||||
continue
|
||||
}
|
||||
if len(mrs) < cap(mrs) {
|
||||
mrs = mrs[:len(mrs)+1]
|
||||
} else {
|
||||
@@ -104,11 +99,11 @@ func selfScraper(scrapeInterval time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
func addLabel(dst []prompbmarshal.Label, key, value string) []prompbmarshal.Label {
|
||||
func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
|
||||
if len(dst) < cap(dst) {
|
||||
dst = dst[:len(dst)+1]
|
||||
} else {
|
||||
dst = append(dst, prompbmarshal.Label{})
|
||||
dst = append(dst, prompb.Label{})
|
||||
}
|
||||
lb := &dst[len(dst)-1]
|
||||
lb.Name = key
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
"name": "subquery-aggregation",
|
||||
"issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/184",
|
||||
"data": [
|
||||
"forms_daily_count;item=x 1 {TIME_S-59s}",
|
||||
"forms_daily_count;item=x 2 {TIME_S-1m59s}",
|
||||
"forms_daily_count;item=y 3 {TIME_S-59s}",
|
||||
"forms_daily_count;item=y 4 {TIME_S-1m59s}"],
|
||||
"forms_daily_count;item=x 1 {TIME_S-1m}",
|
||||
"forms_daily_count;item=x 2 {TIME_S-2m}",
|
||||
"forms_daily_count;item=y 3 {TIME_S-1m}",
|
||||
"forms_daily_count;item=y 4 {TIME_S-2m}"],
|
||||
"query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}&latency_offset=1ms"],
|
||||
"result_query": {
|
||||
"status":"success",
|
||||
|
||||
@@ -1,274 +0,0 @@
|
||||
package datadog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
)
|
||||
|
||||
var (
|
||||
datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Datadog tags to be used as stream fields.")
|
||||
datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Datadog tags to ignore.")
|
||||
)
|
||||
|
||||
var parserPool fastjson.ParserPool
|
||||
|
||||
// RequestHandler processes Datadog insert requests
|
||||
func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
case "/api/v1/validate":
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
case "/api/v2/logs":
|
||||
return datadogLogsIngestion(w, r)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
|
||||
w.Header().Add("Content-Type", "application/json")
|
||||
startTime := time.Now()
|
||||
v2LogsRequestsTotal.Inc()
|
||||
reader := r.Body
|
||||
|
||||
var ts int64
|
||||
if tsValue := r.Header.Get("dd-message-timestamp"); tsValue != "" && tsValue != "0" {
|
||||
var err error
|
||||
ts, err = strconv.ParseInt(tsValue, 10, 64)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "could not parse dd-message-timestamp header value: %s", err)
|
||||
return true
|
||||
}
|
||||
ts *= 1e6
|
||||
} else {
|
||||
ts = startTime.UnixNano()
|
||||
}
|
||||
|
||||
if r.Header.Get("Content-Encoding") == "gzip" {
|
||||
zr, err := common.GetGzipReader(reader)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read gzipped logs request: %s", err)
|
||||
return true
|
||||
}
|
||||
defer common.PutGzipReader(zr)
|
||||
reader = zr
|
||||
}
|
||||
|
||||
wcr := writeconcurrencylimiter.GetReader(reader)
|
||||
data, err := io.ReadAll(wcr)
|
||||
writeconcurrencylimiter.PutReader(wcr)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read request body: %s", err)
|
||||
return true
|
||||
}
|
||||
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
|
||||
if len(cp.StreamFields) == 0 {
|
||||
cp.StreamFields = *datadogStreamFields
|
||||
}
|
||||
if len(cp.IgnoreFields) == 0 {
|
||||
cp.IgnoreFields = *datadogIgnoreFields
|
||||
}
|
||||
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("datadog")
|
||||
err = readLogsRequest(ts, data, lmp)
|
||||
lmp.MustClose()
|
||||
if err != nil {
|
||||
logger.Warnf("cannot decode log message in /api/v2/logs request: %s, stream fields: %s", err, cp.StreamFields)
|
||||
return true
|
||||
}
|
||||
|
||||
// update v2LogsRequestDuration only for successfully parsed requests
|
||||
// There is no need in updating v2LogsRequestDuration for request errors,
|
||||
// since their timings are usually much smaller than the timing for successful request parsing.
|
||||
v2LogsRequestDuration.UpdateDuration(startTime)
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
|
||||
var (
|
||||
v2LogsRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
|
||||
v2LogsRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
|
||||
)
|
||||
|
||||
// datadog message field has two formats:
|
||||
// - regular log message with string text
|
||||
// - nested json format for serverless plugins
|
||||
// which has folowing format:
|
||||
// {"message": {"message": "text","lamdba": {"arn": "string","requestID": "string"}, "timestamp": int64} }
|
||||
//
|
||||
// See https://github.com/DataDog/datadog-lambda-extension/blob/28b90c7e4e985b72d60b5f5a5147c69c7ac693c4/bottlecap/src/logs/lambda/mod.rs#L24
|
||||
func appendMsgFields(fields []logstorage.Field, v *fastjson.Value) ([]logstorage.Field, error) {
|
||||
switch v.Type() {
|
||||
case fastjson.TypeString:
|
||||
val := v.GetStringBytes()
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: "_msg",
|
||||
Value: bytesutil.ToUnsafeString(val),
|
||||
})
|
||||
case fastjson.TypeObject:
|
||||
var firstErr error
|
||||
v.GetObject().Visit(func(k []byte, v *fastjson.Value) {
|
||||
if firstErr != nil {
|
||||
return
|
||||
}
|
||||
switch bytesutil.ToUnsafeString(k) {
|
||||
case "message":
|
||||
val := v.GetStringBytes()
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: "_msg",
|
||||
Value: bytesutil.ToUnsafeString(val),
|
||||
})
|
||||
case "status":
|
||||
val := v.GetStringBytes()
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: "status",
|
||||
Value: bytesutil.ToUnsafeString(val),
|
||||
})
|
||||
case "lamdba":
|
||||
obj, err := v.Object()
|
||||
if err != nil {
|
||||
firstErr = err
|
||||
firstErr = fmt.Errorf("unexpected lambda value type for %q:%q; want object", k, v)
|
||||
return
|
||||
}
|
||||
obj.Visit(func(k []byte, v *fastjson.Value) {
|
||||
if firstErr != nil {
|
||||
return
|
||||
}
|
||||
val, err := v.StringBytes()
|
||||
if err != nil {
|
||||
firstErr = fmt.Errorf("unexpected lambda label value type for %q:%q; want string", k, v)
|
||||
return
|
||||
}
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: bytesutil.ToUnsafeString(k),
|
||||
Value: bytesutil.ToUnsafeString(val),
|
||||
})
|
||||
})
|
||||
|
||||
}
|
||||
})
|
||||
default:
|
||||
return fields, fmt.Errorf("unsupported message type %q", v.Type().String())
|
||||
}
|
||||
return fields, nil
|
||||
}
|
||||
|
||||
// readLogsRequest parses data according to DataDog logs format
|
||||
// https://docs.datadoghq.com/api/latest/logs/#send-logs
|
||||
func readLogsRequest(ts int64, data []byte, lmp insertutils.LogMessageProcessor) error {
|
||||
p := parserPool.Get()
|
||||
defer parserPool.Put(p)
|
||||
v, err := p.ParseBytes(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse JSON request body: %w", err)
|
||||
}
|
||||
records, err := v.Array()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot extract array from parsed JSON: %w", err)
|
||||
}
|
||||
|
||||
var fields []logstorage.Field
|
||||
for _, r := range records {
|
||||
o, err := r.Object()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not extract log record: %w", err)
|
||||
}
|
||||
o.Visit(func(k []byte, v *fastjson.Value) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
switch bytesutil.ToUnsafeString(k) {
|
||||
case "message":
|
||||
fields, err = appendMsgFields(fields, v)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
case "timestamp":
|
||||
val, e := v.Int64()
|
||||
if e != nil {
|
||||
err = fmt.Errorf("failed to parse timestamp for %q:%q", k, v)
|
||||
}
|
||||
if val > 0 {
|
||||
ts = val * 1e6
|
||||
}
|
||||
case "ddtags":
|
||||
// https://docs.datadoghq.com/getting_started/tagging/
|
||||
val, e := v.StringBytes()
|
||||
if e != nil {
|
||||
err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
|
||||
return
|
||||
}
|
||||
var pair []byte
|
||||
idx := 0
|
||||
for idx >= 0 {
|
||||
idx = bytes.IndexByte(val, ',')
|
||||
if idx < 0 {
|
||||
pair = val
|
||||
} else {
|
||||
pair = val[:idx]
|
||||
val = val[idx+1:]
|
||||
}
|
||||
if len(pair) > 0 {
|
||||
n := bytes.IndexByte(pair, ':')
|
||||
if n < 0 {
|
||||
// No tag value.
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: bytesutil.ToUnsafeString(pair),
|
||||
Value: "no_label_value",
|
||||
})
|
||||
}
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: bytesutil.ToUnsafeString(pair[:n]),
|
||||
Value: bytesutil.ToUnsafeString(pair[n+1:]),
|
||||
})
|
||||
}
|
||||
}
|
||||
default:
|
||||
val, e := v.StringBytes()
|
||||
if e != nil {
|
||||
err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
|
||||
return
|
||||
}
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: bytesutil.ToUnsafeString(k),
|
||||
Value: bytesutil.ToUnsafeString(val),
|
||||
})
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lmp.AddRow(ts, fields, nil)
|
||||
fields = fields[:0]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,104 +0,0 @@
|
||||
package datadog
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
)
|
||||
|
||||
func TestReadLogsRequestFailure(t *testing.T) {
|
||||
f := func(data string) {
|
||||
t.Helper()
|
||||
|
||||
ts := time.Now().UnixNano()
|
||||
|
||||
lmp := &insertutils.TestLogMessageProcessor{}
|
||||
if err := readLogsRequest(ts, []byte(data), lmp); err == nil {
|
||||
t.Fatalf("expecting non-empty error")
|
||||
}
|
||||
if err := lmp.Verify(nil, ""); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
}
|
||||
f("foobar")
|
||||
f(`{}`)
|
||||
f(`["create":{}]`)
|
||||
f(`{"create":{}}
|
||||
foobar`)
|
||||
}
|
||||
|
||||
func TestReadLogsRequestSuccess(t *testing.T) {
|
||||
f := func(data string, rowsExpected int, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
ts := time.Now().UnixNano()
|
||||
var timestampsExpected []int64
|
||||
for i := 0; i < rowsExpected; i++ {
|
||||
timestampsExpected = append(timestampsExpected, ts)
|
||||
}
|
||||
lmp := &insertutils.TestLogMessageProcessor{}
|
||||
if err := readLogsRequest(ts, []byte(data), lmp); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := lmp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify non-empty data
|
||||
data := `[
|
||||
{
|
||||
"ddsource":"nginx",
|
||||
"ddtags":"tag1:value1,tag2:value2",
|
||||
"hostname":"127.0.0.1",
|
||||
"message":"bar",
|
||||
"service":"test"
|
||||
}, {
|
||||
"ddsource":"nginx",
|
||||
"ddtags":"tag1:value1,tag2:value2",
|
||||
"hostname":"127.0.0.1",
|
||||
"message":{"message": "nested"},
|
||||
"service":"test"
|
||||
}, {
|
||||
"ddsource":"nginx",
|
||||
"ddtags":"tag1:value1,tag2:value2",
|
||||
"hostname":"127.0.0.1",
|
||||
"message":"foobar",
|
||||
"service":"test"
|
||||
}, {
|
||||
"ddsource":"nginx",
|
||||
"ddtags":"tag1:value1,tag2:value2",
|
||||
"hostname":"127.0.0.1",
|
||||
"message":"baz",
|
||||
"service":"test"
|
||||
}, {
|
||||
"ddsource":"nginx",
|
||||
"ddtags":"tag1:value1,tag2:value2",
|
||||
"hostname":"127.0.0.1",
|
||||
"message":"xyz",
|
||||
"service":"test"
|
||||
}, {
|
||||
"ddsource": "nginx",
|
||||
"ddtags":"tag1:value1,tag2:value2,",
|
||||
"hostname":"127.0.0.1",
|
||||
"message":"xyz",
|
||||
"service":"test"
|
||||
}, {
|
||||
"ddsource":"nginx",
|
||||
"ddtags":",tag1:value1,tag2:value2",
|
||||
"hostname":"127.0.0.1",
|
||||
"message":"xyz",
|
||||
"service":"test"
|
||||
}
|
||||
]`
|
||||
rowsExpected := 7
|
||||
resultExpected := `{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"bar","service":"test"}
|
||||
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"nested","service":"test"}
|
||||
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"foobar","service":"test"}
|
||||
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"baz","service":"test"}
|
||||
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
|
||||
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
|
||||
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}`
|
||||
f(data, rowsExpected, resultExpected)
|
||||
}
|
||||
@@ -1,10 +1,14 @@
|
||||
package elasticsearch
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -53,12 +57,6 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
|
||||
// Return fake response for Logstash APIs requests.
|
||||
// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
switch path {
|
||||
case "/":
|
||||
switch r.Method {
|
||||
@@ -99,10 +97,9 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
lmp := cp.NewLogMessageProcessor("elasticsearch_bulk")
|
||||
lmp := cp.NewLogMessageProcessor()
|
||||
isGzip := r.Header.Get("Content-Encoding") == "gzip"
|
||||
streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
|
||||
n, err := readBulkRequest(streamName, r.Body, isGzip, cp.TimeField, cp.MsgFields, lmp)
|
||||
n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, lmp)
|
||||
lmp.MustClose()
|
||||
if err != nil {
|
||||
logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
|
||||
@@ -128,10 +125,11 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
|
||||
var (
|
||||
bulkRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
|
||||
rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
|
||||
bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
|
||||
)
|
||||
|
||||
func readBulkRequest(streamName string, r io.Reader, isGzip bool, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (int, error) {
|
||||
func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string, lmp insertutils.LogMessageProcessor) (int, error) {
|
||||
// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
|
||||
|
||||
if isGzip {
|
||||
@@ -146,29 +144,48 @@ func readBulkRequest(streamName string, r io.Reader, isGzip bool, timeField stri
|
||||
wcr := writeconcurrencylimiter.GetReader(r)
|
||||
defer writeconcurrencylimiter.PutReader(wcr)
|
||||
|
||||
lr := insertutils.NewLineReader(streamName, wcr)
|
||||
lb := lineBufferPool.Get()
|
||||
defer lineBufferPool.Put(lb)
|
||||
|
||||
lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
|
||||
sc := bufio.NewScanner(wcr)
|
||||
sc.Buffer(lb.B, len(lb.B))
|
||||
|
||||
n := 0
|
||||
nCheckpoint := 0
|
||||
for {
|
||||
ok, err := readBulkLine(lr, timeField, msgFields, lmp)
|
||||
ok, err := readBulkLine(sc, timeField, msgField, lmp)
|
||||
wcr.DecConcurrency()
|
||||
if err != nil || !ok {
|
||||
rowsIngestedTotal.Add(n - nCheckpoint)
|
||||
return n, err
|
||||
}
|
||||
n++
|
||||
if batchSize := n - nCheckpoint; n >= 1000 {
|
||||
rowsIngestedTotal.Add(batchSize)
|
||||
nCheckpoint = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readBulkLine(lr *insertutils.LineReader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (bool, error) {
|
||||
var lineBufferPool bytesutil.ByteBufferPool
|
||||
|
||||
func readBulkLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
|
||||
var line []byte
|
||||
|
||||
// Read the command, must be "create" or "index"
|
||||
for len(line) == 0 {
|
||||
if !lr.NextLine() {
|
||||
err := lr.Err()
|
||||
return false, err
|
||||
if !sc.Scan() {
|
||||
if err := sc.Err(); err != nil {
|
||||
if errors.Is(err, bufio.ErrTooLong) {
|
||||
return false, fmt.Errorf(`cannot read "create" or "index" command, since its size exceeds -insert.maxLineSizeBytes=%d`,
|
||||
insertutils.MaxLineSizeBytes.IntN())
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
line = lr.Line
|
||||
line = sc.Bytes()
|
||||
}
|
||||
lineStr := bytesutil.ToUnsafeString(line)
|
||||
if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
|
||||
@@ -176,18 +193,16 @@ func readBulkLine(lr *insertutils.LineReader, timeField string, msgFields []stri
|
||||
}
|
||||
|
||||
// Decode log message
|
||||
if !lr.NextLine() {
|
||||
if err := lr.Err(); err != nil {
|
||||
if !sc.Scan() {
|
||||
if err := sc.Err(); err != nil {
|
||||
if errors.Is(err, bufio.ErrTooLong) {
|
||||
return false, fmt.Errorf("cannot read log message, since its size exceeds -insert.maxLineSizeBytes=%d", insertutils.MaxLineSizeBytes.IntN())
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
|
||||
}
|
||||
line = lr.Line
|
||||
if len(line) == 0 {
|
||||
// Special case - the line could be too long, so it was skipped.
|
||||
// Continue parsing next lines.
|
||||
return true, nil
|
||||
}
|
||||
line = sc.Bytes()
|
||||
p := logstorage.GetJSONParser()
|
||||
if err := p.ParseLogMessage(line); err != nil {
|
||||
return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
|
||||
@@ -200,8 +215,8 @@ func readBulkLine(lr *insertutils.LineReader, timeField string, msgFields []stri
|
||||
if ts == 0 {
|
||||
ts = time.Now().UnixNano()
|
||||
}
|
||||
logstorage.RenameField(p.Fields, msgFields, "_msg")
|
||||
lmp.AddRow(ts, p.Fields, nil)
|
||||
logstorage.RenameField(p.Fields, msgField, "_msg")
|
||||
lmp.AddRow(ts, p.Fields)
|
||||
logstorage.PutJSONParser(p)
|
||||
|
||||
return true, nil
|
||||
@@ -230,8 +245,19 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
|
||||
// Try parsing timestamp in seconds or milliseconds
|
||||
return insertutils.ParseUnixTimestamp(s)
|
||||
// Try parsing timestamp in milliseconds
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
|
||||
}
|
||||
if n > int64(math.MaxInt64)/1e6 {
|
||||
return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
|
||||
}
|
||||
if n < int64(math.MinInt64)/1e6 {
|
||||
return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
|
||||
}
|
||||
n *= 1e6
|
||||
return n, nil
|
||||
}
|
||||
if len(s) == len("YYYY-MM-DD") {
|
||||
t, err := time.Parse("2006-01-02", s)
|
||||
|
||||
@@ -15,7 +15,7 @@ func TestReadBulkRequest_Failure(t *testing.T) {
|
||||
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
r := bytes.NewBufferString(data)
|
||||
rows, err := readBulkRequest("test", r, false, "_time", []string{"_msg"}, tlp)
|
||||
rows, err := readBulkRequest(r, false, "_time", "_msg", tlp)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-empty error")
|
||||
}
|
||||
@@ -33,22 +33,21 @@ foobar`)
|
||||
}
|
||||
|
||||
func TestReadBulkRequest_Success(t *testing.T) {
|
||||
f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
|
||||
f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
msgFields := []string{"non_existing_foo", msgField, "non_exiting_bar"}
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
|
||||
// Read the request without compression
|
||||
r := bytes.NewBufferString(data)
|
||||
rows, err := readBulkRequest("test", r, false, timeField, msgFields, tlp)
|
||||
rows, err := readBulkRequest(r, false, timeField, msgField, tlp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if rows != len(timestampsExpected) {
|
||||
t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
|
||||
if rows != rowsExpected {
|
||||
t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
|
||||
}
|
||||
if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -56,41 +55,39 @@ func TestReadBulkRequest_Success(t *testing.T) {
|
||||
tlp = &insertutils.TestLogMessageProcessor{}
|
||||
compressedData := compressData(data)
|
||||
r = bytes.NewBufferString(compressedData)
|
||||
rows, err = readBulkRequest("test", r, true, timeField, msgFields, tlp)
|
||||
rows, err = readBulkRequest(r, true, timeField, msgField, tlp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if rows != len(timestampsExpected) {
|
||||
t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
|
||||
if rows != rowsExpected {
|
||||
t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
|
||||
}
|
||||
if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatalf("verification failure after compression: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify an empty data
|
||||
f("", "_time", "_msg", nil, "")
|
||||
f("\n", "_time", "_msg", nil, "")
|
||||
f("\n\n", "_time", "_msg", nil, "")
|
||||
f("", "_time", "_msg", 0, nil, "")
|
||||
f("\n", "_time", "_msg", 0, nil, "")
|
||||
f("\n\n", "_time", "_msg", 0, nil, "")
|
||||
|
||||
// Verify non-empty data
|
||||
data := `{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
|
||||
{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"@timestamp":"2023-06-06 04:48:12.735+01:00","message":"baz"}
|
||||
{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
|
||||
{"index":{"_index":"filebeat-8.8.0"}}
|
||||
{"message":"xyz","@timestamp":"1686026893735","x":"y"}
|
||||
{"create":{"_index":"filebeat-8.8.0"}}
|
||||
{"message":"qwe rty","@timestamp":"1686026893"}
|
||||
{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
|
||||
`
|
||||
timeField := "@timestamp"
|
||||
msgField := "message"
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
|
||||
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"_msg":"baz"}
|
||||
{"_msg":"xyz","x":"y"}
|
||||
{"_msg":"qwe rty"}`
|
||||
f(data, timeField, msgField, timestampsExpected, resultExpected)
|
||||
rowsExpected := 3
|
||||
timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
|
||||
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"@timestamp":"","_msg":"baz"}
|
||||
{"_msg":"xyz","@timestamp":"","x":"y"}`
|
||||
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
func compressData(s string) string {
|
||||
|
||||
@@ -32,7 +32,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
|
||||
dataBytes := bytesutil.ToUnsafeBytes(data)
|
||||
|
||||
timeField := "@timestamp"
|
||||
msgFields := []string{"message"}
|
||||
msgField := "message"
|
||||
blp := &insertutils.BenchmarkLogMessageProcessor{}
|
||||
|
||||
b.ReportAllocs()
|
||||
@@ -41,7 +41,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
|
||||
r := &bytes.Reader{}
|
||||
for pb.Next() {
|
||||
r.Reset(dataBytes)
|
||||
_, err := readBulkRequest("test", r, isGzip, timeField, msgFields, blp)
|
||||
_, err := readBulkRequest(r, isGzip, timeField, msgField, blp)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
package insertutils
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -19,21 +15,15 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultMsgValue = flag.String("defaultMsgValue", "missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field",
|
||||
"Default value for _msg field if the ingested log entry doesn't contain it; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field")
|
||||
)
|
||||
|
||||
// CommonParams contains common HTTP parameters used by log ingestion APIs.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
|
||||
type CommonParams struct {
|
||||
TenantID logstorage.TenantID
|
||||
TimeField string
|
||||
MsgFields []string
|
||||
MsgField string
|
||||
StreamFields []string
|
||||
IgnoreFields []string
|
||||
ExtraFields []logstorage.Field
|
||||
|
||||
Debug bool
|
||||
DebugRequestURI string
|
||||
@@ -48,27 +38,22 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
timeField := "_time"
|
||||
if tf := httputils.GetRequestValue(r, "_time_field", "VL-Time-Field"); tf != "" {
|
||||
// Extract time field name from _time_field query arg
|
||||
var timeField = "_time"
|
||||
if tf := r.FormValue("_time_field"); tf != "" {
|
||||
timeField = tf
|
||||
}
|
||||
|
||||
msgFields := httputils.GetArray(r, "_msg_field", "VL-Msg-Field")
|
||||
streamFields := httputils.GetArray(r, "_stream_fields", "VL-Stream-Fields")
|
||||
ignoreFields := httputils.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
|
||||
|
||||
extraFields, err := getExtraFields(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// Extract message field name from _msg_field query arg
|
||||
var msgField = ""
|
||||
if msgf := r.FormValue("_msg_field"); msgf != "" {
|
||||
msgField = msgf
|
||||
}
|
||||
|
||||
debug := false
|
||||
if dv := httputils.GetRequestValue(r, "debug", "VL-Debug"); dv != "" {
|
||||
debug, err = strconv.ParseBool(dv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse debug=%q: %w", dv, err)
|
||||
}
|
||||
}
|
||||
streamFields := httputils.GetArray(r, "_stream_fields")
|
||||
ignoreFields := httputils.GetArray(r, "ignore_fields")
|
||||
|
||||
debug := httputils.GetBool(r, "debug")
|
||||
debugRequestURI := ""
|
||||
debugRemoteAddr := ""
|
||||
if debug {
|
||||
@@ -79,57 +64,28 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
||||
cp := &CommonParams{
|
||||
TenantID: tenantID,
|
||||
TimeField: timeField,
|
||||
MsgFields: msgFields,
|
||||
MsgField: msgField,
|
||||
StreamFields: streamFields,
|
||||
IgnoreFields: ignoreFields,
|
||||
ExtraFields: extraFields,
|
||||
Debug: debug,
|
||||
DebugRequestURI: debugRequestURI,
|
||||
DebugRemoteAddr: debugRemoteAddr,
|
||||
}
|
||||
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
|
||||
efs := httputils.GetArray(r, "extra_fields", "VL-Extra-Fields")
|
||||
if len(efs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
extraFields := make([]logstorage.Field, len(efs))
|
||||
for i, ef := range efs {
|
||||
n := strings.Index(ef, "=")
|
||||
if n <= 0 || n == len(ef)-1 {
|
||||
return nil, fmt.Errorf(`invalid extra_field format: %q; must be in the form "field=value"`, ef)
|
||||
}
|
||||
extraFields[i] = logstorage.Field{
|
||||
Name: ef[:n],
|
||||
Value: ef[n+1:],
|
||||
}
|
||||
}
|
||||
return extraFields, nil
|
||||
}
|
||||
|
||||
// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
|
||||
func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields []string, extraFields []logstorage.Field) *CommonParams {
|
||||
func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams {
|
||||
// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
|
||||
if streamFields == nil {
|
||||
streamFields = []string{
|
||||
"hostname",
|
||||
"app_name",
|
||||
"proc_id",
|
||||
}
|
||||
}
|
||||
cp := &CommonParams{
|
||||
TenantID: tenantID,
|
||||
TimeField: "timestamp",
|
||||
MsgFields: []string{
|
||||
"message",
|
||||
MsgField: "message",
|
||||
StreamFields: []string{
|
||||
"hostname",
|
||||
"app_name",
|
||||
"proc_id",
|
||||
},
|
||||
StreamFields: streamFields,
|
||||
IgnoreFields: ignoreFields,
|
||||
ExtraFields: extraFields,
|
||||
}
|
||||
|
||||
return cp
|
||||
@@ -137,12 +93,10 @@ func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignore
|
||||
|
||||
// LogMessageProcessor is an interface for log message processors.
|
||||
type LogMessageProcessor interface {
|
||||
// AddRow must add row to the LogMessageProcessor with the given timestamp and fields.
|
||||
//
|
||||
// If streamFields is non-nil, then the given streamFields must be used as log stream fields instead of pre-configured fields.
|
||||
// AddRow must add row to the LogMessageProcessor with the given timestamp and the given fields.
|
||||
//
|
||||
// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
|
||||
AddRow(timestamp int64, fields, streamFields []logstorage.Field)
|
||||
AddRow(timestamp int64, fields []logstorage.Field)
|
||||
|
||||
// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
|
||||
MustClose()
|
||||
@@ -156,9 +110,6 @@ type logMessageProcessor struct {
|
||||
|
||||
cp *CommonParams
|
||||
lr *logstorage.LogRows
|
||||
|
||||
rowsIngestedTotal *metrics.Counter
|
||||
bytesIngestedTotal *metrics.Counter
|
||||
}
|
||||
|
||||
func (lmp *logMessageProcessor) initPeriodicFlush() {
|
||||
@@ -188,28 +139,22 @@ func (lmp *logMessageProcessor) initPeriodicFlush() {
|
||||
}
|
||||
|
||||
// AddRow adds new log message to lmp with the given timestamp and fields.
|
||||
//
|
||||
// If streamFields is non-nil, then it is used as log stream fields instead of the pre-configured stream fields.
|
||||
func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
|
||||
func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
|
||||
lmp.mu.Lock()
|
||||
defer lmp.mu.Unlock()
|
||||
|
||||
lmp.rowsIngestedTotal.Inc()
|
||||
n := logstorage.EstimatedJSONRowLen(fields)
|
||||
lmp.bytesIngestedTotal.Add(n)
|
||||
|
||||
if len(fields) > *MaxFieldsPerLine {
|
||||
line := logstorage.MarshalFieldsToJSON(nil, fields)
|
||||
logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
|
||||
rf := logstorage.RowFormatter(fields)
|
||||
logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
|
||||
rowsDroppedTotalTooManyFields.Inc()
|
||||
return
|
||||
}
|
||||
|
||||
lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields, streamFields)
|
||||
lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields)
|
||||
if lmp.cp.Debug {
|
||||
s := lmp.lr.GetRowString(0)
|
||||
lmp.lr.ResetKeepSettings()
|
||||
logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
|
||||
logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
|
||||
rowsDroppedTotalDebug.Inc()
|
||||
return
|
||||
}
|
||||
@@ -238,17 +183,12 @@ func (lmp *logMessageProcessor) MustClose() {
|
||||
// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
|
||||
//
|
||||
// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
|
||||
func (cp *CommonParams) NewLogMessageProcessor(protocolName string) LogMessageProcessor {
|
||||
lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.ExtraFields, *defaultMsgValue)
|
||||
rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
|
||||
bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
|
||||
func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
|
||||
lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
|
||||
lmp := &logMessageProcessor{
|
||||
cp: cp,
|
||||
lr: lr,
|
||||
|
||||
rowsIngestedTotal: rowsIngestedTotal,
|
||||
bytesIngestedTotal: bytesIngestedTotal,
|
||||
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
lmp.initPeriodicFlush()
|
||||
@@ -256,7 +196,5 @@ func (cp *CommonParams) NewLogMessageProcessor(protocolName string) LogMessagePr
|
||||
return lmp
|
||||
}
|
||||
|
||||
var (
|
||||
rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
|
||||
rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
|
||||
)
|
||||
var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
|
||||
var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
|
||||
|
||||
@@ -8,10 +8,8 @@ import (
|
||||
|
||||
var (
|
||||
// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
|
||||
MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
|
||||
MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
|
||||
|
||||
// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
|
||||
MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
|
||||
MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
|
||||
)
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
package insertutils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
|
||||
)
|
||||
|
||||
// LineReader reads newline-delimited lines from the underlying reader
|
||||
type LineReader struct {
|
||||
// Line contains the next line read after the call to NextLine
|
||||
//
|
||||
// The Line contents is valid until the next call to NextLine.
|
||||
Line []byte
|
||||
|
||||
// name is the LineReader name
|
||||
name string
|
||||
|
||||
// r is the underlying reader to read data from
|
||||
r io.Reader
|
||||
|
||||
// buf is a buffer for reading the next line
|
||||
buf []byte
|
||||
|
||||
// bufOffset is the offset at buf to read the next line from
|
||||
bufOffset int
|
||||
|
||||
// err is the last error when reading data from r
|
||||
err error
|
||||
|
||||
// eofReached is set to true when all the data is read from r
|
||||
eofReached bool
|
||||
}
|
||||
|
||||
// NewLineReader returns LineReader for r.
|
||||
func NewLineReader(name string, r io.Reader) *LineReader {
|
||||
return &LineReader{
|
||||
name: name,
|
||||
r: r,
|
||||
}
|
||||
}
|
||||
|
||||
// NextLine reads the next line from the underlying reader.
|
||||
//
|
||||
// It returns true if the next line is successfully read into Line.
|
||||
// If the line length exceeds MaxLineSizeBytes, then this line is skipped
|
||||
// and an empty line is returned instead.
|
||||
//
|
||||
// If false is returned, then no more lines left to read from r.
|
||||
// Check for Err in this case.
|
||||
func (lr *LineReader) NextLine() bool {
|
||||
for {
|
||||
if lr.bufOffset >= len(lr.buf) {
|
||||
if lr.err != nil || lr.eofReached {
|
||||
return false
|
||||
}
|
||||
if !lr.readMoreData() {
|
||||
return false
|
||||
}
|
||||
if lr.bufOffset >= len(lr.buf) && lr.eofReached {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
buf := lr.buf[lr.bufOffset:]
|
||||
if n := bytes.IndexByte(buf, '\n'); n >= 0 {
|
||||
lr.Line = buf[:n]
|
||||
lr.bufOffset += n + 1
|
||||
return true
|
||||
}
|
||||
if lr.eofReached {
|
||||
lr.Line = buf
|
||||
lr.bufOffset += len(buf)
|
||||
return true
|
||||
}
|
||||
if !lr.readMoreData() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Err returns the last error after NextLine call.
|
||||
func (lr *LineReader) Err() error {
|
||||
if lr.err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s: %s", lr.name, lr.err)
|
||||
}
|
||||
|
||||
func (lr *LineReader) readMoreData() bool {
|
||||
if lr.bufOffset > 0 {
|
||||
lr.buf = append(lr.buf[:0], lr.buf[lr.bufOffset:]...)
|
||||
lr.bufOffset = 0
|
||||
}
|
||||
|
||||
bufLen := len(lr.buf)
|
||||
if bufLen >= MaxLineSizeBytes.IntN() {
|
||||
logger.Warnf("%s: the line length exceeds -insert.maxLineSizeBytes=%d; skipping it; line contents=%q", lr.name, MaxLineSizeBytes.IntN(), lr.buf)
|
||||
tooLongLinesSkipped.Inc()
|
||||
return lr.skipUntilNextLine()
|
||||
}
|
||||
|
||||
lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
|
||||
n, err := lr.r.Read(lr.buf[bufLen:])
|
||||
lr.buf = lr.buf[:bufLen+n]
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
lr.eofReached = true
|
||||
return true
|
||||
}
|
||||
lr.err = fmt.Errorf("cannot read the next line: %s", err)
|
||||
}
|
||||
return n > 0
|
||||
}
|
||||
|
||||
var tooLongLinesSkipped = metrics.NewCounter("vl_too_long_lines_skipped_total")
|
||||
|
||||
func (lr *LineReader) skipUntilNextLine() bool {
|
||||
for {
|
||||
lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
|
||||
n, err := lr.r.Read(lr.buf)
|
||||
lr.buf = lr.buf[:n]
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
lr.eofReached = true
|
||||
lr.buf = lr.buf[:0]
|
||||
return true
|
||||
}
|
||||
lr.err = fmt.Errorf("cannot skip the current line: %s", err)
|
||||
return false
|
||||
}
|
||||
if n := bytes.IndexByte(lr.buf, '\n'); n >= 0 {
|
||||
// Include \n in the buf, so too long line is replaced with an empty line.
|
||||
// This is needed for maintaining synchorinzation consistency between lines
|
||||
// in protocols such as Elasticsearch bulk import.
|
||||
lr.buf = append(lr.buf[:0], lr.buf[n:]...)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
package insertutils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLineReader_Success(t *testing.T) {
|
||||
f := func(data string, linesExpected []string) {
|
||||
t.Helper()
|
||||
|
||||
r := bytes.NewBufferString(data)
|
||||
lr := NewLineReader("foo", r)
|
||||
var lines []string
|
||||
for lr.NextLine() {
|
||||
lines = append(lines, string(lr.Line))
|
||||
}
|
||||
if err := lr.Err(); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if lr.NextLine() {
|
||||
t.Fatalf("expecting error on the second call to NextLine()")
|
||||
}
|
||||
if !reflect.DeepEqual(lines, linesExpected) {
|
||||
t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f("", nil)
|
||||
f("\n", []string{""})
|
||||
f("\n\n", []string{"", ""})
|
||||
f("foo", []string{"foo"})
|
||||
f("foo\n", []string{"foo"})
|
||||
f("\nfoo", []string{"", "foo"})
|
||||
f("foo\n\n", []string{"foo", ""})
|
||||
f("foo\nbar", []string{"foo", "bar"})
|
||||
f("foo\nbar\n", []string{"foo", "bar"})
|
||||
f("\nfoo\n\nbar\n\n", []string{"", "foo", "", "bar", ""})
|
||||
}
|
||||
|
||||
func TestLineReader_SkipUntilNextLine(t *testing.T) {
|
||||
f := func(data string, linesExpected []string) {
|
||||
t.Helper()
|
||||
|
||||
r := bytes.NewBufferString(data)
|
||||
lr := NewLineReader("foo", r)
|
||||
var lines []string
|
||||
for lr.NextLine() {
|
||||
lines = append(lines, string(lr.Line))
|
||||
}
|
||||
if err := lr.Err(); err != nil {
|
||||
t.Fatalf("unexpected error for data=%q: %s", data, err)
|
||||
}
|
||||
if lr.NextLine() {
|
||||
t.Fatalf("expecting error on the second call to NextLine()")
|
||||
}
|
||||
if !reflect.DeepEqual(lines, linesExpected) {
|
||||
t.Fatalf("unexpected lines for data=%q\ngot\n%q\nwant\n%q", data, lines, linesExpected)
|
||||
}
|
||||
}
|
||||
|
||||
for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
|
||||
longLineLen := MaxLineSizeBytes.IntN() + overflow
|
||||
longLine := string(make([]byte, longLineLen))
|
||||
|
||||
// Single long line
|
||||
data := longLine
|
||||
f(data, nil)
|
||||
|
||||
// Multiple long lines
|
||||
data = longLine + "\n" + longLine
|
||||
f(data, []string{""})
|
||||
|
||||
data = longLine + "\n" + longLine + "\n"
|
||||
f(data, []string{"", ""})
|
||||
|
||||
// Long line in the middle
|
||||
data = "foo\n" + longLine + "\nbar"
|
||||
f(data, []string{"foo", "", "bar"})
|
||||
|
||||
// Multiple long lines in the middle
|
||||
data = "foo\n" + longLine + "\n" + longLine + "\nbar"
|
||||
f(data, []string{"foo", "", "", "bar"})
|
||||
|
||||
// Long line in the end
|
||||
data = "foo\n" + longLine
|
||||
f(data, []string{"foo"})
|
||||
|
||||
// Long line in the end
|
||||
data = "foo\n" + longLine + "\n"
|
||||
f(data, []string{"foo", ""})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLineReader_Failure(t *testing.T) {
|
||||
f := func(data string, linesExpected []string) {
|
||||
t.Helper()
|
||||
|
||||
fr := &failureReader{
|
||||
r: bytes.NewBufferString(data),
|
||||
}
|
||||
lr := NewLineReader("foo", fr)
|
||||
var lines []string
|
||||
for lr.NextLine() {
|
||||
lines = append(lines, string(lr.Line))
|
||||
}
|
||||
if err := lr.Err(); err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
}
|
||||
if lr.NextLine() {
|
||||
t.Fatalf("expecting error on the second call to NextLine()")
|
||||
}
|
||||
if err := lr.Err(); err == nil {
|
||||
t.Fatalf("expecting non-nil error on the second call")
|
||||
}
|
||||
if !reflect.DeepEqual(lines, linesExpected) {
|
||||
t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f("", nil)
|
||||
f("foo", nil)
|
||||
f("foo\n", []string{"foo"})
|
||||
f("\n", []string{""})
|
||||
f("foo\nbar", []string{"foo"})
|
||||
f("foo\nbar\n", []string{"foo", "bar"})
|
||||
f("\nfoo\nbar\n\n", []string{"", "foo", "bar", ""})
|
||||
|
||||
// long line
|
||||
longLineLen := MaxLineSizeBytes.IntN()
|
||||
for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
|
||||
longLine := string(make([]byte, longLineLen+overflow))
|
||||
|
||||
data := longLine
|
||||
f(data, nil)
|
||||
|
||||
data = "foo\n" + longLine
|
||||
f(data, []string{"foo"})
|
||||
|
||||
data = longLine + "\nfoo"
|
||||
f(data, []string{""})
|
||||
|
||||
data = longLine + "\nfoo\n"
|
||||
f(data, []string{"", "foo"})
|
||||
}
|
||||
}
|
||||
|
||||
type failureReader struct {
|
||||
r io.Reader
|
||||
}
|
||||
|
||||
func (r *failureReader) Read(p []byte) (int, error) {
|
||||
n, _ := r.r.Read(p)
|
||||
if n > 0 {
|
||||
return n, nil
|
||||
}
|
||||
return 0, fmt.Errorf("some error")
|
||||
}
|
||||
@@ -15,10 +15,7 @@ type TestLogMessageProcessor struct {
|
||||
}
|
||||
|
||||
// AddRow adds row with the given timestamp and fields to tlp
|
||||
func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
|
||||
if streamFields != nil {
|
||||
panic(fmt.Errorf("BUG: streamFields must be nil; got %v", streamFields))
|
||||
}
|
||||
func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
|
||||
tlp.timestamps = append(tlp.timestamps, timestamp)
|
||||
tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
|
||||
}
|
||||
@@ -28,10 +25,10 @@ func (tlp *TestLogMessageProcessor) MustClose() {
|
||||
}
|
||||
|
||||
// Verify verifies the number of rows, timestamps and results after AddRow calls.
|
||||
func (tlp *TestLogMessageProcessor) Verify(timestampsExpected []int64, resultExpected string) error {
|
||||
func (tlp *TestLogMessageProcessor) Verify(rowsExpected int, timestampsExpected []int64, resultExpected string) error {
|
||||
result := strings.Join(tlp.rows, "\n")
|
||||
if len(tlp.rows) != len(timestampsExpected) {
|
||||
return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), len(timestampsExpected), result, resultExpected)
|
||||
if len(tlp.rows) != rowsExpected {
|
||||
return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), rowsExpected, result, resultExpected)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
|
||||
@@ -48,7 +45,7 @@ func (tlp *TestLogMessageProcessor) Verify(timestampsExpected []int64, resultExp
|
||||
type BenchmarkLogMessageProcessor struct{}
|
||||
|
||||
// AddRow implements LogMessageProcessor interface.
|
||||
func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _, _ []logstorage.Field) {
|
||||
func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _ []logstorage.Field) {
|
||||
}
|
||||
|
||||
// MustClose implements LogMessageProcessor interface.
|
||||
|
||||
@@ -2,69 +2,32 @@ package insertutils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
)
|
||||
|
||||
// ExtractTimestampFromFields extracts timestamp in nanoseconds from the field with the name timeField at fields.
|
||||
// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
|
||||
//
|
||||
// The value for the timeField is set to empty string after returning from the function,
|
||||
// so it could be ignored during data ingestion.
|
||||
//
|
||||
// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
|
||||
func ExtractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
|
||||
func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
|
||||
for i := range fields {
|
||||
f := &fields[i]
|
||||
if f.Name != timeField {
|
||||
continue
|
||||
}
|
||||
nsecs, err := parseTimestamp(f.Value)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", timeField, err)
|
||||
if f.Value == "" || f.Value == "0" {
|
||||
return time.Now().UnixNano(), nil
|
||||
}
|
||||
nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp from %s=%q", timeField, f.Value)
|
||||
}
|
||||
f.Value = ""
|
||||
if nsecs == 0 {
|
||||
nsecs = time.Now().UnixNano()
|
||||
}
|
||||
return nsecs, nil
|
||||
}
|
||||
return time.Now().UnixNano(), nil
|
||||
}
|
||||
|
||||
func parseTimestamp(s string) (int64, error) {
|
||||
if s == "" || s == "0" {
|
||||
return time.Now().UnixNano(), nil
|
||||
}
|
||||
if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
|
||||
return ParseUnixTimestamp(s)
|
||||
}
|
||||
nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp %q", s)
|
||||
}
|
||||
return nsecs, nil
|
||||
}
|
||||
|
||||
// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
|
||||
func ParseUnixTimestamp(s string) (int64, error) {
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
|
||||
}
|
||||
if n < (1<<31) && n >= (-1<<31) {
|
||||
// The timestamp is in seconds.
|
||||
return n * 1e9, nil
|
||||
}
|
||||
if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
|
||||
// The timestamp is in milliseconds.
|
||||
return n * 1e6, nil
|
||||
}
|
||||
if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
|
||||
// The timestamp is in microseconds.
|
||||
return n * 1e3, nil
|
||||
}
|
||||
// The timestamp is in nanoseconds
|
||||
return n, nil
|
||||
}
|
||||
|
||||
@@ -6,11 +6,11 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
)
|
||||
|
||||
func TestExtractTimestampFromFields_Success(t *testing.T) {
|
||||
func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
|
||||
f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
|
||||
t.Helper()
|
||||
|
||||
nsecs, err := ExtractTimestampFromFields(timeField, fields)
|
||||
nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
@@ -27,63 +27,35 @@ func TestExtractTimestampFromFields_Success(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// UTC time
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "2024-06-18T23:37:20Z"},
|
||||
}, 1718753840000000000)
|
||||
|
||||
// Time with timezone
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
|
||||
}, 1718725040000000000)
|
||||
|
||||
// SQL datetime format
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "2024-06-18 23:37:20.123-05:30"},
|
||||
{Name: "time", Value: "2024-06-18T23:37:20.123-05:30"},
|
||||
}, 1718773640123000000)
|
||||
|
||||
// Time with nanosecond precision
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
|
||||
{Name: "foo", Value: "bar"},
|
||||
}, 1718773640123456789)
|
||||
|
||||
// Unix timestamp in nanoseconds
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "1718773640123456789"},
|
||||
}, 1718773640123456789)
|
||||
|
||||
// Unix timestamp in microseconds
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "1718773640123456"},
|
||||
}, 1718773640123456000)
|
||||
|
||||
// Unix timestamp in milliseconds
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "1718773640123"},
|
||||
}, 1718773640123000000)
|
||||
|
||||
// Unix timestamp in seconds
|
||||
f("time", []logstorage.Field{
|
||||
{Name: "foo", Value: "bar"},
|
||||
{Name: "time", Value: "1718773640"},
|
||||
}, 1718773640000000000)
|
||||
}
|
||||
|
||||
func TestExtractTimestampFromFields_Error(t *testing.T) {
|
||||
func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
fields := []logstorage.Field{
|
||||
{Name: "time", Value: s},
|
||||
}
|
||||
nsecs, err := ExtractTimestampFromFields("time", fields)
|
||||
nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
}
|
||||
@@ -92,9 +64,11 @@ func TestExtractTimestampFromFields_Error(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// invalid time
|
||||
f("foobar")
|
||||
|
||||
// no Z at the end
|
||||
f("2024-06-18T23:37:20")
|
||||
|
||||
// incomplete time
|
||||
f("2024-06-18")
|
||||
f("2024-06-18T23:37")
|
||||
|
||||
@@ -1,250 +0,0 @@
|
||||
package journald
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
const (
|
||||
journaldEntryMaxNameLen = 64
|
||||
)
|
||||
|
||||
var (
|
||||
bodyBufferPool bytesutil.ByteBufferPool
|
||||
allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]+`)
|
||||
)
|
||||
|
||||
var (
|
||||
journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Journal fields to be used as stream fields. "+
|
||||
"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
|
||||
journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Journal fields to ignore. "+
|
||||
"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
|
||||
journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Journal field to be used as time field. "+
|
||||
"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
|
||||
journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint.")
|
||||
journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include journal entry fields, which with double underscores.")
|
||||
)
|
||||
|
||||
func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
|
||||
tenantID, err := logstorage.ParseTenantID(*journaldTenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse -journald.tenantID=%q for journald: %w", *journaldTenantID, err)
|
||||
}
|
||||
cp.TenantID = tenantID
|
||||
}
|
||||
if cp.TimeField != "" {
|
||||
cp.TimeField = *journaldTimeField
|
||||
}
|
||||
if len(cp.StreamFields) == 0 {
|
||||
cp.StreamFields = *journaldStreamFields
|
||||
}
|
||||
if len(cp.IgnoreFields) == 0 {
|
||||
cp.IgnoreFields = *journaldIgnoreFields
|
||||
}
|
||||
cp.MsgFields = []string{"MESSAGE"}
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
// RequestHandler processes Journald Export insert requests
|
||||
func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
case "/upload":
|
||||
if r.Header.Get("Content-Type") != "application/vnd.fdo.journal" {
|
||||
httpserver.Errorf(w, r, "only application/vnd.fdo.journal encoding is supported for Journald")
|
||||
return true
|
||||
}
|
||||
handleJournald(r, w)
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// handleJournald parses Journal binary entries
|
||||
func handleJournald(r *http.Request, w http.ResponseWriter) {
|
||||
startTime := time.Now()
|
||||
requestsJournaldTotal.Inc()
|
||||
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
|
||||
reader := r.Body
|
||||
var err error
|
||||
|
||||
wcr := writeconcurrencylimiter.GetReader(reader)
|
||||
data, err := io.ReadAll(wcr)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read request body: %s", err)
|
||||
return
|
||||
}
|
||||
writeconcurrencylimiter.PutReader(wcr)
|
||||
bb := bodyBufferPool.Get()
|
||||
defer bodyBufferPool.Put(bb)
|
||||
if r.Header.Get("Content-Encoding") == "zstd" {
|
||||
bb.B, err = zstd.Decompress(bb.B[:0], data)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot decompress zstd-encoded request with length %d: %s", len(data), err)
|
||||
return
|
||||
}
|
||||
data = bb.B
|
||||
}
|
||||
cp, err := getCommonParams(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("journald")
|
||||
err = parseJournaldRequest(data, lmp, cp)
|
||||
lmp.MustClose()
|
||||
if err != nil {
|
||||
errorsTotal.Inc()
|
||||
httpserver.Errorf(w, r, "cannot parse Journald protobuf request: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
// update requestJournaldDuration only for successfully parsed requests
|
||||
// There is no need in updating requestJournaldDuration for request errors,
|
||||
// since their timings are usually much smaller than the timing for successful request parsing.
|
||||
requestJournaldDuration.UpdateDuration(startTime)
|
||||
}
|
||||
|
||||
var (
|
||||
requestsJournaldTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
|
||||
|
||||
requestJournaldDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
|
||||
)
|
||||
|
||||
// See https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
|
||||
func parseJournaldRequest(data []byte, lmp insertutils.LogMessageProcessor, cp *insertutils.CommonParams) error {
|
||||
var fields []logstorage.Field
|
||||
var ts int64
|
||||
var size uint64
|
||||
var name, value string
|
||||
var line []byte
|
||||
|
||||
currentTimestamp := time.Now().UnixNano()
|
||||
|
||||
for len(data) > 0 {
|
||||
idx := bytes.IndexByte(data, '\n')
|
||||
switch {
|
||||
case idx > 0:
|
||||
// process fields
|
||||
line = data[:idx]
|
||||
data = data[idx+1:]
|
||||
case idx == 0:
|
||||
// next message or end of file
|
||||
// double new line is a separator for the next message
|
||||
if len(fields) > 0 {
|
||||
if ts == 0 {
|
||||
ts = currentTimestamp
|
||||
}
|
||||
lmp.AddRow(ts, fields, nil)
|
||||
fields = fields[:0]
|
||||
}
|
||||
// skip newline separator
|
||||
data = data[1:]
|
||||
continue
|
||||
case idx < 0:
|
||||
return fmt.Errorf("missing new line separator, unread data left=%d", len(data))
|
||||
}
|
||||
|
||||
idx = bytes.IndexByte(line, '=')
|
||||
// could b either e key=value\n pair
|
||||
// or just key\n
|
||||
// with binary data at the buffer
|
||||
if idx > 0 {
|
||||
name = bytesutil.ToUnsafeString(line[:idx])
|
||||
value = bytesutil.ToUnsafeString(line[idx+1:])
|
||||
} else {
|
||||
name = bytesutil.ToUnsafeString(line)
|
||||
if len(data) == 0 {
|
||||
return fmt.Errorf("unexpected zero data for binary field value of key=%s", name)
|
||||
}
|
||||
// size of binary data encoded as le i64 at the begging
|
||||
idx, err := binary.Decode(data, binary.LittleEndian, &size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to extract binary field %q value size: %w", name, err)
|
||||
}
|
||||
// skip binary data sise
|
||||
data = data[idx:]
|
||||
if size == 0 {
|
||||
return fmt.Errorf("unexpected zero binary data size decoded %d", size)
|
||||
}
|
||||
if int(size) > len(data) {
|
||||
return fmt.Errorf("binary data size=%d cannot exceed size of the data at buffer=%d", size, len(data))
|
||||
}
|
||||
value = bytesutil.ToUnsafeString(data[:size])
|
||||
data = data[int(size):]
|
||||
// binary data must has new line separator for the new line or next field
|
||||
if len(data) == 0 {
|
||||
return fmt.Errorf("unexpected empty buffer after binary field=%s read", name)
|
||||
}
|
||||
lastB := data[0]
|
||||
if lastB != '\n' {
|
||||
return fmt.Errorf("expected new line separator after binary field=%s, got=%s", name, string(lastB))
|
||||
}
|
||||
data = data[1:]
|
||||
}
|
||||
// https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
|
||||
if len(name) > journaldEntryMaxNameLen {
|
||||
return fmt.Errorf("journald entry name should not exceed %d symbols, got: %q", journaldEntryMaxNameLen, name)
|
||||
}
|
||||
if !allowedJournaldEntryNameChars.MatchString(name) {
|
||||
return fmt.Errorf("journald entry name should consist of `A-Z0-9_` characters and must start from non-digit symbol")
|
||||
}
|
||||
if name == cp.TimeField {
|
||||
n, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse Journald timestamp, %w", err)
|
||||
}
|
||||
ts = n * 1e3
|
||||
continue
|
||||
}
|
||||
|
||||
if slices.Contains(cp.MsgFields, name) {
|
||||
name = "_msg"
|
||||
}
|
||||
|
||||
if *journaldIncludeEntryMetadata || !strings.HasPrefix(name, "__") {
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: name,
|
||||
Value: value,
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(fields) > 0 {
|
||||
if ts == 0 {
|
||||
ts = currentTimestamp
|
||||
}
|
||||
lmp.AddRow(ts, fields, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package journald
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
)
|
||||
|
||||
func TestPushJournaldOk(t *testing.T) {
|
||||
f := func(src string, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
cp := &insertutils.CommonParams{
|
||||
TimeField: "__REALTIME_TIMESTAMP",
|
||||
MsgFields: []string{"MESSAGE"},
|
||||
}
|
||||
if err := parseJournaldRequest([]byte(src), tlp, cp); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Single event
|
||||
f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n",
|
||||
[]int64{91723819283000},
|
||||
"{\"_msg\":\"Test message\"}",
|
||||
)
|
||||
|
||||
// Multiple events
|
||||
f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n",
|
||||
[]int64{91723819283000, 91723819284000},
|
||||
"{\"_msg\":\"Test message\"}\n{\"_msg\":\"Test message2\"}",
|
||||
)
|
||||
|
||||
// Parse binary data
|
||||
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
|
||||
[]int64{1729698775704404000},
|
||||
"{\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
|
||||
)
|
||||
}
|
||||
|
||||
func TestPushJournald_Failure(t *testing.T) {
|
||||
f := func(data string) {
|
||||
t.Helper()
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
cp := &insertutils.CommonParams{
|
||||
TimeField: "__REALTIME_TIMESTAMP",
|
||||
MsgFields: []string{"MESSAGE"},
|
||||
}
|
||||
if err := parseJournaldRequest([]byte(data), tlp, cp); err == nil {
|
||||
t.Fatalf("expected non nil error")
|
||||
}
|
||||
}
|
||||
// missing new line terminator for binary encoded message
|
||||
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasda2")
|
||||
// missing new line terminator
|
||||
f("__REALTIME_TIMESTAMP=91723819283\n=Test message")
|
||||
// empty field name
|
||||
f("__REALTIME_TIMESTAMP=91723819283\n=Test message\n")
|
||||
// field name starting with number
|
||||
f("__REALTIME_TIMESTAMP=91723819283\n1incorrect=Test message\n")
|
||||
// field name exceeds 64 limit
|
||||
f("__REALTIME_TIMESTAMP=91723819283\ntoolooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongcorrecooooooooooooong=Test message\n")
|
||||
// Only allow A-Z0-9 and '_'
|
||||
f("__REALTIME_TIMESTAMP=91723819283\nbadC!@$!@$as=Test message\n")
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package jsonline
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -8,6 +10,7 @@ import (
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
@@ -49,9 +52,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
|
||||
reader = zr
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("jsonline")
|
||||
streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
|
||||
err = processStreamInternal(streamName, reader, cp.TimeField, cp.MsgFields, lmp)
|
||||
lmp := cp.NewLogMessageProcessor()
|
||||
err = processStreamInternal(reader, cp.TimeField, cp.MsgField, lmp)
|
||||
lmp.MustClose()
|
||||
|
||||
if err != nil {
|
||||
@@ -64,15 +66,20 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) error {
|
||||
func processStreamInternal(r io.Reader, timeField, msgField string, lmp insertutils.LogMessageProcessor) error {
|
||||
wcr := writeconcurrencylimiter.GetReader(r)
|
||||
defer writeconcurrencylimiter.PutReader(wcr)
|
||||
|
||||
lr := insertutils.NewLineReader(streamName, wcr)
|
||||
lb := lineBufferPool.Get()
|
||||
defer lineBufferPool.Put(lb)
|
||||
|
||||
lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
|
||||
sc := bufio.NewScanner(wcr)
|
||||
sc.Buffer(lb.B, len(lb.B))
|
||||
|
||||
n := 0
|
||||
for {
|
||||
ok, err := readLine(lr, timeField, msgFields, lmp)
|
||||
ok, err := readLine(sc, timeField, msgField, lmp)
|
||||
wcr.DecConcurrency()
|
||||
if err != nil {
|
||||
errorsTotal.Inc()
|
||||
@@ -82,35 +89,45 @@ func processStreamInternal(streamName string, r io.Reader, timeField string, msg
|
||||
return nil
|
||||
}
|
||||
n++
|
||||
rowsIngestedTotal.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
func readLine(lr *insertutils.LineReader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (bool, error) {
|
||||
func readLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
|
||||
var line []byte
|
||||
for len(line) == 0 {
|
||||
if !lr.NextLine() {
|
||||
err := lr.Err()
|
||||
return false, err
|
||||
if !sc.Scan() {
|
||||
if err := sc.Err(); err != nil {
|
||||
if errors.Is(err, bufio.ErrTooLong) {
|
||||
return false, fmt.Errorf(`cannot read json line, since its size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
line = lr.Line
|
||||
line = sc.Bytes()
|
||||
}
|
||||
|
||||
p := logstorage.GetJSONParser()
|
||||
if err := p.ParseLogMessage(line); err != nil {
|
||||
return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
|
||||
}
|
||||
ts, err := insertutils.ExtractTimestampFromFields(timeField, p.Fields)
|
||||
ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot get timestamp: %w", err)
|
||||
}
|
||||
logstorage.RenameField(p.Fields, msgFields, "_msg")
|
||||
lmp.AddRow(ts, p.Fields, nil)
|
||||
logstorage.RenameField(p.Fields, msgField, "_msg")
|
||||
lmp.AddRow(ts, p.Fields)
|
||||
logstorage.PutJSONParser(p)
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
var lineBufferPool bytesutil.ByteBufferPool
|
||||
|
||||
var (
|
||||
rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
|
||||
|
||||
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
|
||||
|
||||
|
||||
@@ -8,43 +8,32 @@ import (
|
||||
)
|
||||
|
||||
func TestProcessStreamInternal_Success(t *testing.T) {
|
||||
f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
|
||||
f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
msgFields := []string{msgField}
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
r := bytes.NewBufferString(data)
|
||||
if err := processStreamInternal("test", r, timeField, msgFields, tlp); err != nil {
|
||||
if err := processStreamInternal(r, timeField, msgField, tlp); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
|
||||
{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
|
||||
{"message":"xyz","@timestamp":"2023-06-06 04:48:13.735Z","x":"y"}
|
||||
{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
|
||||
{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
|
||||
`
|
||||
timeField := "@timestamp"
|
||||
msgField := "message"
|
||||
timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
|
||||
resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"_msg":"baz"}
|
||||
{"_msg":"xyz","x":"y"}`
|
||||
f(data, timeField, msgField, timestampsExpected, resultExpected)
|
||||
|
||||
// Non-existing msgField
|
||||
data = `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
|
||||
{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
|
||||
`
|
||||
timeField = "@timestamp"
|
||||
msgField = "foobar"
|
||||
timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
|
||||
resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","message":"foobar"}
|
||||
{"message":"baz"}`
|
||||
f(data, timeField, msgField, timestampsExpected, resultExpected)
|
||||
rowsExpected := 3
|
||||
timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
|
||||
resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
|
||||
{"@timestamp":"","_msg":"baz"}
|
||||
{"_msg":"xyz","@timestamp":"","x":"y"}`
|
||||
f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
func TestProcessStreamInternal_Failure(t *testing.T) {
|
||||
@@ -53,7 +42,7 @@ func TestProcessStreamInternal_Failure(t *testing.T) {
|
||||
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
r := bytes.NewBufferString(data)
|
||||
if err := processStreamInternal("test", r, "time", nil, tlp); err == nil {
|
||||
if err := processStreamInternal(r, "time", "", tlp); err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,9 +8,6 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
@@ -18,6 +15,8 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson"
|
||||
)
|
||||
|
||||
var parserPool fastjson.ParserPool
|
||||
@@ -53,15 +52,16 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
lmp := cp.NewLogMessageProcessor("loki_json")
|
||||
useDefaultStreamFields := len(cp.StreamFields) == 0
|
||||
err = parseJSONRequest(data, lmp, useDefaultStreamFields)
|
||||
lmp := cp.NewLogMessageProcessor()
|
||||
n, err := parseJSONRequest(data, lmp)
|
||||
lmp.MustClose()
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse Loki json request: %s; data=%s", err, data)
|
||||
httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
rowsIngestedJSONTotal.Add(n)
|
||||
|
||||
// update requestJSONDuration only for successfully parsed requests
|
||||
// There is no need in updating requestJSONDuration for request errors,
|
||||
// since their timings are usually much smaller than the timing for successful request parsing.
|
||||
@@ -69,29 +69,31 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
|
||||
}
|
||||
|
||||
var (
|
||||
requestsJSONTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
requestsJSONTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
rowsIngestedJSONTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
|
||||
requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
)
|
||||
|
||||
func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
|
||||
func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
|
||||
p := parserPool.Get()
|
||||
defer parserPool.Put(p)
|
||||
v, err := p.ParseBytes(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse JSON request body: %w", err)
|
||||
return 0, fmt.Errorf("cannot parse JSON request body: %w", err)
|
||||
}
|
||||
|
||||
streamsV := v.Get("streams")
|
||||
if streamsV == nil {
|
||||
return fmt.Errorf("missing `streams` item in the parsed JSON")
|
||||
return 0, fmt.Errorf("missing `streams` item in the parsed JSON: %q", v)
|
||||
}
|
||||
streams, err := streamsV.Array()
|
||||
if err != nil {
|
||||
return fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
|
||||
return 0, fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
|
||||
}
|
||||
|
||||
currentTimestamp := time.Now().UnixNano()
|
||||
var commonFields []logstorage.Field
|
||||
rowsIngested := 0
|
||||
for _, stream := range streams {
|
||||
// populate common labels from `stream` dict
|
||||
commonFields = commonFields[:0]
|
||||
@@ -100,11 +102,14 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
|
||||
if labelsV != nil {
|
||||
o, err := labelsV.Object()
|
||||
if err != nil {
|
||||
return fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
|
||||
return rowsIngested, fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
|
||||
}
|
||||
labels = o
|
||||
}
|
||||
labels.Visit(func(k []byte, v *fastjson.Value) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
vStr, errLocal := v.StringBytes()
|
||||
if errLocal != nil {
|
||||
err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
|
||||
@@ -116,37 +121,37 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error when parsing `stream` object: %w", err)
|
||||
return rowsIngested, fmt.Errorf("error when parsing `stream` object: %w", err)
|
||||
}
|
||||
|
||||
// populate messages from `values` array
|
||||
linesV := stream.Get("values")
|
||||
if linesV == nil {
|
||||
return fmt.Errorf("missing `values` item in the parsed `stream` object %q", stream)
|
||||
return rowsIngested, fmt.Errorf("missing `values` item in the parsed JSON %q", stream)
|
||||
}
|
||||
lines, err := linesV.Array()
|
||||
if err != nil {
|
||||
return fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
|
||||
return rowsIngested, fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
|
||||
}
|
||||
|
||||
fields := commonFields
|
||||
for _, line := range lines {
|
||||
lineA, err := line.Array()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
|
||||
return rowsIngested, fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
|
||||
}
|
||||
if len(lineA) < 2 || len(lineA) > 3 {
|
||||
return fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2 or 3", line, len(lineA))
|
||||
if len(lineA) != 2 {
|
||||
return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2", line, len(lineA))
|
||||
}
|
||||
|
||||
// parse timestamp
|
||||
timestamp, err := lineA[0].StringBytes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
|
||||
return rowsIngested, fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
|
||||
}
|
||||
ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
|
||||
return rowsIngested, fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
|
||||
}
|
||||
if ts == 0 {
|
||||
ts = currentTimestamp
|
||||
@@ -155,46 +160,19 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
|
||||
// parse log message
|
||||
msg, err := lineA[1].StringBytes()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
|
||||
return rowsIngested, fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
|
||||
}
|
||||
|
||||
fields = append(fields[:len(commonFields)], logstorage.Field{
|
||||
Name: "_msg",
|
||||
Value: bytesutil.ToUnsafeString(msg),
|
||||
})
|
||||
|
||||
// parse structured metadata - see https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
|
||||
if len(lineA) > 2 {
|
||||
structuredMetadata, err := lineA[2].Object()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unexpected structured metadata type for %q; want JSON object", lineA[2])
|
||||
}
|
||||
|
||||
structuredMetadata.Visit(func(k []byte, v *fastjson.Value) {
|
||||
vStr, errLocal := v.StringBytes()
|
||||
if errLocal != nil {
|
||||
err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
|
||||
return
|
||||
}
|
||||
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: bytesutil.ToUnsafeString(k),
|
||||
Value: bytesutil.ToUnsafeString(vStr),
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error when parsing `structuredMetadata` object: %w", err)
|
||||
}
|
||||
}
|
||||
var streamFields []logstorage.Field
|
||||
if useDefaultStreamFields {
|
||||
streamFields = commonFields
|
||||
}
|
||||
lmp.AddRow(ts, fields, streamFields)
|
||||
lmp.AddRow(ts, fields)
|
||||
}
|
||||
rowsIngested += len(lines)
|
||||
}
|
||||
|
||||
return nil
|
||||
return rowsIngested, nil
|
||||
}
|
||||
|
||||
func parseLokiTimestamp(s string) (int64, error) {
|
||||
|
||||
@@ -11,11 +11,12 @@ func TestParseJSONRequest_Failure(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
if err := parseJSONRequest([]byte(s), tlp, false); err == nil {
|
||||
n, err := parseJSONRequest([]byte(s), tlp)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
}
|
||||
if err := tlp.Verify(nil, ""); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
if n != 0 {
|
||||
t.Fatalf("unexpected number of parsed lines: %d; want 0", n)
|
||||
}
|
||||
}
|
||||
f(``)
|
||||
@@ -44,19 +45,13 @@ func TestParseJSONRequest_Failure(t *testing.T) {
|
||||
// Invalid length of `values` individual item
|
||||
f(`{"streams":[{"values":[[]]}]}`)
|
||||
f(`{"streams":[{"values":[["123"]]}]}`)
|
||||
f(`{"streams":[{"values":[["123","456","789","8123"]]}]}`)
|
||||
f(`{"streams":[{"values":[["123","456","789"]]}]}`)
|
||||
|
||||
// Invalid type for timestamp inside `values` individual item
|
||||
f(`{"streams":[{"values":[[123,"456"]}]}`)
|
||||
|
||||
// Invalid type for log message
|
||||
f(`{"streams":[{"values":[["123",1234]]}]}`)
|
||||
|
||||
// invalid structured metadata type
|
||||
f(`{"streams":[{"values":[["1577836800000000001", "foo bar", ["metadata_1", "md_value"]]]}]}`)
|
||||
|
||||
// structured metadata with unexpected value type
|
||||
f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": 1}]] }]}`)
|
||||
}
|
||||
|
||||
func TestParseJSONRequest_Success(t *testing.T) {
|
||||
@@ -65,10 +60,11 @@ func TestParseJSONRequest_Success(t *testing.T) {
|
||||
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
|
||||
if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
|
||||
n, err := parseJSONRequest([]byte(s), tlp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -120,8 +116,4 @@ func TestParseJSONRequest_Success(t *testing.T) {
|
||||
}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
|
||||
{"foo":"bar","a":"b","_msg":"abc"}
|
||||
{"x":"y","_msg":"yx"}`)
|
||||
|
||||
// values with metadata
|
||||
f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar","metadata_1":"md_value"}`)
|
||||
f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
|
||||
}
|
||||
|
||||
@@ -28,7 +28,8 @@ func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
data := getJSONBody(streams, rows, labels)
|
||||
for pb.Next() {
|
||||
if err := parseJSONRequest(data, blp, false); err != nil {
|
||||
_, err := parseJSONRequest(data, blp)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,15 +44,16 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
lmp := cp.NewLogMessageProcessor("loki_protobuf")
|
||||
useDefaultStreamFields := len(cp.StreamFields) == 0
|
||||
err = parseProtobufRequest(data, lmp, useDefaultStreamFields)
|
||||
lmp := cp.NewLogMessageProcessor()
|
||||
n, err := parseProtobufRequest(data, lmp)
|
||||
lmp.MustClose()
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
rowsIngestedProtobufTotal.Add(n)
|
||||
|
||||
// update requestProtobufDuration only for successfully parsed requests
|
||||
// There is no need in updating requestProtobufDuration for request errors,
|
||||
// since their timings are usually much smaller than the timing for successful request parsing.
|
||||
@@ -60,17 +61,18 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
}
|
||||
|
||||
var (
|
||||
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
|
||||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
)
|
||||
|
||||
func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
|
||||
func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
|
||||
bb := bytesBufPool.Get()
|
||||
defer bytesBufPool.Put(bb)
|
||||
|
||||
buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
|
||||
return 0, fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
|
||||
}
|
||||
bb.B = buf
|
||||
|
||||
@@ -79,12 +81,13 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
|
||||
|
||||
err = req.UnmarshalProtobuf(bb.B)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse request body: %w", err)
|
||||
return 0, fmt.Errorf("cannot parse request body: %w", err)
|
||||
}
|
||||
|
||||
fields := getFields()
|
||||
defer putFields(fields)
|
||||
|
||||
rowsIngested := 0
|
||||
streams := req.Streams
|
||||
currentTimestamp := time.Now().UnixNano()
|
||||
for i := range streams {
|
||||
@@ -93,7 +96,7 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
|
||||
// Labels are same for all entries in the stream.
|
||||
fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
|
||||
return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
|
||||
}
|
||||
commonFieldsLen := len(fields.fields)
|
||||
|
||||
@@ -119,14 +122,11 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
|
||||
ts = currentTimestamp
|
||||
}
|
||||
|
||||
var streamFields []logstorage.Field
|
||||
if useDefaultStreamFields {
|
||||
streamFields = fields.fields[:commonFieldsLen]
|
||||
}
|
||||
lmp.AddRow(ts, fields.fields, streamFields)
|
||||
lmp.AddRow(ts, fields.fields)
|
||||
}
|
||||
rowsIngested += len(stream.Entries)
|
||||
}
|
||||
return nil
|
||||
return rowsIngested, nil
|
||||
}
|
||||
|
||||
func getFields() *fields {
|
||||
|
||||
@@ -15,10 +15,7 @@ type testLogMessageProcessor struct {
|
||||
pr PushRequest
|
||||
}
|
||||
|
||||
func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
|
||||
if streamFields != nil {
|
||||
panic(fmt.Errorf("unexpected non-nil streamFields: %v", streamFields))
|
||||
}
|
||||
func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
|
||||
msg := ""
|
||||
for _, f := range fields {
|
||||
if f.Name == "_msg" {
|
||||
@@ -53,21 +50,23 @@ func TestParseProtobufRequest_Success(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
tlp := &testLogMessageProcessor{}
|
||||
if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
|
||||
n, err := parseJSONRequest([]byte(s), tlp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if len(tlp.pr.Streams) != len(timestampsExpected) {
|
||||
t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
|
||||
if n != len(tlp.pr.Streams) {
|
||||
t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
|
||||
}
|
||||
|
||||
data := tlp.pr.MarshalProtobuf(nil)
|
||||
encodedData := snappy.Encode(nil, data)
|
||||
|
||||
tlp2 := &insertutils.TestLogMessageProcessor{}
|
||||
if err := parseProtobufRequest(encodedData, tlp2, false); err != nil {
|
||||
n, err = parseProtobufRequest(encodedData, tlp2)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
if err := tlp2.Verify(n, timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,8 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
body := getProtobufBody(streams, rows, labels)
|
||||
for pb.Next() {
|
||||
if err := parseProtobufRequest(body, blp, false); err != nil {
|
||||
_, err := parseProtobufRequest(body, blp)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ var mp easyproto.MarshalerPool
|
||||
|
||||
// PushRequest represents Loki PushRequest
|
||||
//
|
||||
// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L14
|
||||
// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
|
||||
type PushRequest struct {
|
||||
Streams []Stream
|
||||
|
||||
@@ -87,7 +87,7 @@ func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []Labe
|
||||
|
||||
// Stream represents Loki stream.
|
||||
//
|
||||
// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L23
|
||||
// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
|
||||
type Stream struct {
|
||||
Labels string
|
||||
Entries []Entry
|
||||
@@ -139,7 +139,7 @@ func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair,
|
||||
|
||||
// Entry represents Loki entry.
|
||||
//
|
||||
// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L38
|
||||
// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
|
||||
type Entry struct {
|
||||
Timestamp time.Time
|
||||
Line string
|
||||
@@ -203,7 +203,7 @@ func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]Label
|
||||
|
||||
// LabelPair represents Loki label pair.
|
||||
//
|
||||
// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L33
|
||||
// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
|
||||
type LabelPair struct {
|
||||
Name string
|
||||
Value string
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
package vlinsert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/opentelemetry"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
|
||||
)
|
||||
|
||||
@@ -27,7 +23,6 @@ func Stop() {
|
||||
// RequestHandler handles insert requests for VictoriaLogs
|
||||
func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
path := r.URL.Path
|
||||
|
||||
if !strings.HasPrefix(path, "/insert/") {
|
||||
// Skip requests, which do not start with /insert/, since these aren't our requests.
|
||||
return false
|
||||
@@ -35,15 +30,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
path = strings.TrimPrefix(path, "/insert")
|
||||
path = strings.ReplaceAll(path, "//", "/")
|
||||
|
||||
switch path {
|
||||
case "/jsonline":
|
||||
if path == "/jsonline" {
|
||||
jsonline.RequestHandler(w, r)
|
||||
return true
|
||||
case "/ready":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(200)
|
||||
fmt.Fprintf(w, `{"status":"ok"}`)
|
||||
return true
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(path, "/elasticsearch/"):
|
||||
@@ -52,15 +41,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
case strings.HasPrefix(path, "/loki/"):
|
||||
path = strings.TrimPrefix(path, "/loki")
|
||||
return loki.RequestHandler(path, w, r)
|
||||
case strings.HasPrefix(path, "/opentelemetry/"):
|
||||
path = strings.TrimPrefix(path, "/opentelemetry")
|
||||
return opentelemetry.RequestHandler(path, w, r)
|
||||
case strings.HasPrefix(path, "/journald/"):
|
||||
path = strings.TrimPrefix(path, "/journald")
|
||||
return journald.RequestHandler(path, w, r)
|
||||
case strings.HasPrefix(path, "/datadog/"):
|
||||
path = strings.TrimPrefix(path, "/datadog")
|
||||
return datadog.RequestHandler(path, w, r)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
package opentelemetry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
// RequestHandler processes Opentelemetry insert requests
|
||||
func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
// use the same path as opentelemetry collector
|
||||
// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
|
||||
case "/v1/logs":
|
||||
if r.Header.Get("Content-Type") == "application/json" {
|
||||
httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
|
||||
return true
|
||||
}
|
||||
handleProtobuf(r, w)
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
startTime := time.Now()
|
||||
requestsProtobufTotal.Inc()
|
||||
reader := r.Body
|
||||
if r.Header.Get("Content-Encoding") == "gzip" {
|
||||
zr, err := common.GetGzipReader(reader)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
|
||||
return
|
||||
}
|
||||
defer common.PutGzipReader(zr)
|
||||
reader = zr
|
||||
}
|
||||
|
||||
wcr := writeconcurrencylimiter.GetReader(reader)
|
||||
data, err := io.ReadAll(wcr)
|
||||
writeconcurrencylimiter.PutReader(wcr)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot read request body: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
cp, err := insertutils.GetCommonParams(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
|
||||
return
|
||||
}
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf")
|
||||
useDefaultStreamFields := len(cp.StreamFields) == 0
|
||||
err = pushProtobufRequest(data, lmp, useDefaultStreamFields)
|
||||
lmp.MustClose()
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot parse OpenTelemetry protobuf request: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
// update requestProtobufDuration only for successfully parsed requests
|
||||
// There is no need in updating requestProtobufDuration for request errors,
|
||||
// since their timings are usually much smaller than the timing for successful request parsing.
|
||||
requestProtobufDuration.UpdateDuration(startTime)
|
||||
}
|
||||
|
||||
var (
|
||||
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
|
||||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
)
|
||||
|
||||
func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
|
||||
var req pb.ExportLogsServiceRequest
|
||||
if err := req.UnmarshalProtobuf(data); err != nil {
|
||||
errorsTotal.Inc()
|
||||
return fmt.Errorf("cannot unmarshal request from %d bytes: %w", len(data), err)
|
||||
}
|
||||
|
||||
var commonFields []logstorage.Field
|
||||
for _, rl := range req.ResourceLogs {
|
||||
attributes := rl.Resource.Attributes
|
||||
commonFields = slicesutil.SetLength(commonFields, len(attributes))
|
||||
for i, attr := range attributes {
|
||||
commonFields[i].Name = attr.Key
|
||||
commonFields[i].Value = attr.Value.FormatString()
|
||||
}
|
||||
commonFieldsLen := len(commonFields)
|
||||
for _, sc := range rl.ScopeLogs {
|
||||
commonFields = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp, useDefaultStreamFields)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) []logstorage.Field {
|
||||
fields := commonFields
|
||||
for _, lr := range sc.LogRecords {
|
||||
fields = fields[:len(commonFields)]
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: "_msg",
|
||||
Value: lr.Body.FormatString(),
|
||||
})
|
||||
for _, attr := range lr.Attributes {
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: attr.Key,
|
||||
Value: attr.Value.FormatString(),
|
||||
})
|
||||
}
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: "severity",
|
||||
Value: lr.FormatSeverity(),
|
||||
})
|
||||
|
||||
var streamFields []logstorage.Field
|
||||
if useDefaultStreamFields {
|
||||
streamFields = commonFields
|
||||
}
|
||||
lmp.AddRow(lr.ExtractTimestampNano(), fields, streamFields)
|
||||
}
|
||||
return fields
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package opentelemetry
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
|
||||
)
|
||||
|
||||
func TestPushProtoOk(t *testing.T) {
|
||||
f := func(src []pb.ResourceLogs, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
lr := pb.ExportLogsServiceRequest{
|
||||
ResourceLogs: src,
|
||||
}
|
||||
|
||||
pData := lr.MarshalProtobuf(nil)
|
||||
tlp := &insertutils.TestLogMessageProcessor{}
|
||||
if err := pushProtobufRequest(pData, tlp, false); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// single line without resource attributes
|
||||
f([]pb.ResourceLogs{
|
||||
{
|
||||
ScopeLogs: []pb.ScopeLogs{
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]int64{1234},
|
||||
`{"_msg":"log-line-message","severity":"Trace"}`,
|
||||
)
|
||||
// multi-line with resource attributes
|
||||
f([]pb.ResourceLogs{
|
||||
{
|
||||
Resource: pb.Resource{
|
||||
Attributes: []*pb.KeyValue{
|
||||
{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
|
||||
{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
|
||||
{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
|
||||
Values: []*pb.KeyValue{
|
||||
{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
|
||||
{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
|
||||
},
|
||||
}}},
|
||||
},
|
||||
},
|
||||
ScopeLogs: []pb.ScopeLogs{
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 21, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
|
||||
{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]int64{1234, 1235, 1236},
|
||||
`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
|
||||
{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}
|
||||
{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
|
||||
)
|
||||
|
||||
// multi-scope with resource attributes and multi-line
|
||||
f([]pb.ResourceLogs{
|
||||
{
|
||||
Resource: pb.Resource{
|
||||
Attributes: []*pb.KeyValue{
|
||||
{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
|
||||
{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
|
||||
{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
|
||||
Values: []*pb.KeyValue{
|
||||
{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
|
||||
{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
|
||||
},
|
||||
}}},
|
||||
},
|
||||
},
|
||||
ScopeLogs: []pb.ScopeLogs{
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
|
||||
{TimeUnixNano: 1235, SeverityNumber: 5, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ScopeLogs: []pb.ScopeLogs{
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{TimeUnixNano: 2345, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-0")}},
|
||||
{TimeUnixNano: 2346, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-1")}},
|
||||
},
|
||||
},
|
||||
{
|
||||
LogRecords: []pb.LogRecord{
|
||||
{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
|
||||
{ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]int64{1234, 1235, 2345, 2346, 2347, 2348},
|
||||
`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
|
||||
{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Debug"}
|
||||
{"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
|
||||
{"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
|
||||
{"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
|
||||
{"_msg":"log-line-resource-scope-1-1-1","severity":"Info4"}`,
|
||||
)
|
||||
}
|
||||
|
||||
func ptrTo[T any](s T) *T {
|
||||
return &s
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package opentelemetry
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
|
||||
)
|
||||
|
||||
func BenchmarkParseProtobufRequest(b *testing.B) {
|
||||
for _, scopes := range []int{1, 2} {
|
||||
for _, rows := range []int{100, 1000} {
|
||||
for _, attributes := range []int{5, 10} {
|
||||
b.Run(fmt.Sprintf("scopes_%d/rows_%d/attributes_%d", scopes, rows, attributes), func(b *testing.B) {
|
||||
benchmarkParseProtobufRequest(b, scopes, rows, attributes)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
|
||||
blp := &insertutils.BenchmarkLogMessageProcessor{}
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(streams * rows))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
body := getProtobufBody(streams, rows, labels)
|
||||
for pb.Next() {
|
||||
if err := pushProtobufRequest(body, blp, false); err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func getProtobufBody(scopesCount, rowsCount, attributesCount int) []byte {
|
||||
msg := "12345678910"
|
||||
|
||||
attrValues := []*pb.AnyValue{
|
||||
{StringValue: ptrTo("string-attribute")},
|
||||
{IntValue: ptrTo[int64](12345)},
|
||||
{DoubleValue: ptrTo(3.14)},
|
||||
}
|
||||
attrs := make([]*pb.KeyValue, attributesCount)
|
||||
for j := 0; j < attributesCount; j++ {
|
||||
attrs[j] = &pb.KeyValue{
|
||||
Key: fmt.Sprintf("key-%d", j),
|
||||
Value: attrValues[j%3],
|
||||
}
|
||||
}
|
||||
entries := make([]pb.LogRecord, rowsCount)
|
||||
for j := 0; j < rowsCount; j++ {
|
||||
entries[j] = pb.LogRecord{
|
||||
TimeUnixNano: 12345678910, ObservedTimeUnixNano: 12345678910, Body: pb.AnyValue{StringValue: &msg},
|
||||
}
|
||||
}
|
||||
scopes := make([]pb.ScopeLogs, scopesCount)
|
||||
|
||||
for j := 0; j < scopesCount; j++ {
|
||||
scopes[j] = pb.ScopeLogs{
|
||||
LogRecords: entries,
|
||||
}
|
||||
}
|
||||
|
||||
pr := pb.ExportLogsServiceRequest{
|
||||
ResourceLogs: []pb.ResourceLogs{
|
||||
{
|
||||
Resource: pb.Resource{
|
||||
Attributes: attrs,
|
||||
},
|
||||
ScopeLogs: scopes,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return pr.MarshalProtobuf(nil)
|
||||
}
|
||||
@@ -3,13 +3,11 @@ package syslog
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -37,25 +35,10 @@ var (
|
||||
syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
|
||||
"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
|
||||
|
||||
streamFieldsTCP = flagutil.NewArrayString("syslog.streamFields.tcp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
|
||||
`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
|
||||
streamFieldsUDP = flagutil.NewArrayString("syslog.streamFields.udp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.udp. "+
|
||||
`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
|
||||
|
||||
ignoreFieldsTCP = flagutil.NewArrayString("syslog.ignoreFields.tcp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.tcp. "+
|
||||
`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
|
||||
ignoreFieldsUDP = flagutil.NewArrayString("syslog.ignoreFields.udp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. "+
|
||||
`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
|
||||
|
||||
extraFieldsTCP = flagutil.NewArrayString("syslog.extraFields.tcp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. "+
|
||||
`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
|
||||
extraFieldsUDP = flagutil.NewArrayString("syslog.extraFields.udp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. "+
|
||||
`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
|
||||
|
||||
tenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
|
||||
tenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
|
||||
syslogTenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
|
||||
syslogTenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
|
||||
|
||||
listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
|
||||
@@ -167,7 +150,7 @@ func runUDPListener(addr string, argIdx int) {
|
||||
logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
|
||||
}
|
||||
|
||||
tenantIDStr := tenantIDUDP.GetOptionalArg(argIdx)
|
||||
tenantIDStr := syslogTenantIDUDP.GetOptionalArg(argIdx)
|
||||
tenantID, err := logstorage.ParseTenantID(tenantIDStr)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
|
||||
@@ -178,27 +161,9 @@ func runUDPListener(addr string, argIdx int) {
|
||||
|
||||
useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
|
||||
|
||||
streamFieldsStr := streamFieldsUDP.GetOptionalArg(argIdx)
|
||||
streamFields, err := parseFieldsList(streamFieldsStr)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot parse -syslog.streamFields.udp=%q for -syslog.listenAddr.udp=%q: %s", streamFieldsStr, addr, err)
|
||||
}
|
||||
|
||||
ignoreFieldsStr := ignoreFieldsUDP.GetOptionalArg(argIdx)
|
||||
ignoreFields, err := parseFieldsList(ignoreFieldsStr)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot parse -syslog.ignoreFields.udp=%q for -syslog.listenAddr.udp=%q: %s", ignoreFieldsStr, addr, err)
|
||||
}
|
||||
|
||||
extraFieldsStr := extraFieldsUDP.GetOptionalArg(argIdx)
|
||||
extraFields, err := parseExtraFields(extraFieldsStr)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot parse -syslog.extraFields.udp=%q for -syslog.listenAddr.udp=%q: %s", extraFieldsStr, addr, err)
|
||||
}
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
go func() {
|
||||
serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
|
||||
serveUDP(ln, tenantID, compressMethod, useLocalTimestamp)
|
||||
close(doneCh)
|
||||
}()
|
||||
|
||||
@@ -228,7 +193,7 @@ func runTCPListener(addr string, argIdx int) {
|
||||
logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
|
||||
}
|
||||
|
||||
tenantIDStr := tenantIDTCP.GetOptionalArg(argIdx)
|
||||
tenantIDStr := syslogTenantIDTCP.GetOptionalArg(argIdx)
|
||||
tenantID, err := logstorage.ParseTenantID(tenantIDStr)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
|
||||
@@ -239,27 +204,9 @@ func runTCPListener(addr string, argIdx int) {
|
||||
|
||||
useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
|
||||
|
||||
streamFieldsStr := streamFieldsTCP.GetOptionalArg(argIdx)
|
||||
streamFields, err := parseFieldsList(streamFieldsStr)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot parse -syslog.streamFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", streamFieldsStr, addr, err)
|
||||
}
|
||||
|
||||
ignoreFieldsStr := ignoreFieldsTCP.GetOptionalArg(argIdx)
|
||||
ignoreFields, err := parseFieldsList(ignoreFieldsStr)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot parse -syslog.ignoreFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", ignoreFieldsStr, addr, err)
|
||||
}
|
||||
|
||||
extraFieldsStr := extraFieldsTCP.GetOptionalArg(argIdx)
|
||||
extraFields, err := parseExtraFields(extraFieldsStr)
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot parse -syslog.extraFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", extraFieldsStr, addr, err)
|
||||
}
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
go func() {
|
||||
serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
|
||||
serveTCP(ln, tenantID, compressMethod, useLocalTimestamp)
|
||||
close(doneCh)
|
||||
}()
|
||||
|
||||
@@ -281,7 +228,7 @@ func checkCompressMethod(compressMethod, addr, protocol string) {
|
||||
}
|
||||
}
|
||||
|
||||
func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
|
||||
func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
|
||||
gomaxprocs := cgroup.AvailableCPUs()
|
||||
var wg sync.WaitGroup
|
||||
localAddr := ln.LocalAddr()
|
||||
@@ -289,7 +236,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
|
||||
cp := insertutils.GetCommonParamsForSyslog(tenantID)
|
||||
var bb bytesutil.ByteBuffer
|
||||
bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
|
||||
for {
|
||||
@@ -314,7 +261,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
|
||||
}
|
||||
bb.B = bb.B[:n]
|
||||
udpRequestsTotal.Inc()
|
||||
if err := processStream("udp", bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
|
||||
if err := processStream(bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
|
||||
logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
|
||||
}
|
||||
}
|
||||
@@ -323,7 +270,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
|
||||
func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
|
||||
var cm ingestserver.ConnsMap
|
||||
cm.Init("syslog")
|
||||
|
||||
@@ -353,8 +300,8 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
|
||||
if err := processStream("tcp", c, compressMethod, useLocalTimestamp, cp); err != nil {
|
||||
cp := insertutils.GetCommonParamsForSyslog(tenantID)
|
||||
if err := processStream(c, compressMethod, useLocalTimestamp, cp); err != nil {
|
||||
logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
|
||||
}
|
||||
|
||||
@@ -369,12 +316,12 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
|
||||
}
|
||||
|
||||
// processStream parses a stream of syslog messages from r and ingests them into vlstorage.
|
||||
func processStream(protocol string, r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
|
||||
func processStream(r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("syslog_" + protocol)
|
||||
lmp := cp.NewLogMessageProcessor()
|
||||
err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
|
||||
lmp.MustClose()
|
||||
|
||||
@@ -436,6 +383,7 @@ func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertut
|
||||
return fmt.Errorf("cannot read line #%d: %s", n, err)
|
||||
}
|
||||
n++
|
||||
rowsIngestedTotal.Inc()
|
||||
}
|
||||
return slr.Error()
|
||||
}
|
||||
@@ -560,56 +508,24 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
|
||||
if useLocalTimestamp {
|
||||
ts = time.Now().UnixNano()
|
||||
} else {
|
||||
nsecs, err := insertutils.ExtractTimestampFromFields("timestamp", p.Fields)
|
||||
nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
|
||||
}
|
||||
ts = nsecs
|
||||
}
|
||||
logstorage.RenameField(p.Fields, msgFields, "_msg")
|
||||
lmp.AddRow(ts, p.Fields, nil)
|
||||
logstorage.RenameField(p.Fields, "message", "_msg")
|
||||
lmp.AddRow(ts, p.Fields)
|
||||
logstorage.PutSyslogParser(p)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var msgFields = []string{"message"}
|
||||
|
||||
var (
|
||||
rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="syslog"}`)
|
||||
|
||||
errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`)
|
||||
|
||||
udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
|
||||
udpErrorsTotal = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
|
||||
)
|
||||
|
||||
func parseFieldsList(s string) ([]string, error) {
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var a []string
|
||||
err := json.Unmarshal([]byte(s), &a)
|
||||
return a, err
|
||||
}
|
||||
|
||||
func parseExtraFields(s string) ([]logstorage.Field, error) {
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal([]byte(s), &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fields := make([]logstorage.Field, 0, len(m))
|
||||
for k, v := range m {
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: k,
|
||||
Value: v,
|
||||
})
|
||||
}
|
||||
sort.Slice(fields, func(i, j int) bool {
|
||||
return fields[i].Name < fields[j].Name
|
||||
})
|
||||
return fields, nil
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestSyslogLineReader_Failure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestProcessStreamInternal_Success(t *testing.T) {
|
||||
f := func(data string, currentYear int, timestampsExpected []int64, resultExpected string) {
|
||||
f := func(data string, currentYear, rowsExpected int, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
MustInit()
|
||||
@@ -89,7 +89,7 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
||||
if err := processStreamInternal(r, "", false, tlp); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
|
||||
if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -99,11 +99,12 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
||||
48 <165>Jun 4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
|
||||
`
|
||||
currentYear := 2023
|
||||
rowsExpected := 3
|
||||
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
|
||||
resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
|
||||
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
|
||||
f(data, currentYear, timestampsExpected, resultExpected)
|
||||
resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
|
||||
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
|
||||
f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
func TestProcessStreamInternal_Failure(t *testing.T) {
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
# All these commands must run from repository root.
|
||||
|
||||
vlogscli:
|
||||
APP_NAME=vlogscli $(MAKE) app-local
|
||||
|
||||
vlogscli-race:
|
||||
APP_NAME=vlogscli RACE=-race $(MAKE) app-local
|
||||
|
||||
vlogscli-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker
|
||||
|
||||
vlogscli-pure-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-pure
|
||||
|
||||
vlogscli-linux-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-amd64
|
||||
|
||||
vlogscli-linux-arm-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm
|
||||
|
||||
vlogscli-linux-arm64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm64
|
||||
|
||||
vlogscli-linux-ppc64le-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-ppc64le
|
||||
|
||||
vlogscli-linux-386-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-linux-386
|
||||
|
||||
vlogscli-darwin-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-amd64
|
||||
|
||||
vlogscli-darwin-arm64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-arm64
|
||||
|
||||
vlogscli-freebsd-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-freebsd-amd64
|
||||
|
||||
vlogscli-openbsd-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-openbsd-amd64
|
||||
|
||||
vlogscli-windows-amd64-prod:
|
||||
APP_NAME=vlogscli $(MAKE) app-via-docker-windows-amd64
|
||||
|
||||
package-vlogscli:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker
|
||||
|
||||
package-vlogscli-pure:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-pure
|
||||
|
||||
package-vlogscli-amd64:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-amd64
|
||||
|
||||
package-vlogscli-arm:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-arm
|
||||
|
||||
package-vlogscli-arm64:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-arm64
|
||||
|
||||
package-vlogscli-ppc64le:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-ppc64le
|
||||
|
||||
package-vlogscli-386:
|
||||
APP_NAME=vlogscli $(MAKE) package-via-docker-386
|
||||
|
||||
publish-vlogscli:
|
||||
APP_NAME=vlogscli $(MAKE) publish-via-docker
|
||||
|
||||
vlogscli-linux-amd64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-arm:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-arm64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-ppc64le:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-s390x:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-loong64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-linux-386:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-darwin-amd64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-darwin-arm64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-freebsd-amd64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-openbsd-amd64:
|
||||
APP_NAME=vlogscli CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
|
||||
|
||||
vlogscli-windows-amd64:
|
||||
GOARCH=amd64 APP_NAME=vlogscli $(MAKE) app-local-windows-goarch
|
||||
|
||||
vlogscli-pure:
|
||||
APP_NAME=vlogscli $(MAKE) app-local-pure
|
||||
|
||||
run-vlogscli:
|
||||
APP_NAME=vlogscli $(MAKE) run-via-docker
|
||||
@@ -1,5 +0,0 @@
|
||||
# vlogscli
|
||||
|
||||
Command-line utility for querying [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
|
||||
|
||||
See [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/).
|
||||
@@ -1,6 +0,0 @@
|
||||
ARG base_image=non-existing
|
||||
FROM $base_image
|
||||
|
||||
ENTRYPOINT ["/vlogscli-prod"]
|
||||
ARG src_binary=non-existing
|
||||
COPY $src_binary ./vlogscli-prod
|
||||
@@ -1,245 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
)
|
||||
|
||||
type outputMode int
|
||||
|
||||
const (
|
||||
outputModeJSONMultiline = outputMode(0)
|
||||
outputModeJSONSingleline = outputMode(1)
|
||||
outputModeLogfmt = outputMode(2)
|
||||
outputModeCompact = outputMode(3)
|
||||
)
|
||||
|
||||
func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
|
||||
switch outputMode {
|
||||
case outputModeJSONMultiline:
|
||||
return func(w io.Writer, fields []logstorage.Field) error {
|
||||
return writeJSONObject(w, fields, true)
|
||||
}
|
||||
case outputModeJSONSingleline:
|
||||
return func(w io.Writer, fields []logstorage.Field) error {
|
||||
return writeJSONObject(w, fields, false)
|
||||
}
|
||||
case outputModeLogfmt:
|
||||
return writeLogfmtObject
|
||||
case outputModeCompact:
|
||||
return writeCompactObject
|
||||
default:
|
||||
panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
|
||||
}
|
||||
}
|
||||
|
||||
type jsonPrettifier struct {
|
||||
r io.ReadCloser
|
||||
formatter func(w io.Writer, fields []logstorage.Field) error
|
||||
|
||||
d *json.Decoder
|
||||
|
||||
pr *io.PipeReader
|
||||
pw *io.PipeWriter
|
||||
bw *bufio.Writer
|
||||
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier {
|
||||
d := json.NewDecoder(r)
|
||||
pr, pw := io.Pipe()
|
||||
bw := bufio.NewWriter(pw)
|
||||
|
||||
formatter := getOutputFormatter(outputMode)
|
||||
|
||||
jp := &jsonPrettifier{
|
||||
r: r,
|
||||
formatter: formatter,
|
||||
|
||||
d: d,
|
||||
|
||||
pr: pr,
|
||||
pw: pw,
|
||||
bw: bw,
|
||||
}
|
||||
|
||||
jp.wg.Add(1)
|
||||
go func() {
|
||||
defer jp.wg.Done()
|
||||
err := jp.prettifyJSONLines()
|
||||
jp.closePipesWithError(err)
|
||||
}()
|
||||
|
||||
return jp
|
||||
}
|
||||
|
||||
func (jp *jsonPrettifier) closePipesWithError(err error) {
|
||||
_ = jp.pr.CloseWithError(err)
|
||||
_ = jp.pw.CloseWithError(err)
|
||||
}
|
||||
|
||||
func (jp *jsonPrettifier) prettifyJSONLines() error {
|
||||
for jp.d.More() {
|
||||
fields, err := readNextJSONObject(jp.d)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(fields, func(i, j int) bool {
|
||||
return fields[i].Name < fields[j].Name
|
||||
})
|
||||
if err := jp.formatter(jp.bw, fields); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Flush bw after every output line in order to show results as soon as they appear.
|
||||
if err := jp.bw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (jp *jsonPrettifier) Close() error {
|
||||
jp.closePipesWithError(io.ErrUnexpectedEOF)
|
||||
err := jp.r.Close()
|
||||
jp.wg.Wait()
|
||||
return err
|
||||
}
|
||||
|
||||
func (jp *jsonPrettifier) Read(p []byte) (int, error) {
|
||||
return jp.pr.Read(p)
|
||||
}
|
||||
|
||||
func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
|
||||
t, err := d.Token()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read '{': %w", err)
|
||||
}
|
||||
delim, ok := t.(json.Delim)
|
||||
if !ok || delim.String() != "{" {
|
||||
return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim)
|
||||
}
|
||||
|
||||
var fields []logstorage.Field
|
||||
for {
|
||||
// Read object key
|
||||
t, err := d.Token()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read JSON object key or closing brace: %w", err)
|
||||
}
|
||||
delim, ok := t.(json.Delim)
|
||||
if ok {
|
||||
if delim.String() == "}" {
|
||||
return fields, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim)
|
||||
}
|
||||
key, ok := t.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected token read for object key: %v; want string or '}'", t)
|
||||
}
|
||||
|
||||
// read object value
|
||||
t, err = d.Token()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read JSON object value: %w", err)
|
||||
}
|
||||
value, ok := t.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t)
|
||||
}
|
||||
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: key,
|
||||
Value: value,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
|
||||
data := logstorage.MarshalFieldsToLogfmt(nil, fields)
|
||||
_, err := fmt.Fprintf(w, "%s\n", data)
|
||||
return err
|
||||
}
|
||||
|
||||
func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
|
||||
if len(fields) == 1 {
|
||||
// Just write field value as is without name
|
||||
_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
|
||||
return err
|
||||
}
|
||||
if len(fields) == 2 && (fields[0].Name == "_time" || fields[1].Name == "_time") {
|
||||
// Write _time\tfieldValue as is
|
||||
if fields[0].Name == "_time" {
|
||||
_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)
|
||||
return err
|
||||
}
|
||||
_, err := fmt.Fprintf(w, "%s\t%s\n", fields[1].Value, fields[0].Value)
|
||||
return err
|
||||
}
|
||||
|
||||
// Fall back to logfmt
|
||||
return writeLogfmtObject(w, fields)
|
||||
}
|
||||
|
||||
func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
|
||||
if len(fields) == 0 {
|
||||
fmt.Fprintf(w, "{}\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "{")
|
||||
writeNewlineIfNeeded(w, isMultiline)
|
||||
if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range fields[1:] {
|
||||
fmt.Fprintf(w, ",")
|
||||
writeNewlineIfNeeded(w, isMultiline)
|
||||
if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
writeNewlineIfNeeded(w, isMultiline)
|
||||
fmt.Fprintf(w, "}\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeNewlineIfNeeded(w io.Writer, isMultiline bool) {
|
||||
if isMultiline {
|
||||
fmt.Fprintf(w, "\n")
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error {
|
||||
key := getJSONString(f.Name)
|
||||
value := getJSONString(f.Value)
|
||||
if isMultiline {
|
||||
_, err := fmt.Fprintf(w, " %s: %s", key, value)
|
||||
return err
|
||||
}
|
||||
_, err := fmt.Fprintf(w, "%s:%s", key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func getJSONString(s string) string {
|
||||
data, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unexpected error when marshaling string to JSON: %w", err))
|
||||
}
|
||||
return jsonHTMLReplacer.Replace(string(data))
|
||||
}
|
||||
|
||||
var jsonHTMLReplacer = strings.NewReplacer(
|
||||
`\u003c`, "\u003c",
|
||||
`\u003e`, "\u003e",
|
||||
`\u0026`, "\u0026",
|
||||
)
|
||||
@@ -1,120 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"github.com/mattn/go-isatty"
|
||||
)
|
||||
|
||||
func isTerminal() bool {
|
||||
return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
|
||||
}
|
||||
|
||||
func readWithLess(r io.Reader, wrapLongLines bool) error {
|
||||
if !isTerminal() {
|
||||
// Just write everything to stdout if no terminal is available.
|
||||
_, err := io.Copy(os.Stdout, r)
|
||||
if err != nil && !isErrPipe(err) {
|
||||
return fmt.Errorf("error when forwarding data to stdout: %w", err)
|
||||
}
|
||||
if err := os.Stdout.Sync(); err != nil {
|
||||
return fmt.Errorf("cannot sync data to stdout: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
pr, pw, err := os.Pipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create pipe: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = pr.Close()
|
||||
_ = pw.Close()
|
||||
}()
|
||||
|
||||
// Ignore Ctrl+C in the current process, so 'less' could handle it properly
|
||||
cancel := ignoreSignals(os.Interrupt)
|
||||
defer cancel()
|
||||
|
||||
// Start 'less' process
|
||||
path, err := exec.LookPath("less")
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot find 'less' command: %w", err)
|
||||
}
|
||||
opts := []string{"less", "-F", "-X"}
|
||||
if !wrapLongLines {
|
||||
opts = append(opts, "-S")
|
||||
}
|
||||
p, err := os.StartProcess(path, opts, &os.ProcAttr{
|
||||
Env: append(os.Environ(), "LESSCHARSET=utf-8"),
|
||||
Files: []*os.File{pr, os.Stdout, os.Stderr},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot start 'less' process: %w", err)
|
||||
}
|
||||
|
||||
// Close pr after 'less' finishes in a parallel goroutine
|
||||
// in order to unblock forwarding data to stopped 'less' below.
|
||||
waitch := make(chan *os.ProcessState)
|
||||
go func() {
|
||||
// Wait for 'less' process to finish.
|
||||
ps, err := p.Wait()
|
||||
if err != nil {
|
||||
fatalf("unexpected error when waiting for 'less' process: %w", err)
|
||||
}
|
||||
_ = pr.Close()
|
||||
waitch <- ps
|
||||
}()
|
||||
|
||||
// Forward data from r to 'less'
|
||||
_, err = io.Copy(pw, r)
|
||||
_ = pw.Sync()
|
||||
_ = pw.Close()
|
||||
|
||||
// Wait until 'less' finished
|
||||
ps := <-waitch
|
||||
|
||||
// Verify 'less' status.
|
||||
if !ps.Success() {
|
||||
return fmt.Errorf("'less' finished with unexpected code %d", ps.ExitCode())
|
||||
}
|
||||
|
||||
if err != nil && !isErrPipe(err) {
|
||||
return fmt.Errorf("error when forwarding data to 'less': %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isErrPipe(err error) bool {
|
||||
return errors.Is(err, syscall.EPIPE) || errors.Is(err, io.ErrClosedPipe)
|
||||
}
|
||||
|
||||
func ignoreSignals(sigs ...os.Signal) func() {
|
||||
ch := make(chan os.Signal, 1)
|
||||
signal.Notify(ch, sigs...)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
_, ok := <-ch
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
signal.Stop(ch)
|
||||
close(ch)
|
||||
wg.Wait()
|
||||
}
|
||||
}
|
||||
@@ -1,436 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/ergochat/readline"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
)
|
||||
|
||||
var (
|
||||
datasourceURL = flag.String("datasource.url", "http://localhost:9428/select/logsql/query", "URL for querying VictoriaLogs; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/querying/#querying-logs . See also -tail.url")
|
||||
tailURL = flag.String("tail.url", "", "URL for live tailing queries to VictoriaLogs; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing ."+
|
||||
"The url is automatically detected from -datasource.url by replacing /query with /tail at the end if -tail.url is empty")
|
||||
historyFile = flag.String("historyFile", "vlogscli-history", "Path to file with command history")
|
||||
header = flagutil.NewArrayString("header", "Optional header to pass in request -datasource.url in the form 'HeaderName: value'")
|
||||
accountID = flag.Int("accountID", 0, "Account ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
|
||||
projectID = flag.Int("projectID", 0, "Project ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
|
||||
)
|
||||
|
||||
const (
|
||||
firstLinePrompt = ";> "
|
||||
nextLinePrompt = ""
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Write flags and help message to stdout, since it is easier to grep or pipe.
|
||||
flag.CommandLine.SetOutput(os.Stdout)
|
||||
flag.Usage = usage
|
||||
envflag.Parse()
|
||||
buildinfo.Init()
|
||||
logger.InitNoLogFlags()
|
||||
|
||||
hes, err := parseHeaders(*header)
|
||||
if err != nil {
|
||||
fatalf("cannot parse -header command-line flag: %s", err)
|
||||
}
|
||||
headers = hes
|
||||
|
||||
incompleteLine := ""
|
||||
cfg := &readline.Config{
|
||||
Prompt: firstLinePrompt,
|
||||
DisableAutoSaveHistory: true,
|
||||
Listener: func(line []rune, pos int, _ rune) ([]rune, int, bool) {
|
||||
incompleteLine = string(line)
|
||||
return line, pos, false
|
||||
},
|
||||
}
|
||||
rl, err := readline.NewFromConfig(cfg)
|
||||
if err != nil {
|
||||
fatalf("cannot initialize readline: %s", err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(rl, "sending queries to -datasource.url=%s\n", *datasourceURL)
|
||||
fmt.Fprintf(rl, `type ? and press enter to see available commands`+"\n")
|
||||
runReadlineLoop(rl, &incompleteLine)
|
||||
|
||||
if err := rl.Close(); err != nil {
|
||||
fatalf("cannot close readline: %s", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
|
||||
historyLines, err := loadFromHistory(*historyFile)
|
||||
if err != nil {
|
||||
fatalf("cannot load query history: %s", err)
|
||||
}
|
||||
for _, line := range historyLines {
|
||||
if err := rl.SaveToHistory(line); err != nil {
|
||||
fatalf("cannot initialize query history: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
outputMode := outputModeJSONMultiline
|
||||
wrapLongLines := false
|
||||
s := ""
|
||||
for {
|
||||
line, err := rl.ReadLine()
|
||||
if err != nil {
|
||||
switch err {
|
||||
case io.EOF:
|
||||
if s != "" {
|
||||
// This is non-interactive query execution.
|
||||
executeQuery(context.Background(), rl, s, outputMode, wrapLongLines)
|
||||
}
|
||||
return
|
||||
case readline.ErrInterrupt:
|
||||
if s == "" && *incompleteLine == "" {
|
||||
fmt.Fprintf(rl, "interrupted\n")
|
||||
os.Exit(128 + int(syscall.SIGINT))
|
||||
}
|
||||
// Default value for Ctrl+C - clear the prompt and store the incompletely entered line into history
|
||||
s += *incompleteLine
|
||||
historyLines = pushToHistory(rl, historyLines, s)
|
||||
s = ""
|
||||
rl.SetPrompt(firstLinePrompt)
|
||||
continue
|
||||
default:
|
||||
fatalf("unexpected error in readline: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
s += line
|
||||
if s == "" {
|
||||
// Skip empty lines
|
||||
continue
|
||||
}
|
||||
|
||||
if isQuitCommand(s) {
|
||||
fmt.Fprintf(rl, "bye!\n")
|
||||
_ = pushToHistory(rl, historyLines, s)
|
||||
return
|
||||
}
|
||||
if isHelpCommand(s) {
|
||||
printCommandsHelp(rl)
|
||||
historyLines = pushToHistory(rl, historyLines, s)
|
||||
s = ""
|
||||
continue
|
||||
}
|
||||
if s == `\s` {
|
||||
fmt.Fprintf(rl, "singleline json output mode\n")
|
||||
outputMode = outputModeJSONSingleline
|
||||
historyLines = pushToHistory(rl, historyLines, s)
|
||||
s = ""
|
||||
continue
|
||||
}
|
||||
if s == `\m` {
|
||||
fmt.Fprintf(rl, "multiline json output mode\n")
|
||||
outputMode = outputModeJSONMultiline
|
||||
historyLines = pushToHistory(rl, historyLines, s)
|
||||
s = ""
|
||||
continue
|
||||
}
|
||||
if s == `\c` {
|
||||
fmt.Fprintf(rl, "compact output mode\n")
|
||||
outputMode = outputModeCompact
|
||||
historyLines = pushToHistory(rl, historyLines, s)
|
||||
s = ""
|
||||
continue
|
||||
}
|
||||
if s == `\logfmt` {
|
||||
fmt.Fprintf(rl, "logfmt output mode\n")
|
||||
outputMode = outputModeLogfmt
|
||||
historyLines = pushToHistory(rl, historyLines, s)
|
||||
s = ""
|
||||
continue
|
||||
}
|
||||
if s == `\wrap_long_lines` {
|
||||
if wrapLongLines {
|
||||
wrapLongLines = false
|
||||
fmt.Fprintf(rl, "wrapping of long lines is disabled\n")
|
||||
} else {
|
||||
wrapLongLines = true
|
||||
fmt.Fprintf(rl, "wrapping of long lines is enabled\n")
|
||||
}
|
||||
historyLines = pushToHistory(rl, historyLines, s)
|
||||
s = ""
|
||||
continue
|
||||
}
|
||||
if line != "" && !strings.HasSuffix(line, ";") {
|
||||
// Assume the query is incomplete and allow the user finishing the query on the next line
|
||||
s += "\n"
|
||||
rl.SetPrompt(nextLinePrompt)
|
||||
continue
|
||||
}
|
||||
|
||||
// Execute the query
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
executeQuery(ctx, rl, s, outputMode, wrapLongLines)
|
||||
cancel()
|
||||
|
||||
historyLines = pushToHistory(rl, historyLines, s)
|
||||
s = ""
|
||||
rl.SetPrompt(firstLinePrompt)
|
||||
}
|
||||
}
|
||||
|
||||
func pushToHistory(rl *readline.Instance, historyLines []string, s string) []string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(historyLines) == 0 || historyLines[len(historyLines)-1] != s {
|
||||
historyLines = append(historyLines, s)
|
||||
if len(historyLines) > 500 {
|
||||
historyLines = historyLines[len(historyLines)-500:]
|
||||
}
|
||||
if err := saveToHistory(*historyFile, historyLines); err != nil {
|
||||
fatalf("cannot save query history: %s", err)
|
||||
}
|
||||
}
|
||||
if err := rl.SaveToHistory(s); err != nil {
|
||||
fatalf("cannot update query history: %s", err)
|
||||
}
|
||||
return historyLines
|
||||
}
|
||||
|
||||
func loadFromHistory(filePath string) ([]string, error) {
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
linesQuoted := strings.Split(string(data), "\n")
|
||||
lines := make([]string, 0, len(linesQuoted))
|
||||
i := 0
|
||||
for _, lineQuoted := range linesQuoted {
|
||||
i++
|
||||
if lineQuoted == "" {
|
||||
continue
|
||||
}
|
||||
line, err := strconv.Unquote(lineQuoted)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse line #%d at %s: %w; line: [%s]", i, filePath, err, line)
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
func saveToHistory(filePath string, lines []string) error {
|
||||
linesQuoted := make([]string, len(lines))
|
||||
for i, line := range lines {
|
||||
lineQuoted := strconv.Quote(line)
|
||||
linesQuoted[i] = lineQuoted
|
||||
}
|
||||
data := strings.Join(linesQuoted, "\n")
|
||||
return os.WriteFile(filePath, []byte(data), 0600)
|
||||
}
|
||||
|
||||
func isQuitCommand(s string) bool {
|
||||
switch s {
|
||||
case `\q`, "q", "quit", "exit":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isHelpCommand(s string) bool {
|
||||
switch s {
|
||||
case `\h`, "h", "help", "?":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func printCommandsHelp(w io.Writer) {
|
||||
fmt.Fprintf(w, "%s", `Available commands:
|
||||
\q - quit
|
||||
\h - show this help
|
||||
\s - singleline json output mode
|
||||
\m - multiline json output mode
|
||||
\c - compact output mode
|
||||
\logfmt - logfmt output mode
|
||||
\wrap_long_lines - toggles wrapping long lines
|
||||
\tail <query> - live tail <query> results
|
||||
|
||||
See https://docs.victoriametrics.com/victorialogs/querying/vlogscli/ for more details
|
||||
`)
|
||||
}
|
||||
|
||||
func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, wrapLongLines bool) {
|
||||
if strings.HasPrefix(qStr, `\tail `) {
|
||||
tailQuery(ctx, output, qStr, outputMode)
|
||||
return
|
||||
}
|
||||
|
||||
respBody := getQueryResponse(ctx, output, qStr, outputMode, *datasourceURL)
|
||||
if respBody == nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
_ = respBody.Close()
|
||||
}()
|
||||
|
||||
if err := readWithLess(respBody, wrapLongLines); err != nil {
|
||||
fmt.Fprintf(output, "error when reading query response: %s\n", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func tailQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
|
||||
qStr = strings.TrimPrefix(qStr, `\tail `)
|
||||
qURL, err := getTailURL()
|
||||
if err != nil {
|
||||
fmt.Fprintf(output, "%s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
respBody := getQueryResponse(ctx, output, qStr, outputMode, qURL)
|
||||
if respBody == nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
_ = respBody.Close()
|
||||
}()
|
||||
|
||||
if _, err := io.Copy(output, respBody); err != nil {
|
||||
if !errors.Is(err, context.Canceled) && !isErrPipe(err) {
|
||||
fmt.Fprintf(output, "error when live tailing query response: %s\n", err)
|
||||
}
|
||||
fmt.Fprintf(output, "\n")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func getTailURL() (string, error) {
|
||||
if *tailURL != "" {
|
||||
return *tailURL, nil
|
||||
}
|
||||
|
||||
u, err := url.Parse(*datasourceURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot parse -datasource.url=%q: %w", *datasourceURL, err)
|
||||
}
|
||||
if !strings.HasSuffix(u.Path, "/query") {
|
||||
return "", fmt.Errorf("cannot find /query suffix in -datasource.url=%q", *datasourceURL)
|
||||
}
|
||||
u.Path = u.Path[:len(u.Path)-len("/query")] + "/tail"
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func getQueryResponse(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, qURL string) io.ReadCloser {
|
||||
// Parse the query and convert it to canonical view.
|
||||
qStr = strings.TrimSuffix(qStr, ";")
|
||||
q, err := logstorage.ParseQuery(qStr)
|
||||
if err != nil {
|
||||
fmt.Fprintf(output, "cannot parse query: %s\n", err)
|
||||
return nil
|
||||
}
|
||||
qStr = q.String()
|
||||
fmt.Fprintf(output, "executing [%s]...", qStr)
|
||||
|
||||
// Prepare HTTP request for qURL
|
||||
args := make(url.Values)
|
||||
args.Set("query", qStr)
|
||||
data := strings.NewReader(args.Encode())
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", qURL, data)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("BUG: cannot prepare request to server: %w", err))
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
for _, h := range headers {
|
||||
req.Header.Set(h.Name, h.Value)
|
||||
}
|
||||
req.Header.Set("AccountID", strconv.Itoa(*accountID))
|
||||
req.Header.Set("ProjectID", strconv.Itoa(*projectID))
|
||||
|
||||
// Execute HTTP request at qURL
|
||||
startTime := time.Now()
|
||||
resp, err := httpClient.Do(req)
|
||||
queryDuration := time.Since(startTime)
|
||||
fmt.Fprintf(output, "; duration: %.3fs\n", queryDuration.Seconds())
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
fmt.Fprintf(output, "\n")
|
||||
} else {
|
||||
fmt.Fprintf(output, "cannot execute query: %s\n", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify response code
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
body = []byte(fmt.Sprintf("cannot read response body: %s", err))
|
||||
}
|
||||
fmt.Fprintf(output, "unexpected status code: %d; response body:\n%s\n", resp.StatusCode, body)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Prettify the response body
|
||||
jp := newJSONPrettifier(resp.Body, outputMode)
|
||||
|
||||
return jp
|
||||
}
|
||||
|
||||
var httpClient = &http.Client{}
|
||||
|
||||
var headers []headerEntry
|
||||
|
||||
type headerEntry struct {
|
||||
Name string
|
||||
Value string
|
||||
}
|
||||
|
||||
func parseHeaders(a []string) ([]headerEntry, error) {
|
||||
hes := make([]headerEntry, len(a))
|
||||
for i, s := range a {
|
||||
a := strings.SplitN(s, ":", 2)
|
||||
if len(a) != 2 {
|
||||
return nil, fmt.Errorf("cannot parse header=%q; it must contain at least one ':'; for example, 'Cookie: foo'", s)
|
||||
}
|
||||
hes[i] = headerEntry{
|
||||
Name: strings.TrimSpace(a[0]),
|
||||
Value: strings.TrimSpace(a[1]),
|
||||
}
|
||||
}
|
||||
return hes, nil
|
||||
}
|
||||
|
||||
func fatalf(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func usage() {
|
||||
const s = `
|
||||
vlogscli is a command-line tool for querying VictoriaLogs.
|
||||
|
||||
See the docs at https://docs.victoriametrics.com/victorialogs/querying/vlogscli/
|
||||
`
|
||||
flagutil.Usage(s)
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
FROM $root_image
|
||||
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/vlogscli-prod"]
|
||||
ARG TARGETARCH
|
||||
COPY vlogscli-linux-${TARGETARCH}-prod ./vlogscli-prod
|
||||
@@ -45,8 +45,6 @@ var (
|
||||
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||
u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||
i64FieldsPerLog = flag.Int("i64FieldsPerLog", 1, "The number of fields with int64 values to generate per each log entry; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||
floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
|
||||
ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
|
||||
@@ -256,9 +254,6 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
|
||||
for j := 0; j < *u64FieldsPerLog; j++ {
|
||||
fmt.Fprintf(bw, `,"u64_%d":"%d"`, j, rand.Uint64())
|
||||
}
|
||||
for j := 0; j < *i64FieldsPerLog; j++ {
|
||||
fmt.Fprintf(bw, `,"i64_%d":"%d"`, j, int64(rand.Uint64()))
|
||||
}
|
||||
for j := 0; j < *floatFieldsPerLog; j++ {
|
||||
fmt.Fprintf(bw, `,"float_%d":"%v"`, j, math.Round(10_000*rand.Float64())/1000)
|
||||
}
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
{% import (
|
||||
"slices"
|
||||
) %}
|
||||
|
||||
{% stripspace %}
|
||||
|
||||
{% func FacetsResponse(m map[string][]facetEntry) %}
|
||||
{
|
||||
{% code
|
||||
sortedKeys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
sortedKeys = append(sortedKeys, k)
|
||||
}
|
||||
slices.Sort(sortedKeys)
|
||||
%}
|
||||
"facets":[
|
||||
{% if len(sortedKeys) > 0 %}
|
||||
{%= facetsLine(m, sortedKeys[0]) %}
|
||||
{% for _, k := range sortedKeys[1:] %}
|
||||
,{%= facetsLine(m, k) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
]
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% func facetsLine(m map[string][]facetEntry, k string) %}
|
||||
{
|
||||
"field_name":{%q= k %},
|
||||
"values":[
|
||||
{% code fes := m[k] %}
|
||||
{% if len(fes) > 0 %}
|
||||
{%= facetLine(fes[0]) %}
|
||||
{% for _, fe := range fes[1:] %}
|
||||
,{%= facetLine(fe) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
]
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% func facetLine(fe facetEntry) %}
|
||||
{
|
||||
"field_value":{%q= fe.value %},
|
||||
"hits":{%s= fe.hits %}
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
@@ -1,178 +0,0 @@
|
||||
// Code generated by qtc from "facets_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:1
|
||||
package logsql
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:1
|
||||
import (
|
||||
"slices"
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:7
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:7
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:7
|
||||
func StreamFacetsResponse(qw422016 *qt422016.Writer, m map[string][]facetEntry) {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:7
|
||||
qw422016.N().S(`{`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:10
|
||||
sortedKeys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
sortedKeys = append(sortedKeys, k)
|
||||
}
|
||||
slices.Sort(sortedKeys)
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:15
|
||||
qw422016.N().S(`"facets":[`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:17
|
||||
if len(sortedKeys) > 0 {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:18
|
||||
streamfacetsLine(qw422016, m, sortedKeys[0])
|
||||
//line app/vlselect/logsql/facets_response.qtpl:19
|
||||
for _, k := range sortedKeys[1:] {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:19
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:20
|
||||
streamfacetsLine(qw422016, m, k)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:21
|
||||
}
|
||||
//line app/vlselect/logsql/facets_response.qtpl:22
|
||||
}
|
||||
//line app/vlselect/logsql/facets_response.qtpl:22
|
||||
qw422016.N().S(`]}`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
func WriteFacetsResponse(qq422016 qtio422016.Writer, m map[string][]facetEntry) {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
StreamFacetsResponse(qw422016, m)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
func FacetsResponse(m map[string][]facetEntry) string {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
WriteFacetsResponse(qb422016, m)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/facets_response.qtpl:25
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:27
|
||||
func streamfacetsLine(qw422016 *qt422016.Writer, m map[string][]facetEntry, k string) {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:27
|
||||
qw422016.N().S(`{"field_name":`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:29
|
||||
qw422016.N().Q(k)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:29
|
||||
qw422016.N().S(`,"values":[`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:31
|
||||
fes := m[k]
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:32
|
||||
if len(fes) > 0 {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:33
|
||||
streamfacetLine(qw422016, fes[0])
|
||||
//line app/vlselect/logsql/facets_response.qtpl:34
|
||||
for _, fe := range fes[1:] {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:34
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:35
|
||||
streamfacetLine(qw422016, fe)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:36
|
||||
}
|
||||
//line app/vlselect/logsql/facets_response.qtpl:37
|
||||
}
|
||||
//line app/vlselect/logsql/facets_response.qtpl:37
|
||||
qw422016.N().S(`]}`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
func writefacetsLine(qq422016 qtio422016.Writer, m map[string][]facetEntry, k string) {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
streamfacetsLine(qw422016, m, k)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
func facetsLine(m map[string][]facetEntry, k string) string {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
writefacetsLine(qb422016, m, k)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/facets_response.qtpl:40
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:42
|
||||
func streamfacetLine(qw422016 *qt422016.Writer, fe facetEntry) {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:42
|
||||
qw422016.N().S(`{"field_value":`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:44
|
||||
qw422016.N().Q(fe.value)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:44
|
||||
qw422016.N().S(`,"hits":`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:45
|
||||
qw422016.N().S(fe.hits)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:45
|
||||
qw422016.N().S(`}`)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
func writefacetLine(qq422016 qtio422016.Writer, fe facetEntry) {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
streamfacetLine(qw422016, fe)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
func facetLine(fe facetEntry) string {
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
writefacetLine(qb422016, fe)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/facets_response.qtpl:47
|
||||
}
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -14,7 +12,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
@@ -25,82 +22,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
)
|
||||
|
||||
// ProcessFacetsRequest handles /select/logsql/facets request.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#querying-facets
|
||||
func ProcessFacetsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
q, tenantIDs, err := parseCommonArgs(r)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
|
||||
limit, err := httputils.GetInt(r, "limit")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
maxValuesPerField, err := httputils.GetInt(r, "max_values_per_field")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
maxValueLen, err := httputils.GetInt(r, "max_value_len")
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
keepConstFields := httputils.GetBool(r, "keep_const_fields")
|
||||
|
||||
q.DropAllPipes()
|
||||
q.AddFacetsPipe(limit, maxValuesPerField, maxValueLen, keepConstFields)
|
||||
|
||||
var mLock sync.Mutex
|
||||
m := make(map[string][]facetEntry)
|
||||
writeBlock := func(_ uint, _ []int64, columns []logstorage.BlockColumn) {
|
||||
if len(columns) == 0 || len(columns[0].Values) == 0 {
|
||||
return
|
||||
}
|
||||
if len(columns) != 3 {
|
||||
logger.Panicf("BUG: expecting 3 columns; got %d columns", len(columns))
|
||||
}
|
||||
|
||||
fieldNames := columns[0].Values
|
||||
fieldValues := columns[1].Values
|
||||
hits := columns[2].Values
|
||||
|
||||
bb := blockResultPool.Get()
|
||||
for i := range fieldNames {
|
||||
fieldName := strings.Clone(fieldNames[i])
|
||||
fieldValue := strings.Clone(fieldValues[i])
|
||||
hitsStr := strings.Clone(hits[i])
|
||||
|
||||
mLock.Lock()
|
||||
m[fieldName] = append(m[fieldName], facetEntry{
|
||||
value: fieldValue,
|
||||
hits: hitsStr,
|
||||
})
|
||||
mLock.Unlock()
|
||||
}
|
||||
blockResultPool.Put(bb)
|
||||
}
|
||||
|
||||
// Execute the query
|
||||
if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
|
||||
httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Write response
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
WriteFacetsResponse(w, m)
|
||||
}
|
||||
|
||||
type facetEntry struct {
|
||||
value string
|
||||
hits string
|
||||
}
|
||||
|
||||
// ProcessHitsRequest handles /select/logsql/hits request.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#querying-hits-stats
|
||||
@@ -123,7 +44,6 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
if step <= 0 {
|
||||
httpserver.Errorf(w, r, "'step' must be bigger than zero")
|
||||
return
|
||||
}
|
||||
|
||||
// Obtain offset
|
||||
@@ -151,6 +71,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
|
||||
// Prepare the query for hits count.
|
||||
q.Optimize()
|
||||
q.DropAllPipes()
|
||||
q.AddCountByTimePipe(int64(step), int64(offset), fields)
|
||||
|
||||
@@ -281,6 +202,7 @@ func ProcessFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *htt
|
||||
}
|
||||
|
||||
// Obtain field names for the given query
|
||||
q.Optimize()
|
||||
fieldNames, err := vlstorage.GetFieldNames(ctx, tenantIDs, q)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot obtain field names: %s", err)
|
||||
@@ -320,6 +242,7 @@ func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *ht
|
||||
}
|
||||
|
||||
// Obtain unique values for the given field
|
||||
q.Optimize()
|
||||
values, err := vlstorage.GetFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot obtain values for field %q: %s", fieldName, err)
|
||||
@@ -342,6 +265,7 @@ func ProcessStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter,
|
||||
}
|
||||
|
||||
// Obtain stream field names for the given query
|
||||
q.Optimize()
|
||||
names, err := vlstorage.GetStreamFieldNames(ctx, tenantIDs, q)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot obtain stream field names: %s", err)
|
||||
@@ -380,6 +304,7 @@ func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter,
|
||||
}
|
||||
|
||||
// Obtain stream field values for the given query and the given fieldName
|
||||
q.Optimize()
|
||||
values, err := vlstorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot obtain stream field values: %s", err)
|
||||
@@ -411,6 +336,7 @@ func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http
|
||||
}
|
||||
|
||||
// Obtain streamIDs for the given query
|
||||
q.Optimize()
|
||||
streamIDs, err := vlstorage.GetStreamIDs(ctx, tenantIDs, q, uint64(limit))
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot obtain stream_ids: %s", err)
|
||||
@@ -442,6 +368,7 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
// Obtain streams for the given query
|
||||
q.Optimize()
|
||||
streams, err := vlstorage.GetStreams(ctx, tenantIDs, q, uint64(limit))
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot obtain streams: %s", err)
|
||||
@@ -453,8 +380,6 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
// ProcessLiveTailRequest processes live tailing request to /select/logsq/tail
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#live-tailing
|
||||
func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
liveTailRequests.Inc()
|
||||
defer liveTailRequests.Dec()
|
||||
@@ -465,10 +390,9 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
if !q.CanLiveTail() {
|
||||
httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
|
||||
return
|
||||
httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
|
||||
}
|
||||
q.Optimize()
|
||||
|
||||
refreshIntervalMsecs, err := httputils.GetDuration(r, "refresh_interval", 1000)
|
||||
if err != nil {
|
||||
@@ -477,37 +401,25 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
}
|
||||
refreshInterval := time.Millisecond * time.Duration(refreshIntervalMsecs)
|
||||
|
||||
startOffsetMsecs, err := httputils.GetDuration(r, "start_offset", 5*1000)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
startOffset := startOffsetMsecs * 1e6
|
||||
|
||||
offsetMsecs, err := httputils.GetDuration(r, "offset", 1000)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
offset := offsetMsecs * 1e6
|
||||
|
||||
ctxWithCancel, cancel := context.WithCancel(ctx)
|
||||
tp := newTailProcessor(cancel)
|
||||
|
||||
ticker := time.NewTicker(refreshInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
end := time.Now().UnixNano() - offset
|
||||
start := end - startOffset
|
||||
end := time.Now().UnixNano()
|
||||
doneCh := ctxWithCancel.Done()
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
|
||||
}
|
||||
qOrig := q
|
||||
for {
|
||||
q = qOrig.CloneWithTimeFilter(end, start, end)
|
||||
if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, tp.writeBlock); err != nil {
|
||||
start := end - tailOffsetNsecs
|
||||
end = time.Now().UnixNano()
|
||||
|
||||
qCopy := q.Clone()
|
||||
qCopy.AddTimeFilter(start, end)
|
||||
if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, qCopy, tp.writeBlock); err != nil {
|
||||
httpserver.Errorf(w, r, "cannot execute tail query [%s]: %s", q, err)
|
||||
return
|
||||
}
|
||||
@@ -525,8 +437,6 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
case <-doneCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
start = end - tailOffsetNsecs
|
||||
end = time.Now().UnixNano() - offset
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -650,208 +560,9 @@ func (tp *tailProcessor) getTailRows() ([][]logstorage.Field, error) {
|
||||
return tailRows, nil
|
||||
}
|
||||
|
||||
// ProcessStatsQueryRangeRequest handles /select/logsql/stats_query_range request.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-range-stats
|
||||
func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
q, tenantIDs, err := parseCommonArgs(r)
|
||||
if err != nil {
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Obtain step
|
||||
stepStr := r.FormValue("step")
|
||||
if stepStr == "" {
|
||||
stepStr = "1d"
|
||||
}
|
||||
step, err := promutils.ParseDuration(stepStr)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("cannot parse 'step' arg: %s", err)
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
if step <= 0 {
|
||||
err := fmt.Errorf("'step' must be bigger than zero")
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Obtain `by(...)` fields from the last `| stats` pipe in q.
|
||||
// Add `_time:step` to the `by(...)` list.
|
||||
byFields, err := q.GetStatsByFieldsAddGroupingByTime(int64(step))
|
||||
if err != nil {
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
m := make(map[string]*statsSeries)
|
||||
var mLock sync.Mutex
|
||||
|
||||
timestamp := q.GetTimestamp()
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
clonedColumnNames := make([]string, len(columns))
|
||||
for i, c := range columns {
|
||||
clonedColumnNames[i] = strings.Clone(c.Name)
|
||||
}
|
||||
for i := range timestamps {
|
||||
labels := make([]logstorage.Field, 0, len(byFields))
|
||||
for j, c := range columns {
|
||||
if c.Name == "_time" {
|
||||
nsec, ok := logstorage.TryParseTimestampRFC3339Nano(c.Values[i])
|
||||
if ok {
|
||||
timestamp = nsec
|
||||
continue
|
||||
}
|
||||
}
|
||||
if slices.Contains(byFields, c.Name) {
|
||||
labels = append(labels, logstorage.Field{
|
||||
Name: clonedColumnNames[j],
|
||||
Value: strings.Clone(c.Values[i]),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var dst []byte
|
||||
for j, c := range columns {
|
||||
if !slices.Contains(byFields, c.Name) {
|
||||
name := clonedColumnNames[j]
|
||||
dst = dst[:0]
|
||||
dst = append(dst, name...)
|
||||
dst = logstorage.MarshalFieldsToJSON(dst, labels)
|
||||
key := string(dst)
|
||||
p := statsPoint{
|
||||
Timestamp: timestamp,
|
||||
Value: strings.Clone(c.Values[i]),
|
||||
}
|
||||
|
||||
mLock.Lock()
|
||||
ss := m[key]
|
||||
if ss == nil {
|
||||
ss = &statsSeries{
|
||||
key: key,
|
||||
Name: name,
|
||||
Labels: labels,
|
||||
}
|
||||
m[key] = ss
|
||||
}
|
||||
ss.Points = append(ss.Points, p)
|
||||
mLock.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
|
||||
err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Sort the collected stats by time
|
||||
rows := make([]*statsSeries, 0, len(m))
|
||||
for _, ss := range m {
|
||||
points := ss.Points
|
||||
sort.Slice(points, func(i, j int) bool {
|
||||
return points[i].Timestamp < points[j].Timestamp
|
||||
})
|
||||
rows = append(rows, ss)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
return rows[i].key < rows[j].key
|
||||
})
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
WriteStatsQueryRangeResponse(w, rows)
|
||||
}
|
||||
|
||||
type statsSeries struct {
|
||||
key string
|
||||
|
||||
Name string
|
||||
Labels []logstorage.Field
|
||||
Points []statsPoint
|
||||
}
|
||||
|
||||
type statsPoint struct {
|
||||
Timestamp int64
|
||||
Value string
|
||||
}
|
||||
|
||||
// ProcessStatsQueryRequest handles /select/logsql/stats_query request.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-stats
|
||||
func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
q, tenantIDs, err := parseCommonArgs(r)
|
||||
if err != nil {
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Obtain `by(...)` fields from the last `| stats` pipe in q.
|
||||
byFields, err := q.GetStatsByFields()
|
||||
if err != nil {
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
var rows []statsRow
|
||||
var rowsLock sync.Mutex
|
||||
|
||||
timestamp := q.GetTimestamp()
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
clonedColumnNames := make([]string, len(columns))
|
||||
for i, c := range columns {
|
||||
clonedColumnNames[i] = strings.Clone(c.Name)
|
||||
}
|
||||
for i := range timestamps {
|
||||
labels := make([]logstorage.Field, 0, len(byFields))
|
||||
for j, c := range columns {
|
||||
if slices.Contains(byFields, c.Name) {
|
||||
labels = append(labels, logstorage.Field{
|
||||
Name: clonedColumnNames[j],
|
||||
Value: strings.Clone(c.Values[i]),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for j, c := range columns {
|
||||
if !slices.Contains(byFields, c.Name) {
|
||||
r := statsRow{
|
||||
Name: clonedColumnNames[j],
|
||||
Labels: labels,
|
||||
Timestamp: timestamp,
|
||||
Value: strings.Clone(c.Values[i]),
|
||||
}
|
||||
|
||||
rowsLock.Lock()
|
||||
rows = append(rows, r)
|
||||
rowsLock.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
|
||||
err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
|
||||
httpserver.SendPrometheusError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
WriteStatsQueryResponse(w, rows)
|
||||
}
|
||||
|
||||
type statsRow struct {
|
||||
Name string
|
||||
Labels []logstorage.Field
|
||||
Timestamp int64
|
||||
Value string
|
||||
}
|
||||
|
||||
// ProcessQueryRequest handles /select/logsql/query request.
|
||||
//
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#querying-logs
|
||||
// See https://docs.victoriametrics.com/victorialogs/querying/#http-api
|
||||
func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
q, tenantIDs, err := parseCommonArgs(r)
|
||||
if err != nil {
|
||||
@@ -894,6 +605,7 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
|
||||
|
||||
q.AddPipeLimit(uint64(limit))
|
||||
}
|
||||
q.Optimize()
|
||||
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
if len(columns) == 0 || len(columns[0].Values) == 0 {
|
||||
@@ -924,7 +636,7 @@ type row struct {
|
||||
func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
|
||||
limitUpper := 2 * limit
|
||||
q.AddPipeLimit(uint64(limitUpper))
|
||||
|
||||
q.Optimize()
|
||||
rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -935,58 +647,32 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// Slow path - adjust time range for selecting up to limitUpper rows
|
||||
// Slow path - search for the time range containing up to limitUpper rows.
|
||||
start, end := q.GetFilterTimeRange()
|
||||
d := end/2 - start/2
|
||||
start += d
|
||||
|
||||
qOrig := q
|
||||
for {
|
||||
timestamp := qOrig.GetTimestamp()
|
||||
q = qOrig.CloneWithTimeFilter(timestamp, start, end)
|
||||
q = qOrig.Clone()
|
||||
q.AddTimeFilter(start, end)
|
||||
rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if d == 0 || start >= end {
|
||||
// The [start ... end] time range equals one nanosecond.
|
||||
// Just return up to limit rows.
|
||||
if len(rows) > limit {
|
||||
rows = rows[:limit]
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
dLastBit := d & 1
|
||||
d /= 2
|
||||
|
||||
if len(rows) >= limitUpper {
|
||||
// The number of found rows on the [start ... end] time range exceeds limitUpper,
|
||||
// so reduce the time range to [start+d ... end].
|
||||
start += d
|
||||
continue
|
||||
}
|
||||
if len(rows) >= limit {
|
||||
// The number of found rows is in the range [limit ... limitUpper).
|
||||
// This means that found rows contains the needed limit rows with the biggest timestamps.
|
||||
if len(rows) >= limit && len(rows) < limitUpper || d == 0 {
|
||||
rows = getLastNRows(rows, limit)
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// The number of found rows on [start ... end] time range is below the limit.
|
||||
// This means the time range doesn't cover the needed logs, so it must be extended.
|
||||
|
||||
if len(rows) == 0 {
|
||||
// The [start ... end] time range doesn't contain any rows, so change it to [start-d ... start).
|
||||
end = start - 1
|
||||
start -= d + dLastBit
|
||||
continue
|
||||
lastBit := d & 1
|
||||
d /= 2
|
||||
if len(rows) > limit {
|
||||
start += d
|
||||
} else {
|
||||
start -= d + lastBit
|
||||
}
|
||||
|
||||
// The number of found rows on [start ... end] time range is bigger than 0 but smaller than limit.
|
||||
// Increase the time range to [start-d ... end].
|
||||
start -= d + dLastBit
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1007,25 +693,20 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
|
||||
var rows []row
|
||||
var rowsLock sync.Mutex
|
||||
writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
|
||||
clonedColumnNames := make([]string, len(columns))
|
||||
for i, c := range columns {
|
||||
clonedColumnNames[i] = strings.Clone(c.Name)
|
||||
}
|
||||
rowsLock.Lock()
|
||||
defer rowsLock.Unlock()
|
||||
|
||||
for i, timestamp := range timestamps {
|
||||
fields := make([]logstorage.Field, len(columns))
|
||||
for j := range columns {
|
||||
f := &fields[j]
|
||||
f.Name = clonedColumnNames[j]
|
||||
f.Name = strings.Clone(columns[j].Name)
|
||||
f.Value = strings.Clone(columns[j].Values[i])
|
||||
}
|
||||
|
||||
rowsLock.Lock()
|
||||
rows = append(rows, row{
|
||||
timestamp: timestamp,
|
||||
fields: fields,
|
||||
})
|
||||
rowsLock.Unlock()
|
||||
}
|
||||
|
||||
if len(rows) >= limit {
|
||||
@@ -1047,6 +728,13 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
|
||||
}
|
||||
tenantIDs := []logstorage.TenantID{tenantID}
|
||||
|
||||
// Parse query
|
||||
qStr := r.FormValue("query")
|
||||
q, err := logstorage.ParseQuery(qStr)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
|
||||
}
|
||||
|
||||
// Parse optional start and end args
|
||||
start, okStart, err := getTimeNsec(r, "start")
|
||||
if err != nil {
|
||||
@@ -1056,35 +744,7 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Parse optional time arg
|
||||
timestamp, okTime, err := getTimeNsec(r, "time")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !okTime {
|
||||
// If time arg is missing, then evaluate query either at the end timestamp (if it is set)
|
||||
// or at the current timestamp (if end query arg isn't set)
|
||||
if okEnd {
|
||||
timestamp = end
|
||||
} else {
|
||||
timestamp = time.Now().UnixNano()
|
||||
}
|
||||
}
|
||||
|
||||
// decrease timestamp by one nanosecond in order to avoid capturing logs belonging
|
||||
// to the first nanosecond at the next period of time (month, week, day, hour, etc.)
|
||||
timestamp--
|
||||
|
||||
// Parse query
|
||||
qStr := r.FormValue("query")
|
||||
q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
|
||||
}
|
||||
|
||||
if okStart || okEnd {
|
||||
// Add _time:[start, end] filter if start or end args were set.
|
||||
if !okStart {
|
||||
start = math.MinInt64
|
||||
}
|
||||
@@ -1094,22 +754,6 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
|
||||
q.AddTimeFilter(start, end)
|
||||
}
|
||||
|
||||
// Parse optional extra_filters
|
||||
extraFiltersStr := r.FormValue("extra_filters")
|
||||
extraFilters, err := parseExtraFilters(extraFiltersStr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
q.AddExtraFilters(extraFilters)
|
||||
|
||||
// Parse optional extra_stream_filters
|
||||
extraStreamFiltersStr := r.FormValue("extra_stream_filters")
|
||||
extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
q.AddExtraFilters(extraStreamFilters)
|
||||
|
||||
return q, tenantIDs, nil
|
||||
}
|
||||
|
||||
@@ -1125,115 +769,3 @@ func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
|
||||
}
|
||||
return nsecs, true, nil
|
||||
}
|
||||
|
||||
func parseExtraFilters(s string) (*logstorage.Filter, error) {
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if !strings.HasPrefix(s, `{"`) {
|
||||
return logstorage.ParseFilter(s)
|
||||
}
|
||||
|
||||
// Extra filters in the form {"field":"value",...}.
|
||||
kvs, err := parseExtraFiltersJSON(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filters := make([]string, len(kvs))
|
||||
for i, kv := range kvs {
|
||||
if len(kv.values) == 1 {
|
||||
filters[i] = fmt.Sprintf("%q:=%q", kv.key, kv.values[0])
|
||||
} else {
|
||||
orValues := make([]string, len(kv.values))
|
||||
for j, v := range kv.values {
|
||||
orValues[j] = fmt.Sprintf("%q", v)
|
||||
}
|
||||
filters[i] = fmt.Sprintf("%q:in(%s)", kv.key, strings.Join(orValues, ","))
|
||||
}
|
||||
}
|
||||
s = strings.Join(filters, " ")
|
||||
return logstorage.ParseFilter(s)
|
||||
}
|
||||
|
||||
func parseExtraStreamFilters(s string) (*logstorage.Filter, error) {
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if !strings.HasPrefix(s, `{"`) {
|
||||
return logstorage.ParseFilter(s)
|
||||
}
|
||||
|
||||
// Extra stream filters in the form {"field":"value",...}.
|
||||
kvs, err := parseExtraFiltersJSON(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filters := make([]string, len(kvs))
|
||||
for i, kv := range kvs {
|
||||
if len(kv.values) == 1 {
|
||||
filters[i] = fmt.Sprintf("%q=%q", kv.key, kv.values[0])
|
||||
} else {
|
||||
orValues := make([]string, len(kv.values))
|
||||
for j, v := range kv.values {
|
||||
orValues[j] = regexp.QuoteMeta(v)
|
||||
}
|
||||
filters[i] = fmt.Sprintf("%q=~%q", kv.key, strings.Join(orValues, "|"))
|
||||
}
|
||||
}
|
||||
s = "{" + strings.Join(filters, ",") + "}"
|
||||
return logstorage.ParseFilter(s)
|
||||
}
|
||||
|
||||
type extraFilter struct {
|
||||
key string
|
||||
values []string
|
||||
}
|
||||
|
||||
func parseExtraFiltersJSON(s string) ([]extraFilter, error) {
|
||||
v, err := fastjson.Parse(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o := v.GetObject()
|
||||
|
||||
var errOuter error
|
||||
var filters []extraFilter
|
||||
o.Visit(func(k []byte, v *fastjson.Value) {
|
||||
if errOuter != nil {
|
||||
return
|
||||
}
|
||||
switch v.Type() {
|
||||
case fastjson.TypeString:
|
||||
filters = append(filters, extraFilter{
|
||||
key: string(k),
|
||||
values: []string{string(v.GetStringBytes())},
|
||||
})
|
||||
case fastjson.TypeArray:
|
||||
a := v.GetArray()
|
||||
if len(a) == 0 {
|
||||
return
|
||||
}
|
||||
orValues := make([]string, len(a))
|
||||
for i, av := range a {
|
||||
ov, err := av.StringBytes()
|
||||
if err != nil {
|
||||
errOuter = fmt.Errorf("cannot obtain string item at the array for key %q; item: %s", k, av)
|
||||
return
|
||||
}
|
||||
orValues[i] = string(ov)
|
||||
}
|
||||
filters = append(filters, extraFilter{
|
||||
key: string(k),
|
||||
values: orValues,
|
||||
})
|
||||
default:
|
||||
errOuter = fmt.Errorf("unexpected type of value for key %q: %s; value: %s", k, v.Type(), v)
|
||||
}
|
||||
})
|
||||
if errOuter != nil {
|
||||
return nil, errOuter
|
||||
}
|
||||
return filters, nil
|
||||
}
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
package logsql
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseExtraFilters_Success(t *testing.T) {
|
||||
f := func(s, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
f, err := parseExtraFilters(s)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error in parseExtraFilters: %s", err)
|
||||
}
|
||||
result := f.String()
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected result\ngot\n%s\nwant\n%s", result, resultExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f("", "")
|
||||
|
||||
// JSON string
|
||||
f(`{"foo":"bar"}`, `foo:=bar`)
|
||||
f(`{"foo":["bar","baz"]}`, `foo:in(bar,baz)`)
|
||||
f(`{"z":"=b ","c":["d","e,"],"a":[],"_msg":"x"}`, `z:="=b " c:in(d,"e,") =x`)
|
||||
|
||||
// LogsQL filter
|
||||
f(`foobar`, `foobar`)
|
||||
f(`foo:bar`, `foo:bar`)
|
||||
f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
|
||||
}
|
||||
|
||||
func TestParseExtraFilters_Failure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := parseExtraFilters(s)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
}
|
||||
}
|
||||
|
||||
// Invalid JSON
|
||||
f(`{"foo"}`)
|
||||
f(`[1,2]`)
|
||||
f(`{"foo":[1]}`)
|
||||
|
||||
// Invliad LogsQL filter
|
||||
f(`foo:(bar`)
|
||||
|
||||
// excess pipe
|
||||
f(`foo | count()`)
|
||||
}
|
||||
|
||||
func TestParseExtraStreamFilters_Success(t *testing.T) {
|
||||
f := func(s, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
f, err := parseExtraStreamFilters(s)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error in parseExtraStreamFilters: %s", err)
|
||||
}
|
||||
result := f.String()
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f("", "")
|
||||
|
||||
// JSON string
|
||||
f(`{"foo":"bar"}`, `{foo="bar"}`)
|
||||
f(`{"foo":["bar","baz"]}`, `{foo=~"bar|baz"}`)
|
||||
f(`{"z":"b","c":["d","e|\""],"a":[],"_msg":"x"}`, `{z="b",c=~"d|e\\|\"",_msg="x"}`)
|
||||
|
||||
// LogsQL filter
|
||||
f(`foobar`, `foobar`)
|
||||
f(`foo:bar`, `foo:bar`)
|
||||
f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
|
||||
}
|
||||
|
||||
func TestParseExtraStreamFilters_Failure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := parseExtraStreamFilters(s)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
}
|
||||
}
|
||||
|
||||
// Invalid JSON
|
||||
f(`{"foo"}`)
|
||||
f(`[1,2]`)
|
||||
f(`{"foo":[1]}`)
|
||||
|
||||
// Invliad LogsQL filter
|
||||
f(`foo:(bar`)
|
||||
|
||||
// excess pipe
|
||||
f(`foo | count()`)
|
||||
}
|
||||
@@ -6,31 +6,15 @@
|
||||
|
||||
// JSONRow creates JSON row from the given fields.
|
||||
{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
|
||||
{% code
|
||||
i := 0
|
||||
for i < len(columns) && columns[i].Values[rowIdx] == "" {
|
||||
i++
|
||||
}
|
||||
columns = columns[i:]
|
||||
%}
|
||||
{% if len(columns) == 0 %}
|
||||
{% return %}
|
||||
{% endif %}
|
||||
{
|
||||
{% code c := &columns[0] %}
|
||||
{
|
||||
{% code c := &columns[0] %}
|
||||
{%q= c.Name %}:{%q= c.Values[rowIdx] %}
|
||||
{% code columns = columns[1:] %}
|
||||
{% for colIdx := range columns %}
|
||||
{% code
|
||||
c := &columns[colIdx]
|
||||
v := c.Values[rowIdx]
|
||||
%}
|
||||
{% if v == "" %}
|
||||
{% continue %}
|
||||
{% endif %}
|
||||
{% code c := &columns[colIdx] %}
|
||||
,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
|
||||
{% endfor %}
|
||||
}{% newline %}
|
||||
}{% newline %}
|
||||
{% endfunc %}
|
||||
|
||||
// JSONRows prints formatted rows
|
||||
@@ -39,11 +23,7 @@
|
||||
{% return %}
|
||||
{% endif %}
|
||||
{% for _, fields := range rows %}
|
||||
{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
|
||||
{% if len(fields) == 0 %}
|
||||
{% continue %}
|
||||
{% endif %}
|
||||
{
|
||||
{
|
||||
{% if len(fields) > 0 %}
|
||||
{% code
|
||||
f := fields[0]
|
||||
@@ -51,13 +31,10 @@
|
||||
%}
|
||||
{%q= f.Name %}:{%q= f.Value %}
|
||||
{% for _, f := range fields %}
|
||||
{% if f.Value == "" %}
|
||||
{% continue %}
|
||||
{% endif %}
|
||||
,{%q= f.Name %}:{%q= f.Value %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
}{% newline %}
|
||||
}{% newline %}
|
||||
{% endfor %}
|
||||
{% endfunc %}
|
||||
|
||||
|
||||
@@ -26,176 +26,141 @@ var (
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:8
|
||||
func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:10
|
||||
i := 0
|
||||
for i < len(columns) && columns[i].Values[rowIdx] == "" {
|
||||
i++
|
||||
}
|
||||
columns = columns[i:]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:16
|
||||
if len(columns) == 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:17
|
||||
return
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
//line app/vlselect/logsql/query_response.qtpl:8
|
||||
qw422016.N().S(`{`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:20
|
||||
//line app/vlselect/logsql/query_response.qtpl:10
|
||||
c := &columns[0]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
//line app/vlselect/logsql/query_response.qtpl:11
|
||||
qw422016.N().Q(c.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
//line app/vlselect/logsql/query_response.qtpl:11
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
//line app/vlselect/logsql/query_response.qtpl:11
|
||||
qw422016.N().Q(c.Values[rowIdx])
|
||||
//line app/vlselect/logsql/query_response.qtpl:22
|
||||
//line app/vlselect/logsql/query_response.qtpl:12
|
||||
columns = columns[1:]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:23
|
||||
//line app/vlselect/logsql/query_response.qtpl:13
|
||||
for colIdx := range columns {
|
||||
//line app/vlselect/logsql/query_response.qtpl:25
|
||||
//line app/vlselect/logsql/query_response.qtpl:14
|
||||
c := &columns[colIdx]
|
||||
v := c.Values[rowIdx]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:28
|
||||
if v == "" {
|
||||
//line app/vlselect/logsql/query_response.qtpl:29
|
||||
continue
|
||||
//line app/vlselect/logsql/query_response.qtpl:30
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:30
|
||||
//line app/vlselect/logsql/query_response.qtpl:14
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:31
|
||||
//line app/vlselect/logsql/query_response.qtpl:15
|
||||
qw422016.N().Q(c.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:31
|
||||
//line app/vlselect/logsql/query_response.qtpl:15
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:31
|
||||
//line app/vlselect/logsql/query_response.qtpl:15
|
||||
qw422016.N().Q(c.Values[rowIdx])
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
//line app/vlselect/logsql/query_response.qtpl:16
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
//line app/vlselect/logsql/query_response.qtpl:16
|
||||
qw422016.N().S(`}`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:33
|
||||
//line app/vlselect/logsql/query_response.qtpl:17
|
||||
qw422016.N().S(`
|
||||
`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
StreamJSONRow(qw422016, columns, rowIdx)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
WriteJSONRow(qb422016, columns, rowIdx)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
//line app/vlselect/logsql/query_response.qtpl:18
|
||||
}
|
||||
|
||||
// JSONRows prints formatted rows
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:37
|
||||
//line app/vlselect/logsql/query_response.qtpl:21
|
||||
func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:38
|
||||
//line app/vlselect/logsql/query_response.qtpl:22
|
||||
if len(rows) == 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
//line app/vlselect/logsql/query_response.qtpl:23
|
||||
return
|
||||
//line app/vlselect/logsql/query_response.qtpl:40
|
||||
//line app/vlselect/logsql/query_response.qtpl:24
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:41
|
||||
//line app/vlselect/logsql/query_response.qtpl:25
|
||||
for _, fields := range rows {
|
||||
//line app/vlselect/logsql/query_response.qtpl:42
|
||||
fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:43
|
||||
if len(fields) == 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:44
|
||||
continue
|
||||
//line app/vlselect/logsql/query_response.qtpl:45
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:45
|
||||
//line app/vlselect/logsql/query_response.qtpl:25
|
||||
qw422016.N().S(`{`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:47
|
||||
//line app/vlselect/logsql/query_response.qtpl:27
|
||||
if len(fields) > 0 {
|
||||
//line app/vlselect/logsql/query_response.qtpl:49
|
||||
//line app/vlselect/logsql/query_response.qtpl:29
|
||||
f := fields[0]
|
||||
fields = fields[1:]
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:52
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
qw422016.N().Q(f.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:52
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:52
|
||||
//line app/vlselect/logsql/query_response.qtpl:32
|
||||
qw422016.N().Q(f.Value)
|
||||
//line app/vlselect/logsql/query_response.qtpl:53
|
||||
//line app/vlselect/logsql/query_response.qtpl:33
|
||||
for _, f := range fields {
|
||||
//line app/vlselect/logsql/query_response.qtpl:54
|
||||
if f.Value == "" {
|
||||
//line app/vlselect/logsql/query_response.qtpl:55
|
||||
continue
|
||||
//line app/vlselect/logsql/query_response.qtpl:56
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:56
|
||||
//line app/vlselect/logsql/query_response.qtpl:33
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:57
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qw422016.N().Q(f.Name)
|
||||
//line app/vlselect/logsql/query_response.qtpl:57
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:57
|
||||
//line app/vlselect/logsql/query_response.qtpl:34
|
||||
qw422016.N().Q(f.Value)
|
||||
//line app/vlselect/logsql/query_response.qtpl:58
|
||||
//line app/vlselect/logsql/query_response.qtpl:35
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:59
|
||||
//line app/vlselect/logsql/query_response.qtpl:36
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:59
|
||||
//line app/vlselect/logsql/query_response.qtpl:36
|
||||
qw422016.N().S(`}`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:60
|
||||
//line app/vlselect/logsql/query_response.qtpl:37
|
||||
qw422016.N().S(`
|
||||
`)
|
||||
//line app/vlselect/logsql/query_response.qtpl:61
|
||||
//line app/vlselect/logsql/query_response.qtpl:38
|
||||
}
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
StreamJSONRows(qw422016, rows)
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
func JSONRows(rows [][]logstorage.Field) string {
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
WriteJSONRows(qb422016, rows)
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/query_response.qtpl:62
|
||||
//line app/vlselect/logsql/query_response.qtpl:39
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
{% stripspace %}
|
||||
|
||||
// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
|
||||
{% func StatsQueryRangeResponse(rows []*statsSeries) %}
|
||||
{
|
||||
"status":"success",
|
||||
"data":{
|
||||
"resultType":"matrix",
|
||||
"result":[
|
||||
{% if len(rows) > 0 %}
|
||||
{%= formatStatsSeries(rows[0]) %}
|
||||
{% code rows = rows[1:] %}
|
||||
{% for i := range rows %}
|
||||
,{%= formatStatsSeries(rows[i]) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
]
|
||||
}
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% func formatStatsSeries(ss *statsSeries) %}
|
||||
{
|
||||
"metric":{
|
||||
"__name__":{%q= ss.Name %}
|
||||
{% if len(ss.Labels) > 0 %}
|
||||
{% for _, label := range ss.Labels %}
|
||||
,{%q= label.Name %}:{%q= label.Value %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
},
|
||||
"values":[
|
||||
{% code points := ss.Points %}
|
||||
{% if len(points) > 0 %}
|
||||
{%= formatStatsPoint(&points[0]) %}
|
||||
{% code points = points[1:] %}
|
||||
{% for i := range points %}
|
||||
,{%= formatStatsPoint(&points[i]) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
]
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% func formatStatsPoint(p *statsPoint) %}
|
||||
[
|
||||
{%f= float64(p.Timestamp)/1e9 %},
|
||||
{%q= p.Value %}
|
||||
]
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
@@ -1,188 +0,0 @@
|
||||
// Code generated by qtc from "stats_query_range_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
package logsql
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
func StreamStatsQueryRangeResponse(qw422016 *qt422016.Writer, rows []*statsSeries) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:4
|
||||
qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:10
|
||||
if len(rows) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:11
|
||||
streamformatStatsSeries(qw422016, rows[0])
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:12
|
||||
rows = rows[1:]
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:13
|
||||
for i := range rows {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:13
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:14
|
||||
streamformatStatsSeries(qw422016, rows[i])
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:15
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:16
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:16
|
||||
qw422016.N().S(`]}}`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
func WriteStatsQueryRangeResponse(qq422016 qtio422016.Writer, rows []*statsSeries) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
StreamStatsQueryRangeResponse(qw422016, rows)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
func StatsQueryRangeResponse(rows []*statsSeries) string {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
WriteStatsQueryRangeResponse(qb422016, rows)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:22
|
||||
func streamformatStatsSeries(qw422016 *qt422016.Writer, ss *statsSeries) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:22
|
||||
qw422016.N().S(`{"metric":{"__name__":`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:25
|
||||
qw422016.N().Q(ss.Name)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:26
|
||||
if len(ss.Labels) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:27
|
||||
for _, label := range ss.Labels {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:27
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:28
|
||||
qw422016.N().Q(label.Name)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:28
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:28
|
||||
qw422016.N().Q(label.Value)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:29
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:30
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:30
|
||||
qw422016.N().S(`},"values":[`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:33
|
||||
points := ss.Points
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:34
|
||||
if len(points) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:35
|
||||
streamformatStatsPoint(qw422016, &points[0])
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:36
|
||||
points = points[1:]
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:37
|
||||
for i := range points {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:37
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:38
|
||||
streamformatStatsPoint(qw422016, &points[i])
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:39
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:40
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:40
|
||||
qw422016.N().S(`]}`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
func writeformatStatsSeries(qq422016 qtio422016.Writer, ss *statsSeries) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
streamformatStatsSeries(qw422016, ss)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
func formatStatsSeries(ss *statsSeries) string {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
writeformatStatsSeries(qb422016, ss)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:43
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:45
|
||||
func streamformatStatsPoint(qw422016 *qt422016.Writer, p *statsPoint) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:45
|
||||
qw422016.N().S(`[`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:47
|
||||
qw422016.N().F(float64(p.Timestamp) / 1e9)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:47
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:48
|
||||
qw422016.N().Q(p.Value)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:48
|
||||
qw422016.N().S(`]`)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
func writeformatStatsPoint(qq422016 qtio422016.Writer, p *statsPoint) {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
streamformatStatsPoint(qw422016, p)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
func formatStatsPoint(p *statsPoint) string {
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
writeformatStatsPoint(qb422016, p)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_range_response.qtpl:50
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
{% stripspace %}
|
||||
|
||||
// StatsQueryResponse generates response for /select/logsql/stats_query
|
||||
{% func StatsQueryResponse(rows []statsRow) %}
|
||||
{
|
||||
"status":"success",
|
||||
"data":{
|
||||
"resultType":"vector",
|
||||
"result":[
|
||||
{% if len(rows) > 0 %}
|
||||
{%= formatStatsRow(&rows[0]) %}
|
||||
{% code rows = rows[1:] %}
|
||||
{% for i := range rows %}
|
||||
,{%= formatStatsRow(&rows[i]) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
]
|
||||
}
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% func formatStatsRow(r *statsRow) %}
|
||||
{
|
||||
"metric":{
|
||||
"__name__":{%q= r.Name %}
|
||||
{% if len(r.Labels) > 0 %}
|
||||
{% for _, label := range r.Labels %}
|
||||
,{%q= label.Name %}:{%q= label.Value %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
},
|
||||
"value":[{%f= float64(r.Timestamp)/1e9 %},{%q= r.Value %}]
|
||||
}
|
||||
{% endfunc %}
|
||||
|
||||
{% endstripspace %}
|
||||
@@ -1,133 +0,0 @@
|
||||
// Code generated by qtc from "stats_query_response.qtpl". DO NOT EDIT.
|
||||
// See https://github.com/valyala/quicktemplate for details.
|
||||
|
||||
// StatsQueryResponse generates response for /select/logsql/stats_query
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
package logsql
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
import (
|
||||
qtio422016 "io"
|
||||
|
||||
qt422016 "github.com/valyala/quicktemplate"
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
var (
|
||||
_ = qtio422016.Copy
|
||||
_ = qt422016.AcquireByteBuffer
|
||||
)
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
func StreamStatsQueryResponse(qw422016 *qt422016.Writer, rows []statsRow) {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:4
|
||||
qw422016.N().S(`{"status":"success","data":{"resultType":"vector","result":[`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:10
|
||||
if len(rows) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:11
|
||||
streamformatStatsRow(qw422016, &rows[0])
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:12
|
||||
rows = rows[1:]
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:13
|
||||
for i := range rows {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:13
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:14
|
||||
streamformatStatsRow(qw422016, &rows[i])
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:15
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:16
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:16
|
||||
qw422016.N().S(`]}}`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
func WriteStatsQueryResponse(qq422016 qtio422016.Writer, rows []statsRow) {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
StreamStatsQueryResponse(qw422016, rows)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
func StatsQueryResponse(rows []statsRow) string {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
WriteStatsQueryResponse(qb422016, rows)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:20
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:22
|
||||
func streamformatStatsRow(qw422016 *qt422016.Writer, r *statsRow) {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:22
|
||||
qw422016.N().S(`{"metric":{"__name__":`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:25
|
||||
qw422016.N().Q(r.Name)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:26
|
||||
if len(r.Labels) > 0 {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:27
|
||||
for _, label := range r.Labels {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:27
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:28
|
||||
qw422016.N().Q(label.Name)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:28
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:28
|
||||
qw422016.N().Q(label.Value)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:29
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:30
|
||||
}
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:30
|
||||
qw422016.N().S(`},"value":[`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:32
|
||||
qw422016.N().F(float64(r.Timestamp) / 1e9)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:32
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:32
|
||||
qw422016.N().Q(r.Value)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:32
|
||||
qw422016.N().S(`]}`)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
func writeformatStatsRow(qq422016 qtio422016.Writer, r *statsRow) {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
streamformatStatsRow(qw422016, r)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
}
|
||||
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
func formatStatsRow(r *statsRow) string {
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
writeformatStatsRow(qb422016, r)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
return qs422016
|
||||
//line app/vlselect/logsql/stats_query_response.qtpl:34
|
||||
}
|
||||
@@ -23,7 +23,7 @@ var (
|
||||
"See also -search.maxQueueDuration")
|
||||
maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
|
||||
"limit is reached; see also -search.maxQueryDuration")
|
||||
maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden to a smaller value on a per-query basis via 'timeout' query arg")
|
||||
maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden on a per-query basis via 'timeout' query arg")
|
||||
)
|
||||
|
||||
func getDefaultMaxConcurrentRequests() int {
|
||||
@@ -177,10 +177,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
|
||||
httpserver.EnableCORS(w, r)
|
||||
switch path {
|
||||
case "/select/logsql/facets":
|
||||
logsqlFacetsRequests.Inc()
|
||||
logsql.ProcessFacetsRequest(ctx, w, r)
|
||||
return true
|
||||
case "/select/logsql/field_names":
|
||||
logsqlFieldNamesRequests.Inc()
|
||||
logsql.ProcessFieldNamesRequest(ctx, w, r)
|
||||
@@ -197,14 +193,6 @@ func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
|
||||
logsqlQueryRequests.Inc()
|
||||
logsql.ProcessQueryRequest(ctx, w, r)
|
||||
return true
|
||||
case "/select/logsql/stats_query":
|
||||
logsqlStatsQueryRequests.Inc()
|
||||
logsql.ProcessStatsQueryRequest(ctx, w, r)
|
||||
return true
|
||||
case "/select/logsql/stats_query_range":
|
||||
logsqlStatsQueryRangeRequests.Inc()
|
||||
logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
|
||||
return true
|
||||
case "/select/logsql/stream_field_names":
|
||||
logsqlStreamFieldNamesRequests.Inc()
|
||||
logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
|
||||
@@ -240,13 +228,10 @@ func getMaxQueryDuration(r *http.Request) time.Duration {
|
||||
}
|
||||
|
||||
var (
|
||||
logsqlFacetsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
|
||||
logsqlFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
|
||||
logsqlFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
|
||||
logsqlHitsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
|
||||
logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
|
||||
logsqlStatsQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
|
||||
logsqlStatsQueryRangeRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
|
||||
logsqlStreamFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
|
||||
logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
|
||||
logsqlStreamIDsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
|
||||
|
||||
BIN
app/vlselect/vmui/apple-touch-icon.png
Normal file
BIN
app/vlselect/vmui/apple-touch-icon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
@@ -1,12 +1,13 @@
|
||||
{
|
||||
"files": {
|
||||
"main.css": "./static/css/main.02a1c6cb.css",
|
||||
"main.js": "./static/js/main.55c8060b.js",
|
||||
"static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
|
||||
"main.css": "./static/css/main.1041c3d4.css",
|
||||
"main.js": "./static/js/main.62a82f4a.js",
|
||||
"static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
|
||||
"static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
|
||||
"index.html": "./index.html"
|
||||
},
|
||||
"entrypoints": [
|
||||
"static/css/main.02a1c6cb.css",
|
||||
"static/js/main.55c8060b.js"
|
||||
"static/css/main.1041c3d4.css",
|
||||
"static/js/main.62a82f4a.js"
|
||||
]
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
{
|
||||
"license": {
|
||||
"type": "opensource"
|
||||
}
|
||||
}
|
||||
BIN
app/vlselect/vmui/favicon-32x32.png
Normal file
BIN
app/vlselect/vmui/favicon-32x32.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
@@ -1 +0,0 @@
|
||||
<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB |
@@ -1 +1 @@
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.55c8060b.js"></script><link href="./static/css/main.02a1c6cb.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.62a82f4a.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
@@ -1,11 +1,16 @@
|
||||
{
|
||||
"short_name": "vmui",
|
||||
"name": "vmui",
|
||||
"short_name": "Victoria Metrics UI",
|
||||
"name": "Victoria Metrics UI is a metric explorer for Victoria Metrics",
|
||||
"icons": [
|
||||
{
|
||||
"src": "favicon.svg",
|
||||
"sizes": "any",
|
||||
"type": "image/svg+xml"
|
||||
"src": "favicon-32x32.png",
|
||||
"sizes": "32x32",
|
||||
"type": "image/png"
|
||||
},
|
||||
{
|
||||
"src": "apple-touch-icon.png",
|
||||
"type": "image/png",
|
||||
"sizes": "192x192"
|
||||
}
|
||||
],
|
||||
"start_url": ".",
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 6.5 KiB After Width: | Height: | Size: 67 KiB |
File diff suppressed because one or more lines are too long
1
app/vlselect/vmui/static/css/main.1041c3d4.css
Normal file
1
app/vlselect/vmui/static/css/main.1041c3d4.css
Normal file
File diff suppressed because one or more lines are too long
1
app/vlselect/vmui/static/js/685.bebe1265.chunk.js
Normal file
1
app/vlselect/vmui/static/js/685.bebe1265.chunk.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
2
app/vlselect/vmui/static/js/main.62a82f4a.js
Normal file
2
app/vlselect/vmui/static/js/main.62a82f4a.js
Normal file
File diff suppressed because one or more lines are too long
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
|
||||
/**
|
||||
* @remix-run/router v1.19.2
|
||||
* @remix-run/router v1.17.0
|
||||
*
|
||||
* Copyright (c) Remix Software Inc.
|
||||
*
|
||||
@@ -16,7 +16,7 @@
|
||||
*/
|
||||
|
||||
/**
|
||||
* React Router DOM v6.26.2
|
||||
* React Router DOM v6.24.0
|
||||
*
|
||||
* Copyright (c) Remix Software Inc.
|
||||
*
|
||||
@@ -27,7 +27,7 @@
|
||||
*/
|
||||
|
||||
/**
|
||||
* React Router v6.26.2
|
||||
* React Router v6.24.0
|
||||
*
|
||||
* Copyright (c) Remix Software Inc.
|
||||
*
|
||||
2267
app/vlselect/vmui/static/media/MetricsQL.aaabf95f2c9bf356bde4.md
Normal file
2267
app/vlselect/vmui/static/media/MetricsQL.aaabf95f2c9bf356bde4.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -18,12 +18,12 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
|
||||
retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
|
||||
"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
|
||||
maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
|
||||
"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
|
||||
futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
|
||||
futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/#retention")
|
||||
storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
|
||||
"see https://docs.victoriametrics.com/victorialogs/#storage")
|
||||
@@ -37,8 +37,6 @@ var (
|
||||
"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
|
||||
minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
|
||||
"the storage stops accepting new data")
|
||||
|
||||
forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
|
||||
)
|
||||
|
||||
// Init initializes vlstorage.
|
||||
@@ -89,28 +87,6 @@ func Stop() {
|
||||
strg = nil
|
||||
}
|
||||
|
||||
// RequestHandler is a storage request handler.
|
||||
func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
path := r.URL.Path
|
||||
if path == "/internal/force_merge" {
|
||||
if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
|
||||
return true
|
||||
}
|
||||
// Run force merge in background
|
||||
partitionNamePrefix := r.FormValue("partition_prefix")
|
||||
go func() {
|
||||
activeForceMerges.Inc()
|
||||
defer activeForceMerges.Dec()
|
||||
logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
|
||||
startTime := time.Now()
|
||||
strg.MustForceMerge(partitionNamePrefix)
|
||||
logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
|
||||
}()
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var strg *logstorage.Storage
|
||||
var storageMetrics *metrics.Set
|
||||
|
||||
@@ -229,5 +205,3 @@ func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
|
||||
metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_big_timestamp"}`, ss.RowsDroppedTooBigTimestamp)
|
||||
metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_small_timestamp"}`, ss.RowsDroppedTooSmallTimestamp)
|
||||
}
|
||||
|
||||
var activeForceMerges = metrics.NewCounter("vl_active_force_merges")
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
ARG base_image=non-existing
|
||||
ARG base_image
|
||||
FROM $base_image
|
||||
|
||||
EXPOSE 8429
|
||||
|
||||
ENTRYPOINT ["/vmagent-prod"]
|
||||
ARG src_binary=non-existing
|
||||
ARG src_binary
|
||||
COPY $src_binary ./vmagent-prod
|
||||
|
||||
@@ -36,7 +36,7 @@ var (
|
||||
//
|
||||
// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
|
||||
func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
|
||||
return stream.Parse(r, true, isGzipped, "", "", func(db string, rows []parser.Row) error {
|
||||
return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
|
||||
return insertRows(at, db, rows, nil)
|
||||
})
|
||||
}
|
||||
@@ -50,12 +50,11 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
|
||||
return err
|
||||
}
|
||||
isGzipped := req.Header.Get("Content-Encoding") == "gzip"
|
||||
isStreamMode := req.Header.Get("Stream-Mode") == "1"
|
||||
q := req.URL.Query()
|
||||
precision := q.Get("precision")
|
||||
// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
|
||||
db := q.Get("db")
|
||||
return stream.Parse(req.Body, isStreamMode, isGzipped, precision, db, func(db string, rows []parser.Row) error {
|
||||
return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
|
||||
return insertRows(at, db, rows, extraLabels)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -30,7 +30,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
@@ -46,7 +45,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -79,9 +77,6 @@ var (
|
||||
dryRun = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
|
||||
"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
|
||||
"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
|
||||
maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 0, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
|
||||
maxLabelNameLen = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
|
||||
maxLabelValueLen = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -98,15 +93,6 @@ var (
|
||||
)
|
||||
|
||||
func main() {
|
||||
// vmagent is optimized for reduced memory allocations,
|
||||
// so it can run with the reduced GOGC in order to reduce the used memory,
|
||||
// while keeping CPU usage spent in GC at low levels.
|
||||
//
|
||||
// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
|
||||
// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
|
||||
// exceeds 0.05 for extended periods of time.
|
||||
cgroup.SetGOGC(30)
|
||||
|
||||
// Write flags and help message to stdout, since it is easier to grep or pipe.
|
||||
flag.CommandLine.SetOutput(os.Stdout)
|
||||
flag.Usage = usage
|
||||
@@ -114,7 +100,6 @@ func main() {
|
||||
remotewrite.InitSecretFlags()
|
||||
buildinfo.Init()
|
||||
logger.Init()
|
||||
timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
|
||||
|
||||
if promscrape.IsDryRun() {
|
||||
if err := promscrape.CheckConfig(); err != nil {
|
||||
@@ -333,10 +318,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
influxQueryRequests.Inc()
|
||||
influxutils.WriteDatabaseNames(w)
|
||||
return true
|
||||
case "/influx/health":
|
||||
influxHealthRequests.Inc()
|
||||
influxutils.WriteHealthCheckResponse(w)
|
||||
return true
|
||||
case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
|
||||
opentelemetryPushRequests.Inc()
|
||||
if err := opentelemetry.InsertHandler(nil, r); err != nil {
|
||||
@@ -513,7 +494,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
|
||||
httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
|
||||
return true
|
||||
}
|
||||
at, err := auth.NewTokenPossibleMultitenant(p.AuthToken)
|
||||
at, err := auth.NewToken(p.AuthToken)
|
||||
if err != nil {
|
||||
httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
|
||||
return true
|
||||
@@ -525,13 +506,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
statusCode := http.StatusNoContent
|
||||
if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus/metrics/job/") {
|
||||
// Return 200 status code for pushgateway requests.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
|
||||
statusCode = http.StatusOK
|
||||
}
|
||||
w.WriteHeader(statusCode)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(p.Suffix, "datadog/") {
|
||||
@@ -589,10 +564,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
|
||||
influxQueryRequests.Inc()
|
||||
influxutils.WriteDatabaseNames(w)
|
||||
return true
|
||||
case "influx/health":
|
||||
influxHealthRequests.Inc()
|
||||
influxutils.WriteHealthCheckResponse(w)
|
||||
return true
|
||||
case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
|
||||
opentelemetryPushRequests.Inc()
|
||||
if err := opentelemetry.InsertHandler(at, r); err != nil {
|
||||
@@ -703,8 +674,7 @@ var (
|
||||
influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
|
||||
influxWriteErrors = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
|
||||
|
||||
influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
|
||||
influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
|
||||
influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
|
||||
|
||||
datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
|
||||
datadogv1WriteErrors = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -7,11 +7,12 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
@@ -22,7 +23,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -34,10 +34,8 @@ var (
|
||||
rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
|
||||
"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
|
||||
"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
|
||||
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
|
||||
retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxTime")
|
||||
retryMaxTime = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
|
||||
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
|
||||
sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
|
||||
proxyURL = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
|
||||
"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
|
||||
|
||||
tlsHandshakeTimeout = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
|
||||
@@ -92,9 +90,6 @@ type client struct {
|
||||
fq *persistentqueue.FastQueue
|
||||
hc *http.Client
|
||||
|
||||
retryMinInterval time.Duration
|
||||
retryMaxTime time.Duration
|
||||
|
||||
sendBlock func(block []byte) bool
|
||||
authCfg *promauth.Config
|
||||
awsCfg *awsapi.Config
|
||||
@@ -148,15 +143,13 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
|
||||
Timeout: sendTimeout.GetOptionalArg(argIdx),
|
||||
}
|
||||
c := &client{
|
||||
sanitizedURL: sanitizedURL,
|
||||
remoteWriteURL: remoteWriteURL,
|
||||
authCfg: authCfg,
|
||||
awsCfg: awsCfg,
|
||||
fq: fq,
|
||||
hc: hc,
|
||||
retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
|
||||
retryMaxTime: retryMaxTime.GetOptionalArg(argIdx),
|
||||
stopCh: make(chan struct{}),
|
||||
sanitizedURL: sanitizedURL,
|
||||
remoteWriteURL: remoteWriteURL,
|
||||
authCfg: authCfg,
|
||||
awsCfg: awsCfg,
|
||||
fq: fq,
|
||||
hc: hc,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
c.sendBlock = c.sendBlockHTTP
|
||||
|
||||
@@ -403,11 +396,11 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
|
||||
// sendBlockHTTP sends the given block to c.remoteWriteURL.
|
||||
//
|
||||
// The function returns false only if c.stopCh is closed.
|
||||
// Otherwise, it tries sending the block to remote storage indefinitely.
|
||||
// Otherwise it tries sending the block to remote storage indefinitely.
|
||||
func (c *client) sendBlockHTTP(block []byte) bool {
|
||||
c.rl.Register(len(block))
|
||||
maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
|
||||
retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
|
||||
maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
|
||||
retryDuration := timeutil.AddJitterToDuration(time.Second)
|
||||
retriesCount := 0
|
||||
|
||||
again:
|
||||
@@ -463,10 +456,10 @@ again:
|
||||
|
||||
// Unexpected status code returned
|
||||
retriesCount++
|
||||
retryAfterHeader := parseRetryAfterHeader(resp.Header.Get("Retry-After"))
|
||||
retryDuration = getRetryDuration(retryAfterHeader, retryDuration, maxRetryDuration)
|
||||
|
||||
// Handle response
|
||||
retryDuration *= 2
|
||||
if retryDuration > maxRetryDuration {
|
||||
retryDuration = maxRetryDuration
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
@@ -488,49 +481,3 @@ again:
|
||||
}
|
||||
|
||||
var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
|
||||
|
||||
// getRetryDuration returns retry duration.
|
||||
// retryAfterDuration has the highest priority.
|
||||
// If retryAfterDuration is not specified, retryDuration gets doubled.
|
||||
// retryDuration can't exceed maxRetryDuration.
|
||||
//
|
||||
// Also see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6097
|
||||
func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.Duration) time.Duration {
|
||||
// retryAfterDuration has the highest priority duration
|
||||
if retryAfterDuration > 0 {
|
||||
return timeutil.AddJitterToDuration(retryAfterDuration)
|
||||
}
|
||||
|
||||
// default backoff retry policy
|
||||
retryDuration *= 2
|
||||
if retryDuration > maxRetryDuration {
|
||||
retryDuration = maxRetryDuration
|
||||
}
|
||||
|
||||
return retryDuration
|
||||
}
|
||||
|
||||
// parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
|
||||
// retryAfterString should be in either HTTP-date or a number of seconds.
|
||||
// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
|
||||
func parseRetryAfterHeader(retryAfterString string) (retryAfterDuration time.Duration) {
|
||||
if retryAfterString == "" {
|
||||
return retryAfterDuration
|
||||
}
|
||||
|
||||
defer func() {
|
||||
v := retryAfterDuration.Seconds()
|
||||
logger.Infof("'Retry-After: %s' parsed into %.2f second(s)", retryAfterString, v)
|
||||
}()
|
||||
|
||||
// Retry-After could be in "Mon, 02 Jan 2006 15:04:05 GMT" format.
|
||||
if parsedTime, err := time.Parse(http.TimeFormat, retryAfterString); err == nil {
|
||||
return time.Duration(time.Until(parsedTime).Seconds()) * time.Second
|
||||
}
|
||||
// Retry-After could be in seconds.
|
||||
if seconds, err := strconv.Atoi(retryAfterString); err == nil {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"math"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCalculateRetryDuration(t *testing.T) {
|
||||
// `testFunc` call `calculateRetryDuration` for `n` times
|
||||
// and evaluate if the result of `calculateRetryDuration` is
|
||||
// 1. >= expectMinDuration
|
||||
// 2. <= expectMinDuration + 10% (see timeutil.AddJitterToDuration)
|
||||
f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
|
||||
}
|
||||
|
||||
expectMaxDuration := helper(expectMinDuration)
|
||||
expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
|
||||
|
||||
if !(retryDuration >= expectMinDuration && retryDuration <= expectMaxDuration) {
|
||||
t.Fatalf(
|
||||
"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
|
||||
expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
|
||||
retryDuration.Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Call calculateRetryDuration for 1 time.
|
||||
{
|
||||
// default backoff policy
|
||||
f(0, time.Second, 1, 2*time.Second)
|
||||
// default backoff policy exceed max limit"
|
||||
f(0, 10*time.Minute, 1, time.Minute)
|
||||
|
||||
// retry after > default backoff policy
|
||||
f(10*time.Second, 1*time.Second, 1, 10*time.Second)
|
||||
// retry after < default backoff policy
|
||||
f(1*time.Second, 10*time.Second, 1, 1*time.Second)
|
||||
// retry after invalid and < default backoff policy
|
||||
f(0, time.Second, 1, 2*time.Second)
|
||||
|
||||
}
|
||||
|
||||
// Call calculateRetryDuration for multiple times.
|
||||
{
|
||||
// default backoff policy 2 times
|
||||
f(0, time.Second, 2, 4*time.Second)
|
||||
// default backoff policy 3 times
|
||||
f(0, time.Second, 3, 8*time.Second)
|
||||
// default backoff policy N times exceed max limit
|
||||
f(0, time.Second, 10, time.Minute)
|
||||
|
||||
// retry after 120s 1 times
|
||||
f(120*time.Second, time.Second, 1, 120*time.Second)
|
||||
// retry after 120s 2 times
|
||||
f(120*time.Second, time.Second, 2, 120*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRetryAfterHeader(t *testing.T) {
|
||||
f := func(retryAfterString string, expectResult time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
result := parseRetryAfterHeader(retryAfterString)
|
||||
// expect `expectResult == result` when retryAfterString is in seconds or invalid
|
||||
// expect the difference between result and expectResult to be lower than 10%
|
||||
if !(expectResult == result || math.Abs(float64(expectResult-result))/float64(expectResult) < 0.10) {
|
||||
t.Fatalf(
|
||||
"incorrect retry after duration, want (ms): %d, got (ms): %d",
|
||||
expectResult.Milliseconds(), result.Milliseconds(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// retry after header in seconds
|
||||
f("10", 10*time.Second)
|
||||
// retry after header in date time
|
||||
f(time.Now().Add(30*time.Second).UTC().Format(http.TimeFormat), 30*time.Second)
|
||||
// retry after header invalid
|
||||
f("invalid-retry-after", 0)
|
||||
// retry after header not in GMT
|
||||
f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
|
||||
}
|
||||
|
||||
// helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
|
||||
func helper(d time.Duration) time.Duration {
|
||||
dv := d / 10
|
||||
if dv > 10*time.Second {
|
||||
dv = 10 * time.Second
|
||||
}
|
||||
|
||||
return d + dv
|
||||
}
|
||||
@@ -7,10 +7,13 @@ import (
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
@@ -18,7 +21,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
|
||||
@@ -28,7 +30,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/cespare/xxhash/v2"
|
||||
)
|
||||
@@ -98,6 +99,9 @@ var (
|
||||
// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
|
||||
rwctxsGlobal []*remoteWriteCtx
|
||||
|
||||
// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
|
||||
defaultAuthToken = &auth.Token{}
|
||||
|
||||
// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
|
||||
ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
|
||||
@@ -205,7 +209,7 @@ func Init() {
|
||||
|
||||
initStreamAggrConfigGlobal()
|
||||
|
||||
rwctxsGlobal = newRemoteWriteCtxs(*remoteWriteURLs)
|
||||
rwctxsGlobal = newRemoteWriteCtxs(nil, *remoteWriteURLs)
|
||||
|
||||
disableOnDiskQueues := []bool(*disableOnDiskQueue)
|
||||
disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
|
||||
@@ -290,7 +294,7 @@ var (
|
||||
relabelConfigTimestamp = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
|
||||
)
|
||||
|
||||
func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
|
||||
func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
|
||||
if len(urls) == 0 {
|
||||
logger.Panicf("BUG: urls must be non-empty")
|
||||
}
|
||||
@@ -312,6 +316,11 @@ func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
|
||||
logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
|
||||
}
|
||||
sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
|
||||
if at != nil {
|
||||
// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
|
||||
remoteWriteURL.Path = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL.Path, at.AccountID, at.ProjectID)
|
||||
sanitizedURL = fmt.Sprintf("%s:%d:%d", sanitizedURL, at.AccountID, at.ProjectID)
|
||||
}
|
||||
if *showRemoteWriteURL {
|
||||
sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
|
||||
}
|
||||
@@ -402,6 +411,11 @@ func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
|
||||
func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
|
||||
tss := wr.Timeseries
|
||||
|
||||
if at == nil && MultitenancyEnabled() {
|
||||
// Write data to default tenant if at isn't set when multitenancy is enabled.
|
||||
at = defaultAuthToken
|
||||
}
|
||||
|
||||
var tenantRctx *relabelCtx
|
||||
if at != nil {
|
||||
// Convert at to (vm_account_id, vm_project_id) labels.
|
||||
@@ -427,7 +441,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
|
||||
var rctx *relabelCtx
|
||||
rcs := allRelabelConfigs.Load()
|
||||
pcsGlobal := rcs.global
|
||||
if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
|
||||
if pcsGlobal.Len() > 0 {
|
||||
rctx = getRelabelCtx()
|
||||
defer putRelabelCtx(rctx)
|
||||
}
|
||||
@@ -471,26 +485,11 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
|
||||
rowsCountAfterRelabel := getRowsCount(tssBlock)
|
||||
rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
|
||||
}
|
||||
if timeserieslimits.Enabled() {
|
||||
tmpBlock := tssBlock[:0]
|
||||
for _, ts := range tssBlock {
|
||||
if !timeserieslimits.IsExceeding(ts.Labels) {
|
||||
tmpBlock = append(tmpBlock, ts)
|
||||
}
|
||||
}
|
||||
tssBlock = tmpBlock
|
||||
}
|
||||
sortLabelsIfNeeded(tssBlock)
|
||||
tssBlock = limitSeriesCardinality(tssBlock)
|
||||
if sas.IsEnabled() {
|
||||
matchIdxs := matchIdxsPool.Get()
|
||||
matchIdxs.B = sas.Push(tssBlock, matchIdxs.B)
|
||||
if !*streamAggrGlobalKeepInput {
|
||||
tssBlock = dropAggregatedSeries(tssBlock, matchIdxs.B, *streamAggrGlobalDropInput)
|
||||
}
|
||||
matchIdxsPool.Put(matchIdxs)
|
||||
}
|
||||
if deduplicatorGlobal != nil {
|
||||
tssBlock = sas.Push(tssBlock)
|
||||
} else if deduplicatorGlobal != nil {
|
||||
deduplicatorGlobal.Push(tssBlock)
|
||||
tssBlock = tssBlock[:0]
|
||||
}
|
||||
@@ -724,14 +723,29 @@ func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue i
|
||||
select {
|
||||
case <-logSkippedSeriesTicker.C:
|
||||
// Do not use logger.WithThrottler() here, since this will increase CPU usage
|
||||
// because every call to logSkippedSeries will result to a call to prompbmarshal.LabelsToString.
|
||||
logger.Warnf("skip series %s because %s=%d reached", prompbmarshal.LabelsToString(labels), flagName, flagValue)
|
||||
// because every call to logSkippedSeries will result to a call to labelsToString.
|
||||
logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
var logSkippedSeriesTicker = time.NewTicker(5 * time.Second)
|
||||
|
||||
func labelsToString(labels []prompbmarshal.Label) string {
|
||||
var b []byte
|
||||
b = append(b, '{')
|
||||
for i, label := range labels {
|
||||
b = append(b, label.Name...)
|
||||
b = append(b, '=')
|
||||
b = strconv.AppendQuote(b, label.Value)
|
||||
if i+1 < len(labels) {
|
||||
b = append(b, ',')
|
||||
}
|
||||
}
|
||||
b = append(b, '}')
|
||||
return string(b)
|
||||
}
|
||||
|
||||
var (
|
||||
globalRowsPushedBeforeRelabel = metrics.NewCounter("vmagent_remotewrite_global_rows_pushed_before_relabel_total")
|
||||
rowsDroppedByGlobalRelabel = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
|
||||
@@ -745,9 +759,6 @@ type remoteWriteCtx struct {
|
||||
sas atomic.Pointer[streamaggr.Aggregators]
|
||||
deduplicator *streamaggr.Deduplicator
|
||||
|
||||
streamAggrKeepInput bool
|
||||
streamAggrDropInput bool
|
||||
|
||||
pss []*pendingSeries
|
||||
pssNextIdx atomic.Uint64
|
||||
|
||||
@@ -891,20 +902,14 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
|
||||
// Apply stream aggregation or deduplication if they are configured
|
||||
sas := rwctx.sas.Load()
|
||||
if sas.IsEnabled() {
|
||||
matchIdxs := matchIdxsPool.Get()
|
||||
matchIdxs.B = sas.Push(tss, matchIdxs.B)
|
||||
if !rwctx.streamAggrKeepInput {
|
||||
if rctx == nil {
|
||||
rctx = getRelabelCtx()
|
||||
// Make a copy of tss before dropping aggregated series
|
||||
v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
|
||||
tss = append(*v, tss...)
|
||||
}
|
||||
tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
|
||||
if sas.ExpectModifications() && rctx == nil {
|
||||
rctx = getRelabelCtx()
|
||||
// Make a copy of tss before dropping aggregated series
|
||||
v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
|
||||
tss = append(*v, tss...)
|
||||
}
|
||||
matchIdxsPool.Put(matchIdxs)
|
||||
}
|
||||
if rwctx.deduplicator != nil {
|
||||
tss = sas.Push(tss)
|
||||
} else if rwctx.deduplicator != nil {
|
||||
rwctx.deduplicator.Push(tss)
|
||||
return true
|
||||
}
|
||||
@@ -924,23 +929,6 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
|
||||
return false
|
||||
}
|
||||
|
||||
var matchIdxsPool bytesutil.ByteBufferPool
|
||||
|
||||
func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, dropInput bool) []prompbmarshal.TimeSeries {
|
||||
dst := src[:0]
|
||||
if !dropInput {
|
||||
for i, match := range matchIdxs {
|
||||
if match == 1 {
|
||||
continue
|
||||
}
|
||||
dst = append(dst, src[i])
|
||||
}
|
||||
}
|
||||
tail := src[len(dst):]
|
||||
clear(tail)
|
||||
return dst
|
||||
}
|
||||
|
||||
func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSeries) {
|
||||
if rwctx.tryPushInternal(tss) {
|
||||
return
|
||||
|
||||
@@ -70,8 +70,6 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
|
||||
pss[0] = newPendingSeries(nil, true, 0, 100)
|
||||
rwctx := &remoteWriteCtx{
|
||||
idx: 0,
|
||||
streamAggrKeepInput: keepInput,
|
||||
streamAggrDropInput: dropInput,
|
||||
pss: pss,
|
||||
rowsPushedAfterRelabel: metrics.GetOrCreateCounter(`foo`),
|
||||
rowsDroppedByRelabel: metrics.GetOrCreateCounter(`bar`),
|
||||
|
||||
@@ -3,7 +3,6 @@ package remotewrite
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
@@ -24,7 +23,7 @@ var (
|
||||
streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
|
||||
"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
|
||||
"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
|
||||
streamAggrGlobalDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval on "+
|
||||
streamAggrGlobalDedupInterval = flagutil.NewDuration("streamAggr.dedupInterval", "0s", "Input samples are de-duplicated with this interval on "+
|
||||
"aggregator before optional aggregation with -streamAggr.config . "+
|
||||
"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
|
||||
streamAggrGlobalIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the "+
|
||||
@@ -51,13 +50,11 @@ var (
|
||||
streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
|
||||
"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
|
||||
"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
|
||||
streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
|
||||
"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
|
||||
streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
|
||||
"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
|
||||
"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
|
||||
streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
|
||||
"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
|
||||
"Multiple labels per remoteWrite.url must be delimited by '^^': -remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
|
||||
"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
|
||||
)
|
||||
|
||||
@@ -132,10 +129,11 @@ func initStreamAggrConfigGlobal() {
|
||||
sasGlobal.Store(sas)
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
|
||||
}
|
||||
dedupInterval := *streamAggrGlobalDedupInterval
|
||||
if dedupInterval > 0 {
|
||||
deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
|
||||
} else {
|
||||
dedupInterval := streamAggrGlobalDedupInterval.Duration()
|
||||
if dedupInterval > 0 {
|
||||
deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrDropInputLabels, "dedup-global")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,19 +147,14 @@ func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
|
||||
if sas != nil {
|
||||
filePath := sas.FilePath()
|
||||
rwctx.sas.Store(sas)
|
||||
rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
|
||||
rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
|
||||
metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
|
||||
}
|
||||
dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
|
||||
if dedupInterval > 0 {
|
||||
alias := fmt.Sprintf("dedup-%d", idx+1)
|
||||
var dropLabels []string
|
||||
if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
|
||||
dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
|
||||
} else {
|
||||
dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
|
||||
if dedupInterval > 0 {
|
||||
alias := fmt.Sprintf("dedup-%d", idx+1)
|
||||
rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
|
||||
}
|
||||
rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, dropLabels, alias)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,11 +195,12 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
|
||||
}
|
||||
|
||||
opts := &streamaggr.Options{
|
||||
DedupInterval: *streamAggrGlobalDedupInterval,
|
||||
DedupInterval: streamAggrGlobalDedupInterval.Duration(),
|
||||
DropInputLabels: *streamAggrGlobalDropInputLabels,
|
||||
IgnoreOldSamples: *streamAggrGlobalIgnoreOldSamples,
|
||||
IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
|
||||
KeepInput: *streamAggrGlobalKeepInput,
|
||||
DropInput: *streamAggrGlobalDropInput,
|
||||
}
|
||||
|
||||
sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
|
||||
@@ -230,16 +224,13 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag
|
||||
if *showRemoteWriteURL {
|
||||
alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
|
||||
}
|
||||
var dropLabels []string
|
||||
if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
|
||||
dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
|
||||
}
|
||||
opts := &streamaggr.Options{
|
||||
DedupInterval: streamAggrDedupInterval.GetOptionalArg(idx),
|
||||
DropInputLabels: dropLabels,
|
||||
DropInputLabels: *streamAggrDropInputLabels,
|
||||
IgnoreOldSamples: streamAggrIgnoreOldSamples.GetOptionalArg(idx),
|
||||
IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
|
||||
KeepInput: streamAggrKeepInput.GetOptionalArg(idx),
|
||||
DropInput: streamAggrDropInput.GetOptionalArg(idx),
|
||||
}
|
||||
|
||||
sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
|
||||
|
||||
@@ -41,24 +41,9 @@ Examples:
|
||||
Usage: "disable adding group's Name as label to generated alerts and time series.",
|
||||
Required: false,
|
||||
},
|
||||
&cli.StringSliceFlag{
|
||||
Name: "external.label",
|
||||
Usage: `Optional label in the form 'name=value' to add to all generated recording rules and alerts. Supports an array of values separated by comma or specified via multiple flags.`,
|
||||
Required: false,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "external.url",
|
||||
Usage: `Optional external URL to template in rule's labels or annotations.`,
|
||||
Required: false,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "loggerLevel",
|
||||
Usage: `Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "ERROR").`,
|
||||
Required: false,
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url"), c.String("loggerLevel")); failed {
|
||||
if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
|
||||
return fmt.Errorf("unittest failed")
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
|
||||
ARG certs_image=non-existing
|
||||
ARG root_image=non-existing
|
||||
ARG certs_image
|
||||
ARG root_image
|
||||
FROM $certs_image AS certs
|
||||
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
|
||||
|
||||
|
||||
@@ -43,33 +43,18 @@ func httpWrite(address string, r io.Reader) {
|
||||
// writeInputSeries send input series to vmstorage and flush them
|
||||
func writeInputSeries(input []series, interval *promutils.Duration, startStamp time.Time, dst string) error {
|
||||
r := testutil.WriteRequest{}
|
||||
var err error
|
||||
r.Timeseries, err = parseInputSeries(input, interval, startStamp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data := testutil.Compress(r)
|
||||
// write input series to vm
|
||||
httpWrite(dst, bytes.NewBuffer(data))
|
||||
vmstorage.Storage.DebugFlush()
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseInputSeries(input []series, interval *promutils.Duration, startStamp time.Time) ([]testutil.TimeSeries, error) {
|
||||
var res []testutil.TimeSeries
|
||||
for _, data := range input {
|
||||
expr, err := metricsql.Parse(data.Series)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("failed to parse series %s: %v", data.Series, err)
|
||||
return fmt.Errorf("failed to parse series %s: %v", data.Series, err)
|
||||
}
|
||||
promvals, err := parseInputValue(data.Values, true)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("failed to parse input series value %s: %v", data.Values, err)
|
||||
return fmt.Errorf("failed to parse input series value %s: %v", data.Values, err)
|
||||
}
|
||||
metricExpr, ok := expr.(*metricsql.MetricExpr)
|
||||
if !ok || len(metricExpr.LabelFilterss) != 1 {
|
||||
return res, fmt.Errorf("got invalid input series %s: %v", data.Series, err)
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to parse series %s to metric expr: %v", data.Series, err)
|
||||
}
|
||||
samples := make([]testutil.Sample, 0, len(promvals))
|
||||
ts := startStamp
|
||||
@@ -86,9 +71,14 @@ func parseInputSeries(input []series, interval *promutils.Duration, startStamp t
|
||||
for _, filter := range metricExpr.LabelFilterss[0] {
|
||||
ls = append(ls, testutil.Label{Name: filter.Label, Value: filter.Value})
|
||||
}
|
||||
res = append(res, testutil.TimeSeries{Labels: ls, Samples: samples})
|
||||
r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
|
||||
}
|
||||
return res, nil
|
||||
|
||||
data := testutil.Compress(r)
|
||||
// write input series to vm
|
||||
httpWrite(dst, bytes.NewBuffer(data))
|
||||
vmstorage.Storage.DebugFlush()
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseInputValue support input like "1", "1+1x1 _ -4 3+20x1", see more examples in test.
|
||||
|
||||
@@ -2,10 +2,8 @@ package unittest
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
)
|
||||
|
||||
func TestParseInputValue_Failure(t *testing.T) {
|
||||
@@ -45,7 +43,7 @@ func TestParseInputValue_Success(t *testing.T) {
|
||||
if decimal.IsStaleNaN(outputExpected[i].Value) && decimal.IsStaleNaN(output[i].Value) {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("unexpected Value field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
|
||||
t.Fatalf("unexpeccted Value field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -66,34 +64,3 @@ func TestParseInputValue_Success(t *testing.T) {
|
||||
|
||||
f("1+1x1 _ -4 stale 3+20x1", []sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}})
|
||||
}
|
||||
|
||||
func TestParseInputSeries_Success(t *testing.T) {
|
||||
f := func(input []series) {
|
||||
t.Helper()
|
||||
var interval promutils.Duration
|
||||
_, err := parseInputSeries(input, &interval, time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("expect to see no error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
f([]series{{Series: "test", Values: "1"}})
|
||||
f([]series{{Series: "test{}", Values: "1"}})
|
||||
f([]series{{Series: "test{env=\"prod\",job=\"a\" }", Values: "1"}})
|
||||
f([]series{{Series: "{__name__=\"test\",env=\"prod\",job=\"a\" }", Values: "1"}})
|
||||
}
|
||||
|
||||
func TestParseInputSeries_Fail(t *testing.T) {
|
||||
f := func(input []series) {
|
||||
t.Helper()
|
||||
var interval promutils.Duration
|
||||
_, err := parseInputSeries(input, &interval, time.Now())
|
||||
if err == nil {
|
||||
t.Fatalf("expect to see error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
f([]series{{Series: "", Values: "1"}})
|
||||
f([]series{{Series: "{}", Values: "1"}})
|
||||
f([]series{{Series: "{env=\"prod\",job=\"a\" or env=\"dev\",job=\"b\"}", Values: "1"}})
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
||||
"github.com/VictoriaMetrics/metricsql"
|
||||
)
|
||||
@@ -49,7 +48,7 @@ Outer:
|
||||
}
|
||||
var expSamples []parsedSample
|
||||
for _, s := range mt.ExpSamples {
|
||||
expLb := []prompbmarshal.Label{}
|
||||
expLb := datasource.Labels{}
|
||||
if s.Labels != "" {
|
||||
metricsqlExpr, err := metricsql.Parse(s.Labels)
|
||||
if err != nil {
|
||||
@@ -58,18 +57,16 @@ Outer:
|
||||
continue Outer
|
||||
}
|
||||
metricsqlMetricExpr, ok := metricsqlExpr.(*metricsql.MetricExpr)
|
||||
if !ok || len(metricsqlMetricExpr.LabelFilterss) > 1 {
|
||||
if !ok {
|
||||
checkErrs = append(checkErrs, fmt.Errorf("\n expr: %q, time: %s, err: %v", mt.Expr,
|
||||
mt.EvalTime.Duration().String(), fmt.Errorf("got invalid exp_samples: %q", s.Labels)))
|
||||
mt.EvalTime.Duration().String(), fmt.Errorf("got unsupported metricsql type")))
|
||||
continue Outer
|
||||
}
|
||||
if len(metricsqlMetricExpr.LabelFilterss) > 0 {
|
||||
for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
|
||||
expLb = append(expLb, prompbmarshal.Label{
|
||||
Name: l.Label,
|
||||
Value: l.Value,
|
||||
})
|
||||
}
|
||||
for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
|
||||
expLb = append(expLb, datasource.Label{
|
||||
Name: l.Label,
|
||||
Value: l.Value,
|
||||
})
|
||||
}
|
||||
}
|
||||
sort.Slice(expLb, func(i, j int) bool {
|
||||
|
||||
@@ -13,7 +13,7 @@ tests:
|
||||
- expr: suquery_interval_test
|
||||
eval_time: 4m
|
||||
exp_samples:
|
||||
- labels: '{__name__="suquery_interval_test", instance="localhost:9090", job="vmagent2"}'
|
||||
- labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
|
||||
value: 1
|
||||
|
||||
alert_rule_test:
|
||||
@@ -24,16 +24,20 @@ tests:
|
||||
job: vmagent2
|
||||
severity: page
|
||||
instance: localhost:9090
|
||||
datacenter: dc-123
|
||||
exp_annotations:
|
||||
summary: "Instance localhost:9090 down"
|
||||
description: "localhost:9090 of job vmagent2 in cluster has been down for more than 5 minutes."
|
||||
dashboard: "/d/dashboard?orgId=1"
|
||||
description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
|
||||
|
||||
- eval_time: 0
|
||||
alertname: AlwaysFiring
|
||||
exp_alerts:
|
||||
- {}
|
||||
- exp_labels:
|
||||
datacenter: dc-123
|
||||
|
||||
- eval_time: 0
|
||||
alertname: InstanceDown
|
||||
exp_alerts: []
|
||||
|
||||
external_labels:
|
||||
datacenter: dc-123
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
rule_files:
|
||||
- non-existing-file.yaml
|
||||
|
||||
tests: []
|
||||
@@ -8,8 +8,7 @@ groups:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Instance {{ $labels.instance }} down"
|
||||
description: "{{ $labels.instance }} of job {{ $labels.job }} in cluster {{ $externalLabels.cluster }} has been down for more than 5 minutes."
|
||||
dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
|
||||
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
|
||||
- alert: AlwaysFiring
|
||||
expr: 1
|
||||
- alert: SameAlertNameWithDifferentGroup
|
||||
|
||||
21
app/vmalert-tool/unittest/testdata/test1.yaml
vendored
21
app/vmalert-tool/unittest/testdata/test1.yaml
vendored
@@ -16,8 +16,7 @@ tests:
|
||||
groupname: group1
|
||||
alertname: SameAlertNameWithDifferentGroup
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: prod
|
||||
- {}
|
||||
- eval_time: 1m
|
||||
groupname: group2
|
||||
alertname: SameAlertNameWithDifferentGroup
|
||||
@@ -26,8 +25,7 @@ tests:
|
||||
groupname: group1
|
||||
alertname: SameAlertNameWithDifferentGroup
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: prod
|
||||
- {}
|
||||
- eval_time: 6m
|
||||
groupname: group1
|
||||
alertname: SameAlertNameWithDifferentGroup
|
||||
@@ -63,18 +61,18 @@ tests:
|
||||
eval_time: 4m
|
||||
exp_samples:
|
||||
- value: 4
|
||||
labels: '{__name__="t1", cluster="prod"}'
|
||||
labels: '{__name__="t1", datacenter="dc-123"}'
|
||||
- expr: t2
|
||||
eval_time: 4m
|
||||
exp_samples:
|
||||
- value: 4
|
||||
labels: '{__name__="t2", cluster="prod"}'
|
||||
labels: '{__name__="t2", datacenter="dc-123"}'
|
||||
- expr: t3
|
||||
eval_time: 4m
|
||||
exp_samples:
|
||||
# t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
|
||||
- value: 3
|
||||
labels: '{__name__="t3", cluster="prod"}'
|
||||
labels: '{__name__="t3", datacenter="dc-123"}'
|
||||
|
||||
alert_rule_test:
|
||||
- eval_time: 10m
|
||||
@@ -85,21 +83,22 @@ tests:
|
||||
job: vmagent1
|
||||
severity: page
|
||||
instance: localhost:9090
|
||||
cluster: prod
|
||||
datacenter: dc-123
|
||||
exp_annotations:
|
||||
summary: "Instance localhost:9090 down"
|
||||
description: "localhost:9090 of job vmagent1 in cluster prod has been down for more than 5 minutes."
|
||||
dashboard: "http://grafana:3000/d/dashboard?orgId=1"
|
||||
description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
|
||||
|
||||
- eval_time: 0
|
||||
groupname: group1
|
||||
alertname: AlwaysFiring
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: prod
|
||||
datacenter: dc-123
|
||||
|
||||
- eval_time: 0
|
||||
groupname: alerts
|
||||
alertname: InstanceDown
|
||||
exp_alerts: []
|
||||
|
||||
external_labels:
|
||||
datacenter: dc-123
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user