app/vmagent: logo fix

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/311

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version**
+The line returned when passing `--version` command line flag to binary. For example:
+```
+$ ./victoria-metrics-prod --version
+victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
+```
+
+**Used command-line flags**
+Command-line flags are listed as `flag{name="httpListenAddr", value=":443"} 1` lines at `/metrics` page.
+See the following docs for details:
+
+* [monitoring for single-node VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#monitoring)
+* [montioring for VictoriaMetrics cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#monitoring)
+
+**Additional context**
+Add any other context about the problem here such as error logs from VictoriaMetrics and Prometheus,
+`/metrics` output, screenshots from the official Grafana dashboards for VictoriaMetrics:
+
+* [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
+* [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/github-pages.yml

@@ -0,0 +1,30 @@
+name: github-pages
+on:
+  push:
+    paths:
+      - 'docs/*.md'
+      - 'README.md'
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: publish
+        shell: bash
+        env:
+          TOKEN: ${{secrets.CI_TOKEN}}
+        run: |
+          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git gpages
+          cp docs/*.md gpages
+          cp README.md gpages
+          cd gpages
+          git config --local user.email "info@victoriametrics.com"
+          git config --local user.name "Vika"
+          git add "*.md"
+          git commit -m "update github pages"
+          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git"
+          git push "${remote_repo}"
+          cd ..
+          rm -rf gpages

.github/workflows/main.yml

@@ -0,0 +1,51 @@
+name: main
+on:
+  push:
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+      - name: Code checkout
+        uses: actions/checkout@v1
+      - name: Dependencies
+        env:
+          GO111MODULE: off
+        run: |
+          go get -v golang.org/x/lint/golint
+          go get -u github.com/kisielk/errcheck
+      - name: Build
+        env:
+          GO111MODULE: on
+        run: |
+          export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
+          make check-all
+          git diff --exit-code
+          make test-full
+          make test-pure
+          make test-full-386
+          make victoria-metrics
+          make victoria-metrics-pure
+          make victoria-metrics-arm
+          make victoria-metrics-arm64
+          make vmutils
+          GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
+          GOOS=darwin go build -mod=vendor ./app/victoria-metrics
+      - name: Publish coverage
+        uses: codecov/codecov-action@v1.0.4
+        with:
+          token: ${{secrets.CODECOV_TOKEN}}
+          file: ./coverage.txt
+

.github/workflows/wiki.yml

@@ -0,0 +1,29 @@
+name: wiki
+on:
+  push:
+    paths:
+      - 'docs/*.md'
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: publish
+        shell: bash
+        env:
+          TOKEN: ${{secrets.CI_TOKEN}}
+        run: |
+          cd docs
+          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git wiki
+          find ./ -name '*.md' -exec cp -prv '{}' 'wiki' ';'
+          cd wiki
+          git config --local user.email "info@victoriametrics.com"
+          git config --local user.name "Vika"
+          git add "*.md"
+          git commit -m "update wiki pages"
+          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git"
+          git push "${remote_repo}"
+          cd ..
+          rm -rf wiki

.gitignore

@@ -1,3 +1,4 @@
+/tmp
 /tags
 /pkg
 *.pprof
@@ -7,5 +8,10 @@
 *.swp
 /gocache-for-docker
 /victoria-metrics-data
+/vmagent-remotewrite-data
 /vmstorage-data
 /vmselect-cache
+/package/temp-deb-*
+/package/temp-rpm-*
+/package/*.deb
+/package/*.rpm

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at info@victoriametrics.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -0,0 +1,16 @@
+If you like VictoriaMetrics and want to contribute, then we need the following:
+
+- Filing issues and feature requests [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+- Spreading a word about VictoriaMetrics: conference talks, articles, comments, experience sharing with colleagues.
+- Updating documentation.
+
+We are open to third-party pull requests provided they follow [KISS design principle](https://en.wikipedia.org/wiki/KISS_principle):
+
+- Prefer simple code and architecture.
+- Avoid complex abstractions.
+- Avoid magic code and fancy algorithms.
+- Avoid [big external dependencies](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d).
+- Minimize the number of moving parts in the distributed system.
+- Avoid automated decisions, which may hurt cluster availability, consistency or performance.
+
+Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019 VictoriaMetrics, Inc.
+   Copyright 2019-2020 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,7 +1,7 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
-	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | sha1sum | grep -oP '^.{8}')))
+	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17)))
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
@@ -11,7 +11,10 @@ endif
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(shell date -u +'%Y%m%d-%H%M%S')-$(BUILDINFO_TAG)'
 
 all: \
-	victoria-metrics-prod
+	victoria-metrics-prod \
+	vmagent-prod \
+	vmbackup-prod \
+	vmrestore-prod
 
 include app/*/Makefile
 include deployment/*/Makefile
@@ -19,46 +22,112 @@ include deployment/*/Makefile
 clean:
 	rm -rf bin/*
 
-release: victoria-metrics-prod
-	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod
+publish: \
+	publish-victoria-metrics \
+	publish-vmagent \
+	publish-vmbackup \
+	publish-vmrestore
+
+package: \
+	package-victoria-metrics \
+	package-vmagent \
+	package-vmbackup \
+	package-vmrestore
+
+vmutils: \
+	vmagent \
+	vmbackup \
+	vmrestore
+
+release: \
+	release-victoria-metrics \
+	release-vmutils
+
+release-victoria-metrics: victoria-metrics-prod
+	cd bin && tar czf victoria-metrics-$(PKG_TAG).tar.gz victoria-metrics-prod && \
+		sha256sum victoria-metrics-$(PKG_TAG).tar.gz > victoria-metrics-$(PKG_TAG)_checksums.txt
+
+release-vmutils: \
+	vmagent-prod \
+	vmbackup-prod \
+	vmrestore-prod
+	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmagent-prod vmbackup-prod vmrestore-prod && \
+		sha256sum vmutils-$(PKG_TAG).tar.gz > vmutils-$(PKG_TAG)_checksums.txt
+
+pprof-cpu:
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
 fmt:
-	go fmt $(PKG_PREFIX)/lib/...
-	go fmt $(PKG_PREFIX)/app/...
+	GO111MODULE=on gofmt -l -w -s ./lib
+	GO111MODULE=on gofmt -l -w -s ./app
 
 vet:
-	go vet $(PKG_PREFIX)/lib/...
-	go vet $(PKG_PREFIX)/app/...
+	GO111MODULE=on go vet -mod=vendor ./lib/...
+	GO111MODULE=on go vet -mod=vendor ./app/...
 
 lint: install-golint
 	golint lib/...
 	golint app/...
 
 install-golint:
-	which golint || GO111MODULE=off go get -u github.com/golang/lint/golint
+	which golint || GO111MODULE=off go get -u golang.org/x/lint/golint
 
 errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./lib/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
 
 install-errcheck:
 	which errcheck || GO111MODULE=off go get -u github.com/kisielk/errcheck
 
+check-all: fmt vet lint errcheck golangci-lint
+
 test:
-	go test $(PKG_PREFIX)/lib/...
+	GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
+
+test-pure:
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
+
+test-full:
+	GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+
+test-full-386:
+	GO111MODULE=on GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
-	go test -bench=. $(PKG_PREFIX)/lib/...
+	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
+	GO111MODULE=on go test -mod=vendor -bench=. ./app/...
+
+benchmark-pure:
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./lib/...
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor -bench=. ./app/...
 
 vendor-update:
-	go get -u
-	go mod tidy
-	go mod vendor
+	GO111MODULE=on go get -u ./lib/...
+	GO111MODULE=on go get -u ./app/...
+	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
+
+app-local:
+	CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+
+app-local-pure:
+	CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc
 
 install-qtc:
 	which qtc || GO111MODULE=off go get -u github.com/valyala/quicktemplate/qtc
+
+
+golangci-lint: install-golangci-lint
+	golangci-lint run --exclude '(SA4003|SA1019):' -D errcheck -D structcheck
+
+install-golangci-lint:
+	which golangci-lint || GO111MODULE=off go get -u github.com/golangci/golangci-lint/cmd/golangci-lint

app/victoria-metrics/Makefile

@@ -1,21 +1,107 @@
 # All these commands must run from repository root.
 
+victoria-metrics:
+	APP_NAME=victoria-metrics $(MAKE) app-local
+
 victoria-metrics-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker
 
+victoria-metrics-pure-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-pure
+
+victoria-metrics-amd64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-amd64
+
+victoria-metrics-arm-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm
+
+victoria-metrics-arm64-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-arm64
+
+victoria-metrics-ppc64le-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-ppc64le
+
+victoria-metrics-386-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-386
+
 package-victoria-metrics:
-	APP_NAME=victoria-metrics \
-	$(MAKE) package-via-docker
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker
+
+package-victoria-metrics-pure:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-pure
+
+package-victoria-metrics-amd64:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-amd64
+
+package-victoria-metrics-arm:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-arm
+
+package-victoria-metrics-arm64:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-arm64
+
+package-victoria-metrics-ppc64le:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-ppc64le
+
+package-victoria-metrics-386:
+	APP_NAME=victoria-metrics $(MAKE) package-via-docker-386
 
 publish-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) publish-via-docker
 
 run-victoria-metrics:
 	mkdir -p victoria-metrics-data
-	DOCKER_OPTS='-v $(shell pwd)/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 -p 2003:2003 -p 2003:2003/udp' \
+	DOCKER_OPTS='-v $(shell pwd)/victoria-metrics-data:/victoria-metrics-data' \
 	APP_NAME=victoria-metrics \
 	ARGS='-graphiteListenAddr=:2003 -opentsdbListenAddr=:4242 -retentionPeriod=12 -search.maxUniqueTimeseries=1000000 -search.maxQueryDuration=10m' \
 	$(MAKE) run-via-docker
 
+victoria-metrics-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-amd64 ./app/victoria-metrics
+
 victoria-metrics-arm:
-	CC=arm-linux-gnueabi-gcc CGO_ENABLED=1 GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm ./app/victoria-metrics
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm ./app/victoria-metrics
+
+victoria-metrics-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-arm64 ./app/victoria-metrics
+
+victoria-metrics-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-ppc64le ./app/victoria-metrics
+
+victoria-metrics-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/victoria-metrics-386 ./app/victoria-metrics
+
+victoria-metrics-pure:
+	APP_NAME=victoria-metrics $(MAKE) app-local-pure
+
+### Packaging as DEB - amd64
+victoria-metrics-package-deb: victoria-metrics-prod
+	./package/package_deb.sh amd64
+
+### Packaging as DEB - arm64
+victoria-metrics-package-deb-arm64: victoria-metrics-arm64-prod
+	./package/package_deb.sh arm64
+
+### Packaging as DEB - all
+victoria-metrics-package-deb-all: \
+        victoria-metrics-package-deb \
+        victoria-metrics-package-deb-arm64
+
+### Packaging as RPM - amd64
+victoria-metrics-package-rpm: victoria-metrics-prod
+	./package/package_rpm.sh amd64
+
+### Packaging as RPM - arm64
+victoria-metrics-package-rpm-arm64: victoria-metrics-arm64-prod
+	./package/package_rpm.sh arm64
+
+### Packaging as RPM - all
+victoria-metrics-package-rpm-all: \
+        victoria-metrics-package-rpm \
+        victoria-metrics-package-rpm-arm64
+
+### Packaging as both DEB and RPM - all
+victoria-metrics-package-deb-rpm-all: \
+        victoria-metrics-package-deb \
+        victoria-metrics-package-deb-arm64 \
+        victoria-metrics-package-rpm \
+        victoria-metrics-package-rpm-arm64

app/victoria-metrics/deployment/Dockerfile

@@ -1,5 +1,8 @@
+ARG certs_image
+FROM $certs_image AS certs
 FROM scratch
-COPY --from=local/certs:1.0.2 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY bin/victoria-metrics-prod .
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG src_binary
+COPY $src_binary ./victoria-metrics-prod
 EXPOSE 8428
 ENTRYPOINT ["/victoria-metrics-prod"]

app/victoria-metrics/main.go

@@ -9,41 +9,55 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
-var httpListenAddr = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
+var (
+	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
+	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+
+		"This may be useful for reducing overhead when multiple identically configured Prometheus instances write data to the same VictoriaMetrics. "+
+		"Deduplication is disabled if the -dedup.minScrapeInterval is 0")
+)
 
 func main() {
-	flag.Parse()
+	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
-	logger.Infof("starting VictoraMetrics at %q...", *httpListenAddr)
+	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
+	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
 	vmstorage.Init()
 	vmselect.Init()
 	vminsert.Init()
+	startSelfScraper()
 
 	go httpserver.Serve(*httpListenAddr, requestHandler)
-	logger.Infof("started VictoriaMetrics in %s", time.Since(startTime))
+	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
 
+	stopSelfScraper()
+
 	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
-	logger.Infof("successfully shut down the webservice in %s", time.Since(startTime))
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	vmstorage.Stop()
 	vmselect.Stop()
 
-	logger.Infof("the VictoriaMetrics has been stopped in %s", time.Since(startTime))
+	fs.MustStopDirRemover()
+
+	logger.Infof("the VictoriaMetrics has been stopped in %.3f seconds", time.Since(startTime).Seconds())
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {

app/victoria-metrics/main_test.go

@@ -0,0 +1,496 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	testutil "github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics/test"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+const (
+	testFixturesDir            = "testdata"
+	testStorageSuffix          = "vm-test-storage"
+	testHTTPListenAddr         = ":7654"
+	testStatsDListenAddr       = ":2003"
+	testOpenTSDBListenAddr     = ":4242"
+	testOpenTSDBHTTPListenAddr = ":4243"
+	testLogLevel               = "INFO"
+)
+
+const (
+	testReadHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr
+	testWriteHTTPPath         = "http://127.0.0.1" + testHTTPListenAddr + "/write"
+	testOpenTSDBWriteHTTPPath = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
+	testPromWriteHTTPPath     = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
+	testHealthHTTPPath        = "http://127.0.0.1" + testHTTPListenAddr + "/health"
+)
+
+const (
+	testStorageInitTimeout = 10 * time.Second
+)
+
+var (
+	storagePath   string
+	insertionTime = time.Now().UTC()
+)
+
+type test struct {
+	Name             string     `json:"name"`
+	Data             []string   `json:"data"`
+	Query            []string   `json:"query"`
+	ResultMetrics    []Metric   `json:"result_metrics"`
+	ResultSeries     Series     `json:"result_series"`
+	ResultQuery      Query      `json:"result_query"`
+	ResultQueryRange QueryRange `json:"result_query_range"`
+	Issue            string     `json:"issue"`
+}
+
+type Metric struct {
+	Metric     map[string]string `json:"metric"`
+	Values     []float64         `json:"values"`
+	Timestamps []int64           `json:"timestamps"`
+}
+
+func (r *Metric) UnmarshalJSON(b []byte) error {
+	type plain Metric
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
+type Series struct {
+	Status string              `json:"status"`
+	Data   []map[string]string `json:"data"`
+}
+type Query struct {
+	Status string    `json:"status"`
+	Data   QueryData `json:"data"`
+}
+type QueryData struct {
+	ResultType string            `json:"resultType"`
+	Result     []QueryDataResult `json:"result"`
+}
+
+type QueryDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Value  []interface{}     `json:"value"`
+}
+
+func (r *QueryDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
+type QueryRange struct {
+	Status string         `json:"status"`
+	Data   QueryRangeData `json:"data"`
+}
+type QueryRangeData struct {
+	ResultType string                 `json:"resultType"`
+	Result     []QueryRangeDataResult `json:"result"`
+}
+
+type QueryRangeDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Values [][]interface{}   `json:"values"`
+}
+
+func (r *QueryRangeDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryRangeDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
+}
+
+func TestMain(m *testing.M) {
+	setUp()
+	code := m.Run()
+	tearDown()
+	os.Exit(code)
+}
+
+func setUp() {
+	storagePath = filepath.Join(os.TempDir(), testStorageSuffix)
+	processFlags()
+	logger.Init()
+	vmstorage.InitWithoutMetrics()
+	vmselect.Init()
+	vminsert.Init()
+	go httpserver.Serve(*httpListenAddr, requestHandler)
+	readyStorageCheckFunc := func() bool {
+		resp, err := http.Get(testHealthHTTPPath)
+		if err != nil {
+			return false
+		}
+		resp.Body.Close()
+		return resp.StatusCode == 200
+	}
+	if err := waitFor(testStorageInitTimeout, readyStorageCheckFunc); err != nil {
+		log.Fatalf("http server can't start for %s seconds, err %s", testStorageInitTimeout, err)
+	}
+}
+
+func processFlags() {
+	envflag.Parse()
+	for _, fv := range []struct {
+		flag  string
+		value string
+	}{
+		{flag: "storageDataPath", value: storagePath},
+		{flag: "httpListenAddr", value: testHTTPListenAddr},
+		{flag: "graphiteListenAddr", value: testStatsDListenAddr},
+		{flag: "opentsdbListenAddr", value: testOpenTSDBListenAddr},
+		{flag: "loggerLevel", value: testLogLevel},
+		{flag: "opentsdbHTTPListenAddr", value: testOpenTSDBHTTPListenAddr},
+	} {
+		// panics if flag doesn't exist
+		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
+			log.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
+		}
+	}
+}
+
+func waitFor(timeout time.Duration, f func() bool) error {
+	fraction := timeout / 10
+	for i := fraction; i < timeout; i += fraction {
+		if f() {
+			return nil
+		}
+		time.Sleep(fraction)
+	}
+	return fmt.Errorf("timeout")
+}
+
+func tearDown() {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		log.Printf("cannot stop the webservice: %s", err)
+	}
+	vminsert.Stop()
+	vmstorage.Stop()
+	vmselect.Stop()
+	fs.MustRemoveAll(storagePath)
+}
+
+func TestWriteRead(t *testing.T) {
+	t.Run("write", testWrite)
+	time.Sleep(1 * time.Second)
+	vmstorage.Stop()
+	// open storage after stop in write
+	vmstorage.InitWithoutMetrics()
+	t.Run("read", testRead)
+}
+
+func testWrite(t *testing.T) {
+	t.Run("prometheus", func(t *testing.T) {
+		for _, test := range readIn("prometheus", t, insertionTime) {
+			s := newSuite(t)
+			r := testutil.WriteRequest{}
+			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
+			data, err := testutil.Compress(r)
+			s.greaterThan(len(r.Timeseries), 0)
+			if err != nil {
+				t.Errorf("error compressing %v %s", r, err)
+				t.Fail()
+			}
+			httpWrite(t, testPromWriteHTTPPath, bytes.NewBuffer(data))
+		}
+	})
+
+	t.Run("influxdb", func(t *testing.T) {
+		for _, x := range readIn("influxdb", t, insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				httpWrite(t, testWriteHTTPPath, bytes.NewBufferString(strings.Join(test.Data, "\n")))
+			})
+		}
+	})
+	t.Run("graphite", func(t *testing.T) {
+		for _, x := range readIn("graphite", t, insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				tcpWrite(t, "127.0.0.1"+testStatsDListenAddr, strings.Join(test.Data, "\n"))
+			})
+		}
+	})
+	t.Run("opentsdb", func(t *testing.T) {
+		for _, x := range readIn("opentsdb", t, insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				tcpWrite(t, "127.0.0.1"+testOpenTSDBListenAddr, strings.Join(test.Data, "\n"))
+			})
+		}
+	})
+	t.Run("opentsdbhttp", func(t *testing.T) {
+		for _, x := range readIn("opentsdbhttp", t, insertionTime) {
+			test := x
+			t.Run(test.Name, func(t *testing.T) {
+				t.Parallel()
+				logger.Infof("writing %s", test.Data)
+				httpWrite(t, testOpenTSDBWriteHTTPPath, bytes.NewBufferString(strings.Join(test.Data, "\n")))
+			})
+		}
+	})
+}
+
+func testRead(t *testing.T) {
+	for _, engine := range []string{"prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
+		t.Run(engine, func(t *testing.T) {
+			for _, x := range readIn(engine, t, insertionTime) {
+				test := x
+				t.Run(test.Name, func(t *testing.T) {
+					t.Parallel()
+					for _, q := range test.Query {
+						q = testutil.PopulateTimeTplString(q, insertionTime)
+						if test.Issue != "" {
+							test.Issue = "Regression in " + test.Issue
+						}
+						switch true {
+						case strings.HasPrefix(q, "/api/v1/export"):
+							if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil {
+								t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/series"):
+							s := Series{}
+							httpReadStruct(t, testReadHTTPPath, q, &s)
+							if err := checkSeriesResult(s, test.ResultSeries); err != nil {
+								t.Fatalf("Series. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/query_range"):
+							queryResult := QueryRange{}
+							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
+							if err := checkQueryRangeResult(queryResult, test.ResultQueryRange); err != nil {
+								t.Fatalf("Query Range. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						case strings.HasPrefix(q, "/api/v1/query"):
+							queryResult := Query{}
+							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
+							if err := checkQueryResult(queryResult, test.ResultQuery); err != nil {
+								t.Fatalf("Query. %s fails with error %s.%s", q, err, test.Issue)
+							}
+						default:
+							t.Fatalf("unsupported read query %s", q)
+						}
+					}
+				})
+			}
+		})
+	}
+}
+
+func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
+	t.Helper()
+	s := newSuite(t)
+	var tt []test
+	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if filepath.Ext(path) != ".json" {
+			return nil
+		}
+		b, err := ioutil.ReadFile(path)
+		s.noError(err)
+		item := test{}
+		s.noError(json.Unmarshal(b, &item))
+		for i := range item.Data {
+			item.Data[i] = testutil.PopulateTimeTplString(item.Data[i], insertTime)
+		}
+		tt = append(tt, item)
+		return nil
+	}))
+	if len(tt) == 0 {
+		t.Fatalf("no test found in %s", filepath.Join(testFixturesDir, readFor))
+	}
+	return tt
+}
+
+func httpWrite(t *testing.T, address string, r io.Reader) {
+	t.Helper()
+	s := newSuite(t)
+	resp, err := http.Post(address, "", r)
+	s.noError(err)
+	s.noError(resp.Body.Close())
+	s.equalInt(resp.StatusCode, 204)
+}
+
+func tcpWrite(t *testing.T, address string, data string) {
+	t.Helper()
+	s := newSuite(t)
+	conn, err := net.Dial("tcp", address)
+	s.noError(err)
+	defer conn.Close()
+	n, err := conn.Write([]byte(data))
+	s.noError(err)
+	s.equalInt(n, len(data))
+}
+
+func httpReadMetrics(t *testing.T, address, query string) []Metric {
+	t.Helper()
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
+	defer resp.Body.Close()
+	s.equalInt(resp.StatusCode, 200)
+	var rows []Metric
+	for dec := json.NewDecoder(resp.Body); dec.More(); {
+		var row Metric
+		s.noError(dec.Decode(&row))
+		rows = append(rows, row)
+	}
+	return rows
+}
+func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
+	t.Helper()
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
+	defer resp.Body.Close()
+	s.equalInt(resp.StatusCode, 200)
+	s.noError(json.NewDecoder(resp.Body).Decode(dst))
+}
+
+func checkMetricsResult(got, want []Metric) error {
+	for _, r := range append([]Metric(nil), got...) {
+		want = removeIfFoundMetrics(r, want)
+	}
+	if len(want) > 0 {
+		return fmt.Errorf("exptected metrics %+v not found in %+v", want, got)
+	}
+	return nil
+}
+
+func removeIfFoundMetrics(r Metric, contains []Metric) []Metric {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) &&
+			reflect.DeepEqual(r.Timestamps, item.Timestamps) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkSeriesResult(got, want Series) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	wantData := append([]map[string]string(nil), want.Data...)
+	for _, r := range got.Data {
+		wantData = removeIfFoundSeries(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected seria(s) %+v not found in %+v", wantData, got.Data)
+	}
+	return nil
+}
+
+func removeIfFoundSeries(r map[string]string, contains []map[string]string) []map[string]string {
+	for i, item := range contains {
+		if reflect.DeepEqual(r, item) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkQueryResult(got, want Query) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryData(r QueryDataResult, contains []QueryDataResult) []QueryDataResult {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Value[0], item.Value[0]) && reflect.DeepEqual(r.Value[1], item.Value[1]) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkQueryRangeResult(got, want QueryRange) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryRangeDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryRangeData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query range result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryRangeData(r QueryRangeDataResult, contains []QueryRangeDataResult) []QueryRangeDataResult {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+type suite struct{ t *testing.T }
+
+func newSuite(t *testing.T) *suite { return &suite{t: t} }
+
+func (s *suite) noError(err error) {
+	s.t.Helper()
+	if err != nil {
+		s.t.Errorf("unexpected error %v", err)
+		s.t.FailNow()
+	}
+}
+
+func (s *suite) equalInt(a, b int) {
+	s.t.Helper()
+	if a != b {
+		s.t.Errorf("%d not equal %d", a, b)
+		s.t.FailNow()
+	}
+}
+
+func (s *suite) greaterThan(a, b int) {
+	s.t.Helper()
+	if a <= b {
+		s.t.Errorf("%d less or equal then %d", a, b)
+		s.t.FailNow()
+	}
+}

app/victoria-metrics/self_scraper.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"flag"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+var selfScrapeInterval = flag.Duration("selfScrapeInterval", 0, "Interval for self-scraping own metrics at /metrics page")
+
+var selfScraperStopCh chan struct{}
+var selfScraperWG sync.WaitGroup
+
+func startSelfScraper() {
+	selfScraperStopCh = make(chan struct{})
+	selfScraperWG.Add(1)
+	go func() {
+		defer selfScraperWG.Done()
+		selfScraper(*selfScrapeInterval)
+	}()
+}
+
+func stopSelfScraper() {
+	close(selfScraperStopCh)
+	selfScraperWG.Wait()
+}
+
+func selfScraper(scrapeInterval time.Duration) {
+	if scrapeInterval <= 0 {
+		// Self-scrape is disabled.
+		return
+	}
+	logger.Infof("started self-scraping `/metrics` page with interval %.3f seconds", scrapeInterval.Seconds())
+
+	var bb bytesutil.ByteBuffer
+	var rows prometheus.Rows
+	var mrs []storage.MetricRow
+	var labels []prompb.Label
+	t := time.NewTicker(scrapeInterval)
+	var currentTimestamp int64
+	for {
+		select {
+		case <-selfScraperStopCh:
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			currentTimestamp = currentTime.UnixNano() / 1e6
+		}
+		bb.Reset()
+		httpserver.WritePrometheusMetrics(&bb)
+		s := bytesutil.ToUnsafeString(bb.B)
+		rows.Reset()
+		rows.Unmarshal(s)
+		mrs = mrs[:0]
+		for i := range rows.Rows {
+			r := &rows.Rows[i]
+			labels = labels[:0]
+			labels = addLabel(labels, "", r.Metric)
+			labels = addLabel(labels, "job", "victoria-metrics")
+			labels = addLabel(labels, "instance", "self")
+			for j := range r.Tags {
+				t := &r.Tags[j]
+				labels = addLabel(labels, t.Key, t.Value)
+			}
+			if len(mrs) < cap(mrs) {
+				mrs = mrs[:len(mrs)+1]
+			} else {
+				mrs = append(mrs, storage.MetricRow{})
+			}
+			mr := &mrs[len(mrs)-1]
+			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
+			mr.Timestamp = currentTimestamp
+			mr.Value = r.Value
+		}
+		logger.Infof("writing %d rows at timestamp %d", len(mrs), currentTimestamp)
+		vmstorage.AddRows(mrs)
+	}
+}
+
+func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
+	if len(dst) < cap(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, prompb.Label{})
+	}
+	lb := &dst[len(dst)-1]
+	lb.Name = bytesutil.ToUnsafeBytes(key)
+	lb.Value = bytesutil.ToUnsafeBytes(value)
+	return dst
+}

app/victoria-metrics/test/parser.go

@@ -0,0 +1,52 @@
+package test
+
+import (
+	"fmt"
+	"log"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var (
+	parseTimeExpRegex = regexp.MustCompile(`"?{TIME[^}]*}"?`)
+	extractRegex      = regexp.MustCompile(`"?{([^}]*)}"?`)
+)
+
+// PopulateTimeTplString substitutes {TIME_*} with t in s and returns the result.
+func PopulateTimeTplString(s string, t time.Time) string {
+	return string(PopulateTimeTpl([]byte(s), t))
+}
+
+// PopulateTimeTpl substitutes {TIME_*} with tGlobal in b and returns the result.
+func PopulateTimeTpl(b []byte, tGlobal time.Time) []byte {
+	return parseTimeExpRegex.ReplaceAllFunc(b, func(repl []byte) []byte {
+		t := tGlobal
+		repl = extractRegex.FindSubmatch(repl)[1]
+		parts := strings.SplitN(string(repl), "-", 2)
+		if len(parts) == 2 {
+			duration, err := time.ParseDuration(strings.TrimSpace(parts[1]))
+			if err != nil {
+				log.Fatalf("error %s parsing duration %s in %s", err, parts[1], repl)
+			}
+			t = t.Add(-duration)
+		}
+		switch strings.TrimSpace(parts[0]) {
+		case `TIME_S`:
+			return []byte(fmt.Sprintf("%d", t.Unix()))
+		case `TIME_MSZ`:
+			return []byte(fmt.Sprintf("%d", t.Unix()*1e3))
+		case `TIME_MS`:
+			return []byte(fmt.Sprintf("%d", timeToMillis(t)))
+		case `TIME_NS`:
+			return []byte(fmt.Sprintf("%d", t.UnixNano()))
+		default:
+			log.Fatalf("unknown time pattern %s in %s", parts[0], repl)
+		}
+		return repl
+	})
+}
+
+func timeToMillis(t time.Time) int64 {
+	return t.UnixNano() / 1e6
+}

app/victoria-metrics/test/parser_test.go

@@ -0,0 +1,24 @@
+package test
+
+import (
+	"testing"
+	"time"
+)
+
+func TestPopulateTimeTplString(t *testing.T) {
+	now, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
+	if err != nil {
+		t.Fatalf("unexpected error when parsing time: %s", err)
+	}
+	f := func(s, resultExpected string) {
+		t.Helper()
+		result := PopulateTimeTplString(s, now)
+		if result != resultExpected {
+			t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
+		}
+	}
+	f("", "")
+	f("{TIME_S}", "1136214245")
+	f("now: {TIME_S}, past 30s: {TIME_MS-30s}, now: {TIME_S}", "now: 1136214245, past 30s: 1136214215000, now: 1136214245")
+	f("now: {TIME_MS}, past 30m: {TIME_MSZ-30m}, past 2h: {TIME_NS-2h}", "now: 1136214245000, past 30m: 1136212445000, past 2h: 1136207045000000000")
+}

app/victoria-metrics/test/prom_types.go

@@ -0,0 +1,364 @@
+package test
+
+// Source https://github.com/prometheus/prometheus/blob/master/prompb/remote.pb.go . Code is copy pasted and cleaned up
+import (
+	"encoding/binary"
+	"math"
+	"math/bits"
+)
+
+// WriteRequest is write request
+type WriteRequest struct {
+	Timeseries []TimeSeries `protobuf:"bytes,1,rep,name=timeseries,proto3" json:"timeseries"`
+}
+
+// Size returns m size in bytes after marshaling.
+func (m *WriteRequest) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Timeseries) > 0 {
+		for _, e := range m.Timeseries {
+			l = e.Size()
+			n += 1 + l + sovRemote(uint64(l))
+		}
+	}
+	return n
+}
+func sovRemote(x uint64) (n int) {
+	return (bits.Len64(x|1) + 6) / 7
+}
+
+// Marshal marshals m.
+func (m *WriteRequest) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA
+func (m *WriteRequest) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *WriteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if len(m.Timeseries) > 0 {
+		for iNdEx := len(m.Timeseries) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Timeseries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintRemote(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintRemote(dAtA []byte, offset int, v uint64) int {
+	offset -= sovRemote(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+
+// Sample is time series sample.
+type Sample struct {
+	Value     float64 `protobuf:"fixed64,1,opt,name=value,proto3" json:"value,omitempty"`
+	Timestamp int64   `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+}
+
+// Reset resets m.
+func (m *Sample) Reset() { *m = Sample{} }
+
+// TimeSeries represents samples and labels for a single time series.
+type TimeSeries struct {
+	Labels  []Label  `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"`
+	Samples []Sample `protobuf:"bytes,2,rep,name=samples,proto3" json:"samples"`
+}
+
+// Reset resets m.
+func (m *TimeSeries) Reset() { *m = TimeSeries{} }
+
+// Label is time series label.
+type Label struct {
+	Name  string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+}
+
+// Reset resets m.
+func (m *Label) Reset() { *m = Label{} }
+
+// Labels is a set of labels.
+type Labels struct {
+	Labels []Label `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"`
+}
+
+// Reset resets m.
+func (m *Labels) Reset() { *m = Labels{} }
+
+// Marshal marshals m.
+func (m *Sample) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA.
+func (m *Sample) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *Sample) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if m.Timestamp != 0 {
+		i = encodeVarintTypes(dAtA, i, uint64(m.Timestamp))
+		i--
+		dAtA[i] = 0x10
+	}
+	if m.Value != 0 {
+		i -= 8
+		binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.Value))))
+		i--
+		dAtA[i] = 0x9
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal marshals m.
+func (m *TimeSeries) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA.
+func (m *TimeSeries) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *TimeSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if len(m.Samples) > 0 {
+		for iNdEx := len(m.Samples) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Samples[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Labels) > 0 {
+		for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Labels[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal marshals m.
+func (m *Label) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA.
+func (m *Label) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *Label) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Value) > 0 {
+		i -= len(m.Value)
+		copy(dAtA[i:], m.Value)
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Value)))
+		i--
+		dAtA[i] = 0x12
+	}
+	if len(m.Name) > 0 {
+		i -= len(m.Name)
+		copy(dAtA[i:], m.Name)
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Name)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal marshals m.
+func (m *Labels) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dAtA.
+func (m *Labels) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to dAtA.
+func (m *Labels) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if len(m.Labels) > 0 {
+		for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Labels[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintTypes(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintTypes(dAtA []byte, offset int, v uint64) int {
+	offset -= sovTypes(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+
+// Size returns the size of marshaled m.
+func (m *Sample) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	if m.Value != 0 {
+		n += 9
+	}
+	if m.Timestamp != 0 {
+		n += 1 + sovTypes(uint64(m.Timestamp))
+	}
+	return n
+}
+
+// Size returns the size of marshaled m.
+func (m *TimeSeries) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Labels) > 0 {
+		for _, e := range m.Labels {
+			l = e.Size()
+			n += 1 + l + sovTypes(uint64(l))
+		}
+	}
+	if len(m.Samples) > 0 {
+		for _, e := range m.Samples {
+			l = e.Size()
+			n += 1 + l + sovTypes(uint64(l))
+		}
+	}
+	return n
+}
+
+// Size returns the size of marshaled m.
+func (m *Label) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Name)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	l = len(m.Value)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	return n
+}
+
+// Size returns the size of marshaled m.
+func (m *Labels) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Labels) > 0 {
+		for _, e := range m.Labels {
+			l = e.Size()
+			n += 1 + l + sovTypes(uint64(l))
+		}
+	}
+	return n
+}
+
+func sovTypes(x uint64) (n int) {
+	return (bits.Len64(x|1) + 6) / 7
+}

app/victoria-metrics/test/prom_writter.go

@@ -0,0 +1,12 @@
+package test
+
+import "github.com/golang/snappy"
+
+// Compress marshals and compresses wr.
+func Compress(wr WriteRequest) ([]byte, error) {
+	data, err := wr.Marshal()
+	if err != nil {
+		return nil, err
+	}
+	return snappy.Encode(nil, data), nil
+}

app/victoria-metrics/testdata/graphite/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["graphite.foo.bar.baz;tag1=value1;tag2=value2 123 {TIME_S}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"graphite.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -0,0 +1,16 @@
+{
+  "name": "comparison-not-inf-not-nan",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150",
+  "data": [
+    "not_nan_not_inf;item=x 1 {TIME_S-1m}",
+    "not_nan_not_inf;item=x 1 {TIME_S-2m}",
+    "not_nan_not_inf;item=y 3 {TIME_S-1m}",
+    "not_nan_not_inf;item=y 1 {TIME_S-2m}"],
+  "query": ["/api/v1/query_range?query=1/(not_nan_not_inf-1)!=inf!=nan&start={TIME_S-3m}&end={TIME_S}&step=60"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"item":"y"},"values":[["{TIME_S-1m}","0.5"],["{TIME_S}","0.5"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -0,0 +1,21 @@
+{
+  "name": "max_lookback_set",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/209",
+  "data": [
+    "max_lookback_set 1 {TIME_S-30s}",
+    "max_lookback_set 2 {TIME_S-60s}",
+    "max_lookback_set 3 {TIME_S-120s}",
+    "max_lookback_set 4 {TIME_S-150s}"
+  ],
+  "query": ["/api/v1/query_range?query=max_lookback_set&start={TIME_S-150s}&end={TIME_S}&step=10s&max_lookback=1s"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[{"metric":{"__name__":"max_lookback_set"},"values":[
+	      ["{TIME_S-150s}","4"],
+	      ["{TIME_S-120s}","3"],
+	      ["{TIME_S-60s}","2"],
+	      ["{TIME_S-30s}","1"],
+	      ["{TIME_S-20s}","1"]
+      ]}]}}
+}

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -0,0 +1,30 @@
+{
+  "name": "max_lookback_unset",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/209",
+  "data": [
+    "max_lookback_unset 1 {TIME_S-30s}",
+    "max_lookback_unset 2 {TIME_S-60s}",
+    "max_lookback_unset 3 {TIME_S-120s}",
+    "max_lookback_unset 4 {TIME_S-150s}"
+  ],
+  "query": ["/api/v1/query_range?query=max_lookback_unset&start={TIME_S-150s}&end={TIME_S}&step=10s"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[{"metric":{"__name__":"max_lookback_unset"},"values":[
+	      ["{TIME_S-150s}","4"],
+	      ["{TIME_S-140s}","4"],
+	      ["{TIME_S-130s}","4"],
+	      ["{TIME_S-120s}","3"],
+	      ["{TIME_S-110s}","3"],
+	      ["{TIME_S-100s}","3"],
+	      ["{TIME_S-90s}","3"],
+	      ["{TIME_S-60s}","2"],
+	      ["{TIME_S-50s}","2"],
+	      ["{TIME_S-40s}","2"],
+	      ["{TIME_S-30s}","1"],
+	      ["{TIME_S-20s}","1"],
+	      ["{TIME_S-10s}","1"],
+	      ["{TIME_S-0s}","1"]
+      ]}]}}
+}

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -0,0 +1,18 @@
+{
+  "name": "not-nan-as-missing-data",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153",
+  "data": [
+    "not_nan_as_missing_data;item=x 2 {TIME_S-2m}",
+    "not_nan_as_missing_data;item=x 1 {TIME_S-1m}",
+    "not_nan_as_missing_data;item=y 4 {TIME_S-2m}",
+    "not_nan_as_missing_data;item=y 3 {TIME_S-1m}"
+  ],
+  "query": ["/api/v1/query_range?query=not_nan_as_missing_data>1&start={TIME_S-2m}&end={TIME_S}&step=60"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"x"},"values":[["{TIME_S-2m}","2"]]},
+	      {"metric":{"__name__":"not_nan_as_missing_data","item":"y"},"values":[["{TIME_S-2m}","4"],["{TIME_S-1m}","3"],["{TIME_S}","3"]]}
+      ]}}
+}

app/victoria-metrics/testdata/graphite/subquery-aggregation.json

@@ -0,0 +1,14 @@
+{
+  "name": "subquery-aggregation",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/184",
+  "data": [
+    "forms_daily_count;item=x 1 {TIME_S-1m}",
+    "forms_daily_count;item=x 2 {TIME_S-2m}",
+    "forms_daily_count;item=y 3 {TIME_S-1m}",
+    "forms_daily_count;item=y 4 {TIME_S-2m}"],
+  "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}"],
+  "result_query": {
+    "status":"success",
+    "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","1"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","3"]}]}
+  }
+}

app/victoria-metrics/testdata/influxdb/basic.json

@@ -0,0 +1,9 @@
+{
+  "name": "basic_insertion",
+  "data": ["measurement,tag1=value1,tag2=value2 field1=1.23,field2=123 {TIME_NS}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MS}"]},
+    {"metric":{"__name__":"measurement_field1","tag1":"value1","tag2":"value2"},"values":[1.23], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdb/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["put openstdb.foo.bar.baz {TIME_S} 123 tag1=value1 tag2=value2"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"openstdb.foo.bar.baz","tag1":"value1","tag2":"value2"},"values":[123], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdbhttp/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["{\"metric\": \"opentsdbhttp.foo\", \"value\": 1001, \"timestamp\": {TIME_S}, \"tags\": {\"bar\":\"baz\", \"x\": \"y\"}}"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"opentsdbhttp.foo","bar":"baz","x":"y"},"values":[1001], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/opentsdbhttp/multi_line.json

@@ -0,0 +1,9 @@
+{
+  "name": "multiline",
+  "data": ["[{\"metric\": \"opentsdbhttp.multiline1\", \"value\": 1001, \"timestamp\": \"{TIME_S}\", \"tags\": {\"bar\":\"baz\", \"x\": \"y\"}}, {\"metric\": \"opentsdbhttp.multiline2\", \"value\": 1002, \"timestamp\": {TIME_S}}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"opentsdbhttp.multiline1","bar":"baz","x":"y"},"values":[1001], "timestamps": ["{TIME_MSZ}"]},
+    {"metric":{"__name__":"opentsdbhttp.multiline2"},"values":[1002], "timestamps": ["{TIME_MSZ}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/basic.json

@@ -0,0 +1,8 @@
+{
+  "name": "basic_insertion",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.bar\"},{\"name\":\"baz\",\"value\":\"qux\"}],\"samples\":[{\"value\":100000,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.bar","baz":"qux"},"values":[100000], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/case-sensitive-regex.json

@@ -0,0 +1,10 @@
+{
+  "name": "case-sensitive-regex",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/161",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.sensitiveRegex\"},{\"name\":\"label\",\"value\":\"sensitiveRegex\"}],\"samples\":[{\"value\":2,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.sensitiveRegex\"},{\"name\":\"label\",\"value\":\"SensitiveRegex\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={label=~'(?i)sensitiveregex'}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.sensitiveRegex","label":"sensitiveRegex"},"values":[2], "timestamps": ["{TIME_MS}"]},
+    {"metric":{"__name__":"prometheus.sensitiveRegex","label":"SensitiveRegex"},"values":[1], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/duplicate-label.json

@@ -0,0 +1,9 @@
+{
+  "name": "duplicate_label",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/172",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"prometheus.duplicate_label\"},{\"name\":\"duplicate\",\"value\":\"label\"},{\"name\":\"duplicate\",\"value\":\"label\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/export?match={__name__!=''}"],
+  "result_metrics": [
+    {"metric":{"__name__":"prometheus.duplicate_label","duplicate":"label"},"values":[1], "timestamps": ["{TIME_MS}"]}
+  ]
+}

app/victoria-metrics/testdata/prometheus/match-series.json

@@ -0,0 +1,15 @@
+{
+  "name": "match_series",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/155",
+  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"1\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"2\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"3\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]},{\"labels\":[{\"name\":\"__name__\",\"value\":\"MatchSeries\"},{\"name\":\"db\",\"value\":\"TenMinute\"},{\"name\":\"TurbineType\",\"value\":\"V112\"},{\"name\":\"Park\",\"value\":\"4\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS}\"}]}]"],
+  "query": ["/api/v1/series?match[]={__name__='MatchSeries'}", "/api/v1/series?match[]={__name__=~'MatchSeries.*'}"],
+  "result_series": {
+    "status": "success",
+    "data": [
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"1","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"2","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"3","TurbineType":"V112"},
+      {"__name__":"MatchSeries","db":"TenMinute","Park":"4","TurbineType":"V112"}
+    ]
+  }
+}

app/vmagent/Makefile

@@ -0,0 +1,73 @@
+# All these commands must run from repository root.
+
+vmagent:
+	APP_NAME=vmagent $(MAKE) app-local
+
+vmagent-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker
+
+vmagent-pure-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-pure
+
+vmagent-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-amd64
+
+vmagent-arm-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm
+
+vmagent-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm64
+
+vmagent-ppc64le-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-ppc64le
+
+vmagent-386-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-386
+
+package-vmagent:
+	APP_NAME=vmagent $(MAKE) package-via-docker
+
+package-vmagent-pure:
+	APP_NAME=vmagent $(MAKE) package-via-docker-pure
+
+package-vmagent-amd64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-amd64
+
+package-vmagent-arm:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm
+
+package-vmagent-arm64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm64
+
+package-vmagent-ppc64le:
+	APP_NAME=vmagent $(MAKE) package-via-docker-ppc64le
+
+package-vmagent-386:
+	APP_NAME=vmagent $(MAKE) package-via-docker-386
+
+publish-vmagent:
+	APP_NAME=vmagent $(MAKE) publish-via-docker
+
+run-vmagent:
+	mkdir -p vmagent-data
+	DOCKER_OPTS='-v $(shell pwd)/vmagent-data:/vmagent-data' \
+	APP_NAME=vmagent \
+	$(MAKE) run-via-docker
+
+vmagent-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-amd64 ./app/vmagent
+
+vmagent-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm ./app/vmagent
+
+vmagent-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm64 ./app/vmagent
+
+vmagent-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-ppc64le ./app/vmagent
+
+vmagent-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-386 ./app/vmagent
+
+vmagent-pure:
+	APP_NAME=vmagent $(MAKE) app-local-pure

app/vmagent/README.md

@@ -0,0 +1,159 @@
+## vmagent
+
+`vmagent` is a tiny but brave agent, which helps you collecting metrics from various sources
+and storing them to [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics).
+
+<img alt="vmagent" src="vmagent.png">
+
+
+### Motivation
+
+While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
+and RAM friendly to scrape metrics from Prometheus-compatible exporters to VictoriaMetrics.
+Also, we found that users’ infrastructure is like snowflakes - never alike, and we decided to add more flexibility
+to `vmagent` (like the ability to push metrics instead of pulling them). We did our best and plan to do even more.
+
+
+### Features
+
+* Can be used as drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter).
+  See [Quick Start](#quick-start) for details.
+* Can add, remove and modify labels via Prometheus relabeling. See [these docs](#relabeling) for details.
+* Accepts data via all the ingestion protocols supported by VictoriaMetrics:
+  * Influx line protocol via `http://<vmagent>:8429/write`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
+  * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
+  * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
+* Can replicate collected metrics simultaneously to multiple remote storage systems.
+* Works in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
+  are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as connection
+  to remote storage is recovered.
+* Uses lower amounts of RAM, CPU, disk IO and network bandwidth comparing to Prometheus.
+
+
+### Quick Start
+
+Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
+and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
+
+* `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. Multiple `-remoteWrite.url` args can be set in parallel
+  in order to replicate data concurrently to multiple remote storage systems.
+
+Example command line:
+
+```
+/path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+If you need collecting only Influx data, then the following command line would be enough:
+
+```
+/path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+Then send Influx data to `http://vmagent-host:8429/write`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
+
+`vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/).
+
+Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
+
+
+### How to collect metrics in Prometheus format?
+
+Pass the path to `prometheus.yml` to `-promscrape.config` command-line flag. `vmagent` takes into account the following
+sections from [Prometheus config file](https://prometheus.io/docs/prometheus/latest/configuration/configuration/):
+
+* `global`
+* `scrape_configs`
+
+All the other section are ignored, including [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) section.
+Use `-remoteWrite.*` command-line flags instead for configuring remote write settings:
+
+* `-remoteWrite.url` for pointing to remote storage. Data to remote storage can be sent either via HTTP or HTTPS. See `-remoteWrite.tls*` flags for details.
+* `-remoteWrite.label` for adding labels to metrics before sending them to remote storage.
+* `-remoteWrite.relabelConfig` for applying relabeling to metrics before sending them to remote storage.
+
+The following scrape types in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) section are supported:
+
+* `static_configs` - for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details.
+* `file_sd_configs` - for scraping targets defined in external files aka file-based service discover.
+  See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
+
+File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
+
+
+### Adding labels to metrics
+
+Labels can be added to metrics via the following mechanisms:
+
+* Via `global -> external_labels` section in `-promscrape.config` file. These labels are added only to metrics scraped from targets configured in `-promscrape.config` file.
+* Via `-remoteWrite.label` command-line flag. These labels are added to all the collected metrics before sending them to `-remoteWrite.url`.
+
+
+### Relabeling
+
+`vmagent` supports [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
+Additionally it provides the following extra actions:
+
+* `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`
+* `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`
+
+The relabeling can be defined in the following places:
+
+* At `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to targets when parsing the file during `vmagent` startup
+  or during config reload after sending `SIGHUP` signal to `vmagent`  via `kill -HUP`.
+* At `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to metrics after each scrape for the configured targets.
+* At `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to `-remoteWrite.url`.
+
+Read more about relabeling in the following articles:
+
+* [Life of a label](https://www.robustperception.io/life-of-a-label)
+* [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
+* [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
+* [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
+* [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
+
+
+### Monitoring
+
+`vmagent` exports various metrics in Prometheus exposition format at `/metrics` page. It is recommended setting up regular scraping of this page
+either via `vmagent` itself or via Prometheus, so the exported metrics could be analyzed later.
+
+
+### Troubleshooting
+
+* It is recommended increasing the maximum number of open files in the system (`ulimit -n`) when scraping big number of targets,
+  since `vmagent` establishes at least a single TCP connection per each target.
+
+* It is recommended increasing `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
+  and `vmagent_remotewrite_pending_data_bytes` metric exported by `vmagent` at `/metrics` page constantly grows.
+
+* `vmagent` buffers scraped data at `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
+  The directory can grow big when remote storage is unvailable during extended periods of time. If you don't want
+  sending all the data from the directory to remote storage, just stop `vmagent` and delete the directory.
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmagent` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make vmagent` from the root folder of the repository.
+   It builds `vmagent` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmagent-prod` from the root folder of the repository.
+   It builds `vmagent-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.

app/vmagent/common/push_ctx.go

@@ -0,0 +1,70 @@
+package common
+
+import (
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// PushCtx is a context used for populating WriteRequest.
+type PushCtx struct {
+	WriteRequest prompbmarshal.WriteRequest
+
+	// Labels contains flat list of all the labels used in WriteRequest.
+	Labels []prompbmarshal.Label
+
+	// Samples contains flat list of all the samples used in WriteRequest.
+	Samples []prompbmarshal.Sample
+}
+
+// Reset resets ctx.
+func (ctx *PushCtx) Reset() {
+	tss := ctx.WriteRequest.Timeseries
+	for i := range tss {
+		ts := &tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
+
+	labels := ctx.Labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	ctx.Labels = ctx.Labels[:0]
+
+	ctx.Samples = ctx.Samples[:0]
+}
+
+// GetPushCtx returns PushCtx from pool.
+//
+// Call PutPushCtx when the ctx is no longer needed.
+func GetPushCtx() *PushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*PushCtx)
+		}
+		return &PushCtx{}
+	}
+}
+
+// PutPushCtx returns ctx to the pool.
+//
+// ctx mustn't be used after returning to the pool.
+func PutPushCtx(ctx *PushCtx) {
+	ctx.Reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *PushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG certs_image
+FROM $certs_image AS certs
+FROM scratch
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG src_binary
+COPY $src_binary ./vmagent-prod
+EXPOSE 8429
+ENTRYPOINT ["/vmagent-prod"]

app/vmagent/graphite/request_handler.go

@@ -0,0 +1,65 @@
+package graphite
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="graphite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="graphite"}`)
+)
+
+// InsertHandler processes remote write for graphite plaintext protocol.
+//
+// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/influx/request_handler.go

@@ -0,0 +1,152 @@
+package influx
+
+import (
+	"flag"
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
+)
+
+// InsertHandler processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(db string, rows []parser.Row) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.ctx.Labels[:0]
+	samples := ctx.ctx.Samples[:0]
+	commonLabels := ctx.commonLabels[:0]
+	buf := ctx.buf[:0]
+	for i := range rows {
+		r := &rows[i]
+		commonLabels = commonLabels[:0]
+		hasDBLabel := false
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			if tag.Key == "db" {
+				hasDBLabel = true
+			}
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		if len(db) > 0 && !hasDBLabel {
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  "db",
+				Value: db,
+			})
+		}
+		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
+		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
+		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
+		}
+		for j := range r.Fields {
+			f := &r.Fields[j]
+			bufLen := len(buf)
+			buf = append(buf, ctx.metricGroupBuf...)
+			if !skipFieldKey {
+				buf = append(buf, f.Key...)
+			}
+			metricGroup := bytesutil.ToUnsafeString(buf[bufLen:])
+			labelsLen := len(labels)
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "__name__",
+				Value: metricGroup,
+			})
+			labels = append(labels, commonLabels...)
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: r.Timestamp,
+				Value:     f.Value,
+			})
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
+				Labels:  labels[labelsLen:],
+				Samples: samples[len(samples)-1:],
+			})
+		}
+		rowsTotal += len(r.Fields)
+	}
+	ctx.buf = buf
+	ctx.ctx.WriteRequest.Timeseries = tssDst
+	ctx.ctx.Labels = labels
+	ctx.ctx.Samples = samples
+	ctx.commonLabels = commonLabels
+	remotewrite.Push(&ctx.ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+
+	return nil
+}
+
+type pushCtx struct {
+	ctx            common.PushCtx
+	commonLabels   []prompbmarshal.Label
+	metricGroupBuf []byte
+	buf            []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.ctx.Reset()
+
+	commonLabels := ctx.commonLabels
+	for i := range commonLabels {
+		label := &commonLabels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+
+	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+	ctx.buf = ctx.buf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/main.go

@@ -0,0 +1,149 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
+	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
+	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	httpListenAddr     = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+)
+
+var (
+	graphiteServer     *graphiteserver.Server
+	opentsdbServer     *opentsdbserver.Server
+	opentsdbhttpServer *opentsdbhttpserver.Server
+)
+
+func main() {
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
+	startTime := time.Now()
+	remotewrite.Init()
+	writeconcurrencylimiter.Init()
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
+	}
+
+	promscrape.Init(remotewrite.Push)
+
+	go httpserver.Serve(*httpListenAddr, requestHandler)
+	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	startTime = time.Now()
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+
+	promscrape.Stop()
+
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer.MustStop()
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer.MustStop()
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer.MustStop()
+	}
+	remotewrite.Stop()
+
+	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	switch path {
+	case "/api/v1/write":
+		prometheusWriteRequests.Inc()
+		if err := promremotewrite.InsertHandler(r); err != nil {
+			prometheusWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/targets":
+		w.Header().Set("Content-Type", "text/plain")
+		promscrape.WriteHumanReadableTargetsStatus(w)
+		return true
+	case "/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/write", "/api/v2/write":
+		influxWriteRequests.Inc()
+		if err := influx.InsertHandler(r); err != nil {
+			influxWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/query":
+		// Emulate fake response for influx query.
+		// This is required for TSBS benchmark.
+		influxQueryRequests.Inc()
+		fmt.Fprintf(w, `{"results":[{"series":[{"values":[]}]}]}`)
+		return true
+	}
+	return false
+}
+
+var (
+	prometheusWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/write", protocol="prometheus"}`)
+	prometheusWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/write", protocol="prometheus"}`)
+
+	vmimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import", protocol="vm"}`)
+	vmimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import", protocol="vm"}`)
+
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
+
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
+)

app/vmagent/opentsdb/request_handler.go

@@ -0,0 +1,65 @@
+package opentsdb
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdb"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdb"}`)
+)
+
+// InsertHandler processes remote write for OpenTSDB put protocol.
+//
+// See http://opentsdb.net/docs/build/html/api_telnet/put.html
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/opentsdbhttp/request_handler.go

@@ -0,0 +1,64 @@
+package opentsdbhttp
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdbhttp"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdbhttp"}`)
+)
+
+// InsertHandler processes HTTP OpenTSDB put requests.
+// See http://opentsdb.net/docs/build/html/api_http/put.html
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/promremotewrite/request_handler.go

@@ -0,0 +1,67 @@
+package promremotewrite
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+)
+
+// InsertHandler processes remote write for prometheus.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(timeseries []prompb.TimeSeries) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range timeseries {
+		ts := &timeseries[i]
+		labelsLen := len(labels)
+		for i := range ts.Labels {
+			label := &ts.Labels[i]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(label.Name),
+				Value: bytesutil.ToUnsafeString(label.Value),
+			})
+		}
+		samplesLen := len(samples)
+		for i := range ts.Samples {
+			sample := &ts.Samples[i]
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     sample.Value,
+				Timestamp: sample.Timestamp,
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+		rowsTotal += len(ts.Samples)
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/remotewrite/client.go

@@ -0,0 +1,267 @@
+package remotewrite
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fasthttp"
+)
+
+var (
+	sendTimeout = flag.Duration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to -remoteWrite.url")
+
+	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
+	tlsCertFile           = flag.String("remoteWrite.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url")
+	tlsKeyFile            = flag.String("remoteWrite.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url")
+	tlsCAFile             = flag.String("remoteWrite.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
+		"By default system CA is used")
+
+	basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username to use for -remoteWrite.url")
+	basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password to use for -remoteWrite.url")
+	bearerToken       = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url")
+)
+
+type client struct {
+	urlLabelValue  string
+	remoteWriteURL string
+	host           string
+	requestURI     string
+	authHeader     string
+	fq             *persistentqueue.FastQueue
+	hc             *fasthttp.HostClient
+
+	requestDuration *metrics.Histogram
+	requestsOKCount *metrics.Counter
+	errorsCount     *metrics.Counter
+	retriesCount    *metrics.Counter
+
+	wg     sync.WaitGroup
+	stopCh chan struct{}
+}
+
+func newClient(remoteWriteURL, urlLabelValue string, fq *persistentqueue.FastQueue) *client {
+	authHeader := ""
+	if len(*basicAuthUsername) > 0 || len(*basicAuthPassword) > 0 {
+		// See https://en.wikipedia.org/wiki/Basic_access_authentication
+		token := *basicAuthUsername + ":" + *basicAuthPassword
+		token64 := base64.StdEncoding.EncodeToString([]byte(token))
+		authHeader = "Basic " + token64
+	}
+	if len(*bearerToken) > 0 {
+		if authHeader != "" {
+			logger.Panicf("FATAL: `-remoteWrite.bearerToken`=%q cannot be set when `-remoteWrite.basicAuth.*` flags are set", *bearerToken)
+		}
+		authHeader = "Bearer " + *bearerToken
+	}
+
+	readTimeout := *sendTimeout
+	if readTimeout <= 0 {
+		readTimeout = time.Minute
+	}
+	var u fasthttp.URI
+	u.Update(remoteWriteURL)
+	scheme := string(u.Scheme())
+	switch scheme {
+	case "http", "https":
+	default:
+		logger.Panicf("FATAL: unsupported scheme in -remoteWrite.url=%q: %q. It must be http or https", remoteWriteURL, scheme)
+	}
+	host := string(u.Host())
+	if len(host) == 0 {
+		logger.Panicf("FATAL: invalid -remoteWrite.url=%q: host cannot be empty. Make sure the url looks like `http://host:port/path`", remoteWriteURL)
+	}
+	requestURI := string(u.RequestURI())
+	isTLS := scheme == "https"
+	var tlsCfg *tls.Config
+	if isTLS {
+		var err error
+		tlsCfg, err = getTLSConfig()
+		if err != nil {
+			logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
+		}
+	}
+	if !strings.Contains(host, ":") {
+		if isTLS {
+			host += ":443"
+		} else {
+			host += ":80"
+		}
+	}
+	maxConns := 2 * *queues
+	hc := &fasthttp.HostClient{
+		Addr:                host,
+		Name:                "vmagent",
+		Dial:                statDial,
+		DialDualStack:       netutil.TCP6Enabled(),
+		IsTLS:               isTLS,
+		TLSConfig:           tlsCfg,
+		MaxConns:            maxConns,
+		MaxIdleConnDuration: 10 * readTimeout,
+		ReadTimeout:         readTimeout,
+		WriteTimeout:        10 * time.Second,
+		MaxResponseBodySize: 1024 * 1024,
+	}
+	c := &client{
+		urlLabelValue:  urlLabelValue,
+		remoteWriteURL: remoteWriteURL,
+		host:           host,
+		requestURI:     requestURI,
+		authHeader:     authHeader,
+		fq:             fq,
+		hc:             hc,
+		stopCh:         make(chan struct{}),
+	}
+	c.requestDuration = metrics.NewHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.urlLabelValue))
+	c.requestsOKCount = metrics.NewCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.urlLabelValue))
+	c.errorsCount = metrics.NewCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.urlLabelValue))
+	c.retriesCount = metrics.NewCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.urlLabelValue))
+	for i := 0; i < *queues; i++ {
+		c.wg.Add(1)
+		go func() {
+			defer c.wg.Done()
+			c.runWorker()
+		}()
+	}
+	logger.Infof("initialized client for -remoteWrite.url=%q", c.remoteWriteURL)
+	return c
+}
+
+func (c *client) MustStop() {
+	close(c.stopCh)
+	c.wg.Wait()
+	logger.Infof("stopped client for -remoteWrite.url=%q", c.remoteWriteURL)
+}
+
+func getTLSConfig() (*tls.Config, error) {
+	var tlsRootCA *x509.CertPool
+	var tlsCertificate *tls.Certificate
+	if *tlsCertFile != "" || *tlsKeyFile != "" {
+		cert, err := tls.LoadX509KeyPair(*tlsCertFile, *tlsKeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load TLS certificate for -remoteWrite.tlsCertFile=%q and -remoteWrite.tlsKeyFile=%q: %s", *tlsCertFile, *tlsKeyFile, err)
+		}
+		tlsCertificate = &cert
+	}
+	if *tlsCAFile != "" {
+		data, err := ioutil.ReadFile(*tlsCAFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read -remoteWrite.tlsCAFile=%q: %s", *tlsCAFile, err)
+		}
+		tlsRootCA = x509.NewCertPool()
+		if !tlsRootCA.AppendCertsFromPEM(data) {
+			return nil, fmt.Errorf("cannot parse data -remoteWrite.tlsCAFile=%q", *tlsCAFile)
+		}
+	}
+	tlsCfg := &tls.Config{
+		RootCAs:            tlsRootCA,
+		ClientSessionCache: tls.NewLRUClientSessionCache(0),
+	}
+	if tlsCertificate != nil {
+		tlsCfg.Certificates = []tls.Certificate{*tlsCertificate}
+	}
+	tlsCfg.InsecureSkipVerify = *tlsInsecureSkipVerify
+	return tlsCfg, nil
+}
+
+func (c *client) runWorker() {
+	var ok bool
+	var block []byte
+	ch := make(chan struct{})
+	for {
+		block, ok = c.fq.MustReadBlock(block[:0])
+		if !ok {
+			return
+		}
+		go func() {
+			c.sendBlock(block)
+			ch <- struct{}{}
+		}()
+		select {
+		case <-ch:
+			// The block has been sent successfully
+			continue
+		case <-c.stopCh:
+			// c must be stopped. Wait for a while in the hope the block will be sent.
+			graceDuration := 5 * time.Second
+			select {
+			case <-ch:
+				// The block has been sent successfully.
+			case <-time.After(graceDuration):
+				logger.Errorf("couldn't sent block with size %d bytes to %q in %.3f seconds during shutdown; dropping it",
+					len(block), c.remoteWriteURL, graceDuration.Seconds())
+			}
+			return
+		}
+	}
+}
+
+func (c *client) sendBlock(block []byte) {
+	req := fasthttp.AcquireRequest()
+	req.SetRequestURI(c.requestURI)
+	req.SetHost(c.host)
+	req.Header.SetMethod("POST")
+	req.Header.Add("Content-Type", "application/x-protobuf")
+	req.Header.Add("Content-Encoding", "snappy")
+	if c.authHeader != "" {
+		req.Header.Set("Authorization", c.authHeader)
+	}
+	req.SetBody(block)
+
+	retryDuration := time.Second
+	resp := fasthttp.AcquireResponse()
+
+again:
+	select {
+	case <-c.stopCh:
+		fasthttp.ReleaseRequest(req)
+		fasthttp.ReleaseResponse(resp)
+		return
+	default:
+	}
+
+	startTime := time.Now()
+	// There is no need in calling DoTimeout, since the timeout is set in c.hc.ReadTimeout.
+	err := c.hc.Do(req, resp)
+	c.requestDuration.UpdateDuration(startTime)
+	if err != nil {
+		c.errorsCount.Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
+		}
+		logger.Errorf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, err, retryDuration.Seconds())
+		time.Sleep(retryDuration)
+		c.retriesCount.Inc()
+		goto again
+	}
+	statusCode := resp.StatusCode()
+	if statusCode/100 != 2 {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.urlLabelValue, statusCode)).Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
+		}
+		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q: %d; response body=%q; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, statusCode, resp.Body(), retryDuration.Seconds())
+		time.Sleep(retryDuration)
+		c.retriesCount.Inc()
+		goto again
+	}
+	c.requestsOKCount.Inc()
+
+	// The block has been successfully sent to the remote storage.
+	fasthttp.ReleaseResponse(resp)
+	fasthttp.ReleaseRequest(req)
+}

app/vmagent/remotewrite/pendingseries.go

@@ -0,0 +1,191 @@
+package remotewrite
+
+import (
+	"flag"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
+)
+
+var flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
+	"Higher value reduces network bandwidth usage at the cost of delayed push of scraped data to remote storage")
+
+// the maximum number of rows to send per each block.
+const maxRowsPerBlock = 10000
+
+type pendingSeries struct {
+	mu sync.Mutex
+	wr writeRequest
+
+	stopCh            chan struct{}
+	periodicFlusherWG sync.WaitGroup
+}
+
+func newPendingSeries(pushBlock func(block []byte)) *pendingSeries {
+	var ps pendingSeries
+	ps.wr.pushBlock = pushBlock
+	ps.stopCh = make(chan struct{})
+	ps.periodicFlusherWG.Add(1)
+	go func() {
+		defer ps.periodicFlusherWG.Done()
+		ps.periodicFlusher()
+	}()
+	return &ps
+}
+
+func (ps *pendingSeries) MustStop() {
+	close(ps.stopCh)
+	ps.periodicFlusherWG.Wait()
+}
+
+func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
+	ps.mu.Lock()
+	ps.wr.push(tss)
+	ps.mu.Unlock()
+}
+
+func (ps *pendingSeries) periodicFlusher() {
+	ticker := time.NewTicker(*flushInterval)
+	defer ticker.Stop()
+	mustStop := false
+	for !mustStop {
+		select {
+		case <-ps.stopCh:
+			mustStop = true
+		case <-ticker.C:
+			if time.Since(ps.wr.lastFlushTime) < *flushInterval/2 {
+				continue
+			}
+		}
+		ps.mu.Lock()
+		ps.wr.flush()
+		ps.mu.Unlock()
+	}
+}
+
+type writeRequest struct {
+	wr            prompbmarshal.WriteRequest
+	pushBlock     func(block []byte)
+	lastFlushTime time.Time
+
+	tss []prompbmarshal.TimeSeries
+
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
+	buf     []byte
+}
+
+func (wr *writeRequest) reset() {
+	wr.wr.Timeseries = nil
+
+	for i := range wr.tss {
+		ts := &wr.tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	wr.tss = wr.tss[:0]
+
+	for i := range wr.labels {
+		label := &wr.labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	wr.labels = wr.labels[:0]
+
+	wr.samples = wr.samples[:0]
+	wr.buf = wr.buf[:0]
+}
+
+func (wr *writeRequest) flush() {
+	wr.wr.Timeseries = wr.tss
+	wr.lastFlushTime = time.Now()
+	pushWriteRequest(&wr.wr, wr.pushBlock)
+	wr.reset()
+}
+
+func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
+	tssDst := wr.tss
+	for i := range src {
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		dst := &tssDst[len(tssDst)-1]
+		wr.copyTimeSeries(dst, &src[i])
+		if len(wr.tss) >= maxRowsPerBlock {
+			wr.flush()
+			tssDst = wr.tss
+		}
+	}
+	wr.tss = tssDst
+}
+
+func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
+	labelsDst := wr.labels
+	labelsLen := len(wr.labels)
+	samplesDst := wr.samples
+	buf := wr.buf
+	for i := range src.Labels {
+		labelsDst = append(labelsDst, prompbmarshal.Label{})
+		dstLabel := &labelsDst[len(labelsDst)-1]
+		srcLabel := &src.Labels[i]
+
+		buf = append(buf, srcLabel.Name...)
+		dstLabel.Name = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Name):])
+		buf = append(buf, srcLabel.Value...)
+		dstLabel.Value = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Value):])
+	}
+	dst.Labels = labelsDst[labelsLen:]
+
+	samplesDst = append(samplesDst, prompbmarshal.Sample{})
+	dstSample := &samplesDst[len(samplesDst)-1]
+	if len(src.Samples) != 1 {
+		logger.Panicf("BUG: unexpected number of samples in time series; got %d; want 1", len(src.Samples))
+	}
+	*dstSample = src.Samples[0]
+	dst.Samples = samplesDst[len(samplesDst)-1:]
+
+	wr.samples = samplesDst
+	wr.labels = labelsDst
+	wr.buf = buf
+}
+
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte)) {
+	if len(wr.Timeseries) == 0 {
+		// Nothing to push
+		return
+	}
+	bb := writeRequestBufPool.Get()
+	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
+	zb := snappyBufPool.Get()
+	zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+	writeRequestBufPool.Put(bb)
+	if len(zb.B) <= persistentqueue.MaxBlockSize {
+		pushBlock(zb.B)
+		blockSizeRows.Update(float64(len(wr.Timeseries)))
+		blockSizeBytes.Update(float64(len(zb.B)))
+		snappyBufPool.Put(zb)
+		return
+	}
+	snappyBufPool.Put(zb)
+
+	// Too big block. Recursively split it into smaller parts.
+	timeseries := wr.Timeseries
+	n := len(timeseries) / 2
+	wr.Timeseries = timeseries[:n]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries[n:]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries
+}
+
+var (
+	blockSizeBytes = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
+	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
+)
+
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/relabel.go

@@ -0,0 +1,108 @@
+package remotewrite
+
+import (
+	"flag"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+var (
+	extraLabelsUnparsed = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
+		"Pass multiple -remoteWrite.label flags in order to add multiple flags to metrics before sending them to remote storage")
+	relabelConfigPath = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
+		"before sending them to -remoteWrite.url. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config for details")
+)
+
+var extraLabels []prompbmarshal.Label
+var prcs []promrelabel.ParsedRelabelConfig
+
+// initRelabel must be called after parsing command-line flags.
+func initRelabel() {
+	// Init extraLabels
+	for _, s := range *extraLabelsUnparsed {
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Panicf("FATAL: missing '=' in `-remoteWrite.label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		extraLabels = append(extraLabels, prompbmarshal.Label{
+			Name:  s[:n],
+			Value: s[n+1:],
+		})
+	}
+
+	// Init prcs
+	if len(*relabelConfigPath) > 0 {
+		var err error
+		prcs, err = promrelabel.LoadRelabelConfigs(*relabelConfigPath)
+		if err != nil {
+			logger.Panicf("FATAL: cannot load relabel configs from -remoteWrite.relabelConfig=%q: %s", *relabelConfigPath, err)
+		}
+	}
+}
+
+func resetRelabel() {
+	extraLabels = nil
+	prcs = nil
+}
+
+func (rctx *relabelCtx) applyRelabeling(wr *prompbmarshal.WriteRequest) {
+	if len(extraLabels) == 0 && len(prcs) == 0 {
+		// Nothing to change.
+		return
+	}
+	tss := wr.Timeseries
+	tssDst := tss[:0]
+	labels := rctx.labels[:0]
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
+		for j := range extraLabels {
+			extraLabel := &extraLabels[j]
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, *extraLabel)
+			}
+		}
+		labels = promrelabel.ApplyRelabelConfigs(labels, labelsLen, prcs, true)
+		if len(labels) == labelsLen {
+			// Drop the current time series, since relabeling removed all the labels.
+			continue
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: ts.Samples,
+		})
+	}
+	rctx.labels = labels
+	wr.Timeseries = tssDst
+}
+
+type relabelCtx struct {
+	// pool for labels, which are used during the relabeling.
+	labels []prompbmarshal.Label
+}
+
+func (rctx *relabelCtx) reset() {
+	labels := rctx.labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	rctx.labels = rctx.labels[:0]
+}
+
+var relabelCtxPool = &sync.Pool{
+	New: func() interface{} {
+		return &relabelCtx{}
+	},
+}

app/vmagent/remotewrite/remotewrite.go

@@ -0,0 +1,127 @@
+package remotewrite
+
+import (
+	"flag"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+	xxhash "github.com/cespare/xxhash/v2"
+)
+
+var (
+	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
+		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
+		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
+	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored")
+	queues      = flag.Int("remoteWrite.queues", 1, "The number of concurrent queues to each -remoteWrite.url. Set more queues if a single queue "+
+		"isn't enough for sending high volume of collected data to remote storage")
+	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensistive auth info")
+)
+
+// Init initializes remotewrite.
+//
+// It must be called after flag.Parse().
+//
+// Stop must be called for graceful shutdown.
+func Init() {
+	if len(*remoteWriteURLs) == 0 {
+		logger.Panicf("FATAL: at least one `-remoteWrite.url` must be set")
+	}
+
+	if !*showRemoteWriteURL {
+		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
+		httpserver.RegisterSecretFlag("remoteWrite.url")
+	}
+	initRelabel()
+
+	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / maxRowsPerBlock / 100
+	if maxInmemoryBlocks > 200 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 200
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
+	for i, remoteWriteURL := range *remoteWriteURLs {
+		h := xxhash.Sum64([]byte(remoteWriteURL))
+		path := fmt.Sprintf("%s/persistent-queue/%016X", *tmpDataPath, h)
+		fq := persistentqueue.MustOpenFastQueue(path, remoteWriteURL, maxInmemoryBlocks)
+		urlLabelValue := fmt.Sprintf("secret-url-%d", i+1)
+		if *showRemoteWriteURL {
+			urlLabelValue = remoteWriteURL
+		}
+		_ = metrics.NewGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{url=%q, hash="%016X"}`, urlLabelValue, h), func() float64 {
+			return float64(fq.GetPendingBytes())
+		})
+		_ = metrics.NewGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{url=%q}`, urlLabelValue), func() float64 {
+			return float64(fq.GetInmemoryQueueLen())
+		})
+		c := newClient(remoteWriteURL, urlLabelValue, fq)
+		fqs = append(fqs, fq)
+		cs = append(cs, c)
+	}
+
+	pss = make([]*pendingSeries, *queues)
+	for i := range pss {
+		pss[i] = newPendingSeries(pushBlockToPersistentQueues)
+	}
+}
+
+// Stop stops remotewrite.
+//
+// It is expected that nobody calls Push during and after the call to this func.
+func Stop() {
+	for _, ps := range pss {
+		ps.MustStop()
+	}
+
+	// Close all the persistent queues. This should unblock clients waiting in MustReadBlock.
+	for _, fq := range fqs {
+		fq.MustClose()
+	}
+	fqs = nil
+
+	// Stop all the clients
+	for _, c := range cs {
+		c.MustStop()
+	}
+	cs = nil
+
+	resetRelabel()
+}
+
+// Push sends wr to remote storage systems set via `-remoteWrite.url`.
+//
+// Each timeseries in wr.Timeseries must contain one sample.
+func Push(wr *prompbmarshal.WriteRequest) {
+	rctx := relabelCtxPool.Get().(*relabelCtx)
+	rctx.applyRelabeling(wr)
+
+	idx := atomic.AddUint64(&pssNextIdx, 1) % uint64(len(pss))
+	pss[idx].Push(wr.Timeseries)
+
+	rctx.reset()
+	relabelCtxPool.Put(rctx)
+}
+
+func pushBlockToPersistentQueues(block []byte) {
+	for _, fq := range fqs {
+		fq.MustWriteBlock(block)
+	}
+}
+
+var fqs []*persistentqueue.FastQueue
+var cs []*client
+
+var pssNextIdx uint64
+var pss []*pendingSeries

app/vmagent/remotewrite/statconn.go

@@ -0,0 +1,71 @@
+package remotewrite
+
+import (
+	"net"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fasthttp"
+)
+
+func statDial(addr string) (net.Conn, error) {
+	conn, err := fasthttp.Dial(addr)
+	dialsTotal.Inc()
+	if err != nil {
+		dialErrors.Inc()
+		return nil, err
+	}
+	conns.Inc()
+	sc := &statConn{
+		Conn: conn,
+	}
+	return sc, nil
+}
+
+var (
+	dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
+	dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
+	conns      = metrics.NewCounter(`vmagent_remotewrite_conns`)
+)
+
+type statConn struct {
+	closed uint64
+	net.Conn
+}
+
+func (sc *statConn) Read(p []byte) (int, error) {
+	n, err := sc.Conn.Read(p)
+	connReadsTotal.Inc()
+	if err != nil {
+		connReadErrors.Inc()
+	}
+	connBytesRead.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Write(p []byte) (int, error) {
+	n, err := sc.Conn.Write(p)
+	connWritesTotal.Inc()
+	if err != nil {
+		connWriteErrors.Inc()
+	}
+	connBytesWritten.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Close() error {
+	err := sc.Conn.Close()
+	if atomic.AddUint64(&sc.closed, 1) == 1 {
+		conns.Dec()
+	}
+	return err
+}
+
+var (
+	connReadsTotal   = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
+	connWritesTotal  = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
+	connReadErrors   = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
+	connWriteErrors  = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
+	connBytesRead    = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
+	connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
+)

app/vmagent/vmimport/request_handler.go

@@ -0,0 +1,70 @@
+package vmimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+		values := r.Values
+		timestamps := r.Timestamps
+		_ = timestamps[len(values)-1]
+		samplesLen := len(samples)
+		for j, value := range values {
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     value,
+				Timestamp: timestamps[j],
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+		rowsTotal += len(values)
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmalert/README.md

@@ -0,0 +1,41 @@
+## VM Alert
+
+#### Abstract
+The application which accepts the alert rules, executes them on given source, sends(fires) an alert to(in) alert management system
+
+### Components
+
+#### Alert Config Reader
+It accepts yaml config as input parameter in Prometheus format, parses it into Go struct.
+
+#### Source Caller
+Create own watchdog for every alert group (goroutines), which executes alert query on given source and issues an alert if source returns non-empty result. 
+Source can be any service which supports PromQL (MetricsQL). 
+
+#### Alert Management System Provider
+Send positive alert to alert management system, provides interface for every concrete implementation.
+Should be ingratiated with Prometheus alertmanager. 
+
+open questions:
+- do we really need alert group or can just run every alert in own goroutine?
+
+#### Web Server
+Expose metrics
+
+open questions:
+- should the tool provide API or UI for managing alerting rules? Where to store config updated via the API or UI?
+- should the tool provide “alerting rules validation mode” for validating and debugging alerting rules? This mode is useful when creating and debugging alerting rules.
+
+#### Requirements:
+- Stateless
+- Avoid external dependencies if possible
+- Reuse existing code from VictoriaMetrics repo
+- Makefile rules for common tasks – see Makefiles for other apps in the app/ dir 
+- Every package should be covered by tests 
+- Dockerfile
+- Graceful shutdown 
+- Helm template
+- Application uses command line flags for configuration
+
+
+<img alt="VM Alert" src="vmalert.png">

app/vmalert/config/parser.go

@@ -0,0 +1,32 @@
+package config
+
+import "time"
+
+// Annotations basic annotation for alert rule
+type Annotations struct {
+	Summary     string
+	Description string
+}
+
+// Alert basic alert entity rule
+type Alert struct {
+	Name        string
+	Expr        string
+	For         time.Duration
+	Labels      map[string]string
+	Annotations Annotations
+
+	Start time.Time
+	End   time.Time
+}
+
+// Group grouping array of alert
+type Group struct {
+	Name  string
+	Rules []Alert
+}
+
+// Parse parses config from given file
+func Parse(filepath string) ([]Group, error) {
+	return []Group{}, nil
+}

app/vmalert/datasource/datasource.go

@@ -0,0 +1,14 @@
+package datasource
+
+import "context"
+
+// Metrics the data returns from storage
+type Metrics struct{}
+
+// VMStorage represents vmstorage entity with ability to read and write metrics
+type VMStorage struct{}
+
+//Query basic query to the datasource
+func (s *VMStorage) Query(ctx context.Context, query string) ([]Metrics, error) {
+	return nil, nil
+}

app/vmalert/main.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+)
+
+var (
+	configPath     = flag.String("config", "config.yaml", "Path to alert configuration file")
+	httpListenAddr = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+)
+
+func main() {
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+
+	logger.Infof("reading alert rules configuration file from %s", *configPath)
+	alertGroups, err := config.Parse(*configPath)
+	if err != nil {
+		logger.Fatalf("Cannot parse configuration file %s", err)
+	}
+	w := &watchdog{storage: &datasource.VMStorage{}}
+	for id := range alertGroups {
+		go func(group config.Group) {
+			w.run(group)
+		}(alertGroups[id])
+	}
+	go func() {
+		httpserver.Serve(*httpListenAddr, func(w http.ResponseWriter, r *http.Request) bool {
+			panic("not implemented")
+		})
+	}()
+	sig := procutil.WaitForSigterm()
+	logger.Infof("service received signal %s", sig)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	w.stop()
+}
+
+type watchdog struct {
+	storage *datasource.VMStorage
+}
+
+func (w *watchdog) run(a config.Group) {
+
+}
+
+func (w *watchdog) stop() {
+	panic("not implemented")
+}

app/vmalert/provider/alert_manager_request.qtpl

@@ -0,0 +1,26 @@
+{% import (
+    "time"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+) %}
+{% stripspace %}
+
+{% func amRequest(alert *config.Alert, generatorURL string) %}
+{
+	"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
+	"generatorURL": {%q= generatorURL %},
+    {% if !alert.End.IsZero() %}
+    "endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
+    {% endif %}
+    "labels": {
+        "alertname":{%q= alert.Name %}
+        {% for k,v := range alert.Labels %}
+            ,{%q= k %}:{%q= v %}
+        {% endfor %}
+    },
+    "annotations": {
+        "summary": {%q= alert.Annotations.Summary %},
+        "description": {%q= alert.Annotations.Description %}
+    }
+}
+{% endfunc %}
+{% endstripspace %}

app/vmalert/provider/alert_manager_request.qtpl.go

@@ -0,0 +1,101 @@
+// Code generated by qtc from "alert_manager_request.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/provider/alert_manager_request.qtpl:1
+package provider
+
+//line app/vmalert/provider/alert_manager_request.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"time"
+)
+
+//line app/vmalert/provider/alert_manager_request.qtpl:7
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/provider/alert_manager_request.qtpl:7
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/provider/alert_manager_request.qtpl:7
+func streamamRequest(qw422016 *qt422016.Writer, alert *config.Alert, generatorURL string) {
+//line app/vmalert/provider/alert_manager_request.qtpl:7
+	qw422016.N().S(`{"startsAt":`)
+//line app/vmalert/provider/alert_manager_request.qtpl:9
+	qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
+//line app/vmalert/provider/alert_manager_request.qtpl:9
+	qw422016.N().S(`,"generatorURL":`)
+//line app/vmalert/provider/alert_manager_request.qtpl:10
+	qw422016.N().Q(generatorURL)
+//line app/vmalert/provider/alert_manager_request.qtpl:10
+	qw422016.N().S(`,`)
+//line app/vmalert/provider/alert_manager_request.qtpl:11
+	if !alert.End.IsZero() {
+//line app/vmalert/provider/alert_manager_request.qtpl:11
+		qw422016.N().S(`"endsAt":`)
+//line app/vmalert/provider/alert_manager_request.qtpl:12
+		qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
+//line app/vmalert/provider/alert_manager_request.qtpl:12
+		qw422016.N().S(`,`)
+//line app/vmalert/provider/alert_manager_request.qtpl:13
+	}
+//line app/vmalert/provider/alert_manager_request.qtpl:13
+	qw422016.N().S(`"labels": {"alertname":`)
+//line app/vmalert/provider/alert_manager_request.qtpl:15
+	qw422016.N().Q(alert.Name)
+//line app/vmalert/provider/alert_manager_request.qtpl:16
+	for k, v := range alert.Labels {
+//line app/vmalert/provider/alert_manager_request.qtpl:16
+		qw422016.N().S(`,`)
+//line app/vmalert/provider/alert_manager_request.qtpl:17
+		qw422016.N().Q(k)
+//line app/vmalert/provider/alert_manager_request.qtpl:17
+		qw422016.N().S(`:`)
+//line app/vmalert/provider/alert_manager_request.qtpl:17
+		qw422016.N().Q(v)
+//line app/vmalert/provider/alert_manager_request.qtpl:18
+	}
+//line app/vmalert/provider/alert_manager_request.qtpl:18
+	qw422016.N().S(`},"annotations": {"summary":`)
+//line app/vmalert/provider/alert_manager_request.qtpl:21
+	qw422016.N().Q(alert.Annotations.Summary)
+//line app/vmalert/provider/alert_manager_request.qtpl:21
+	qw422016.N().S(`,"description":`)
+//line app/vmalert/provider/alert_manager_request.qtpl:22
+	qw422016.N().Q(alert.Annotations.Description)
+//line app/vmalert/provider/alert_manager_request.qtpl:22
+	qw422016.N().S(`}}`)
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+}
+
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+func writeamRequest(qq422016 qtio422016.Writer, alert *config.Alert, generatorURL string) {
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+	streamamRequest(qw422016, alert, generatorURL)
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+}
+
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+func amRequest(alert *config.Alert, generatorURL string) string {
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+	writeamRequest(qb422016, alert, generatorURL)
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+	qs422016 := string(qb422016.B)
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+	return qs422016
+//line app/vmalert/provider/alert_manager_request.qtpl:25
+}

app/vmalert/provider/alertmanager.go

@@ -0,0 +1,66 @@
+package provider
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+const alertsPath = "/api/v2/alerts"
+
+var pool = sync.Pool{New: func() interface{} {
+	return &bytes.Buffer{}
+}}
+
+// AlertManager represents integration provider with Prometheus alert manager
+type AlertManager struct {
+	alertURL string
+	argFunc  AlertURLGenerator
+	client   *http.Client
+}
+
+// AlertURLGenerator returns URL to single alert by given name
+type AlertURLGenerator func(name string) string
+
+// NewAlertManager is a constructor for AlertManager
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, c *http.Client) *AlertManager {
+	return &AlertManager{
+		alertURL: strings.TrimSuffix(alertManagerURL, "/") + alertsPath,
+		argFunc:  fn,
+		client:   c,
+	}
+}
+
+const (
+	jsonArrayOpen  byte = 91 // [
+	jsonArrayClose byte = 93 // ]
+)
+
+// Send an alert or resolve message
+func (am *AlertManager) Send(alert *config.Alert) error {
+	b := pool.Get().(*bytes.Buffer)
+	b.Reset()
+	defer pool.Put(b)
+	b.WriteByte(jsonArrayOpen)
+	writeamRequest(b, alert, am.argFunc(alert.Name))
+	b.WriteByte(jsonArrayClose)
+	resp, err := am.client.Post(am.alertURL, "application/json", b)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		b.Reset()
+		if _, err := io.Copy(b, resp.Body); err != nil {
+			logger.Errorf("unable to copy error response body to buffer %s", err)
+		}
+		return fmt.Errorf("invalid response from alertmanager %s", b)
+	}
+	return nil
+}

app/vmalert/provider/alertmanager_test.go

@@ -0,0 +1,80 @@
+package provider
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+)
+
+func TestAlertManager_Send(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc(alertsPath, func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		switch c {
+		case 0:
+			conn, _, _ := w.(http.Hijacker).Hijack()
+			_ = conn.Close()
+		case 1:
+			w.WriteHeader(500)
+		case 2:
+			var a []struct {
+				Labels       map[string]string `json:"labels"`
+				StartsAt     time.Time         `json:"startsAt"`
+				EndAt        time.Time         `json:"endsAt"`
+				Annotations  map[string]string `json:"annotations"`
+				GeneratorURL string            `json:"generatorURL"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
+				t.Errorf("can not unmarshal data into alert %s", err)
+				t.FailNow()
+			}
+			if len(a) != 1 {
+				t.Errorf("expected 1 alert in array got %d", len(a))
+			}
+			if a[0].GeneratorURL != "alert0" {
+				t.Errorf("exptected alert0 as generatorURL got %s", a[0].GeneratorURL)
+			}
+			if a[0].Labels["alertname"] != "alert0" {
+				t.Errorf("exptected alert0 as alert name got %s", a[0].Labels["alertname"])
+			}
+			if a[0].StartsAt.IsZero() {
+				t.Errorf("exptected non-zero start time")
+			}
+			if a[0].EndAt.IsZero() {
+				t.Errorf("exptected non-zero end time")
+			}
+		}
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+	am := NewAlertManager(srv.URL, func(name string) string {
+		return name
+	}, srv.Client())
+	if err := am.Send(&config.Alert{}); err == nil {
+		t.Error("expected connection error got nil")
+	}
+	if err := am.Send(&config.Alert{}); err == nil {
+		t.Error("expected wrong http code error got nil")
+	}
+	if err := am.Send(&config.Alert{
+		Name:  "alert0",
+		Start: time.Now().UTC(),
+		End:   time.Now().UTC(),
+	}); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if c != 2 {
+		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
+	}
+}

app/vmalert/provider/common.go

@@ -0,0 +1,8 @@
+package provider
+
+import "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+
+// AlertProvider is common interface for alert manager provider
+type AlertProvider interface {
+	Send(rule config.Alert) error
+}

app/vmbackup/Makefile

@@ -0,0 +1,67 @@
+# All these commands must run from repository root.
+
+vmbackup:
+	APP_NAME=vmbackup $(MAKE) app-local
+
+vmbackup-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker
+
+vmbackup-pure-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-pure
+
+vmbackup-amd64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-amd64
+
+vmbackup-arm-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-arm
+
+vmbackup-arm64-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-arm64
+
+vmbackup-ppc64le-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-ppc64le
+
+vmbackup-386-prod:
+	APP_NAME=vmbackup $(MAKE) app-via-docker-386
+
+package-vmbackup:
+	APP_NAME=vmbackup $(MAKE) package-via-docker
+
+package-vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-pure
+
+package-vmbackup-amd64:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-amd64
+
+package-vmbackup-arm:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-arm
+
+package-vmbackup-arm64:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-arm64
+
+package-vmbackup-ppc64le:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-ppc64le
+
+package-vmbackup-386:
+	APP_NAME=vmbackup $(MAKE) package-via-docker-386
+
+publish-vmbackup:
+	APP_NAME=vmbackup $(MAKE) publish-via-docker
+
+vmbackup-pure:
+	APP_NAME=vmbackup $(MAKE) app-local-pure
+
+vmbackup-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-amd64 ./app/vmbackup
+
+vmbackup-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm ./app/vmbackup
+
+vmbackup-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-arm64 ./app/vmbackup
+
+vmbackup-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-ppc64le ./app/vmbackup
+
+vmbackup-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmbackup-386 ./app/vmbackup

app/vmbackup/README.md

@@ -0,0 +1,181 @@
+## vmbackup
+
+`vmbackup` creates VictoriaMetrics data backups from [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+
+Supported storage systems for backups:
+
+* [GCS](https://cloud.google.com/storage/). Example: `gcs://<bucket>/<path/to/backup>`
+* [S3](https://aws.amazon.com/s3/). Example: `s3://<bucket>/<path/to/backup>`
+* Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See `-customS3Endpoint` command-line flag.
+* Local filesystem. Example: `fs://</absolute/path/to/backup>`
+
+Incremental backups and full backups are supported. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+Full backups can be sped up with `-origin` pointing to already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
+data between the existing backup and new backup. This saves time and costs on data transfer.
+
+Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
+
+Backed up data can be restored with [vmrestore](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md).
+
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+
+
+### Use cases
+
+#### Regular backups
+
+Regular backup can be performed with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup>
+```
+
+* `</path/to/victoria-metrics-data>` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`.
+  There is no need to stop VictoriaMetrics for creating backups, since they are performed from immutable [instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<local-snapshot>` is the snapshot to backup. See [how to create instant snapshots](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+* `<bucket>` is already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
+* `<path/to/new/backup>` is the destination path where new backup will be placed.
+
+
+#### Regular backups with server-side copy from existing backup
+
+If the destination GCS bucket already contains the previous backup at `-origin` path, then new backup can be sped up
+with the following command:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/new/backup> -origin=gcs://<bucket>/<path/to/existing/backup>
+```
+
+This saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`.
+
+
+#### Incremental backups
+
+Incremental backups are performed if `-dst` points to already existing backup. In this case only new data is uploaded to remote storage.
+This saves time and network bandwidth costs when working with big backups:
+
+```
+vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshotName=<local-snapshot> -dst=gcs://<bucket>/<path/to/existing/backup>
+```
+
+
+#### Smart backups
+
+Smart backups mean storing full daily backups into `YYYYMMDD` folders and creating incremental hourly backup into `latest` folder:
+
+* Run the following command every hour:
+
+```
+vmbackup -snapshotName=<latest-snapshot> -dst=gcs://<bucket>/latest
+```
+
+Where `<latest-snapshot>` is the latest [snapshot](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots).
+The command will upload only changed data to `gcs://<bucket>/latest`.
+
+* Run the following command once a day:
+
+```
+vmbackup -snapshotName=<daily-snapshot> -dst=gcs://<bucket>/<YYYYMMDD> -origin=gcs://<bucket>/latest
+```
+
+Where `<daily-snapshot>` is the snapshot for the last day `<YYYYMMDD>`.
+
+
+This apporach saves network bandwidth costs on hourly backups (since they are incremental) and allows recovering data from either the last hour (`latest` backup)
+or from any day (`YYYYMMDD` backups). Note that hourly backup shouldn't run when creating daily backup.
+
+Do not forget removing old snapshots and backups when they are no longer needed for saving storage costs.
+
+
+### How does it work?
+
+The backup algorithm is the following:
+
+1. Collect information about files in the `-snapshotName`, in the `-dst` and in the `-origin`.
+2. Determine files in `-dst`, which are missing in `-snapshotName`, and delete them. These are usually small files, which are already merged into bigger files in the snapshot.
+3. Determine files from `-snapshotName`, which are missing in `-dst`. These are usually small new files and bigger merged files.
+4. Determine files from step 3, which exist in the `-origin`, and perform server-side copy of these files from `-origin` to `-dst`.
+   This are usually the biggest and the oldest files, which are shared between backups.
+5. Upload the remaining files from setp 3 from `-snapshotName` to `-dst`.
+
+The algorithm splits source files into 100MB chunks in the backup. Each chunk is stored as a separate file in the backup.
+Such splitting minimizes the amounts of data to re-transfer after temporary errors.
+
+`vmbackup` relies on [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) properties:
+
+- All the files in the snapshot are immutable.
+- Old files are periodically merged into new files.
+- Smaller files have higher probability to be merged.
+- Consecutive snapshots share many identical files.
+
+These properties allow performing fast and cheap incremental backups and server-side copying from `-origin` paths.
+See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+`vmbackup` can work improperly or slowly when these properties are violated.
+
+
+### Troubleshooting
+
+* If the backup is slow, then try setting higher value for `-concurrency` flag. This will increase the number of concurrent workers that upload data to backup storage.
+* If `vmbackup` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
+* If `vmbackup` has been interrupted due to temporary error, then just restart it with the same args. It will resume the backup process.
+
+
+### Advanced usage
+
+Run `vmbackup -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce backup duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs (default "default")
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -dst string
+    	Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+    	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+  -maxBytesPerSecond int
+    	The maximum upload speed. There is no limit if it is set to 0
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+  -origin string
+    	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+  -snapshotName string
+    	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
+  -storageDataPath string
+    	Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make vmbackup` from the root folder of the repository.
+   It builds `vmbackup` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmbackup-prod` from the root folder of the repository.
+   It builds `vmbackup-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.

app/vmbackup/deployment/Dockerfile

@@ -0,0 +1,7 @@
+ARG certs_image
+FROM $certs_image AS certs
+FROM scratch
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG src_binary
+COPY $src_binary ./vmbackup-prod
+ENTRYPOINT ["/vmbackup-prod"]

app/vmbackup/main.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+var (
+	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
+	snapshotName    = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots")
+	dst             = flag.String("dst", "", "Where to put the backup on the remote storage. "+
+		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
+		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
+	origin            = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
+	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration")
+	maxBytesPerSecond = flag.Int("maxBytesPerSecond", 0, "The maximum upload speed. There is no limit if it is set to 0")
+)
+
+func main() {
+	flag.Usage = usage
+	envflag.Parse()
+	buildinfo.Init()
+
+	srcFS, err := newSrcFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	dstFS, err := newDstFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	originFS, err := newOriginFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	a := &actions.Backup{
+		Concurrency: *concurrency,
+		Src:         srcFS,
+		Dst:         dstFS,
+		Origin:      originFS,
+	}
+	if err := a.Run(); err != nil {
+		logger.Fatalf("cannot create backup: %s", err)
+	}
+}
+
+func usage() {
+	const s = `
+vmbackup performs backups for VictoriaMetrics data from instant snapshots to gcs, s3
+or local filesystem. Backed up data can be restored with vmrestore.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}
+
+func newSrcFS() (*fslocal.FS, error) {
+	if len(*snapshotName) == 0 {
+		return nil, fmt.Errorf("`-snapshotName` cannot be empty")
+	}
+	snapshotPath := *storageDataPath + "/snapshots/" + *snapshotName
+
+	// Verify the snapshot exists.
+	f, err := os.Open(snapshotPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot open snapshot at %q: %s", snapshotPath, err)
+	}
+	fi, err := f.Stat()
+	_ = f.Close()
+	if err != nil {
+		return nil, fmt.Errorf("cannot stat %q: %s", snapshotPath, err)
+	}
+	if !fi.IsDir() {
+		return nil, fmt.Errorf("snapshot %q must be a directory", snapshotPath)
+	}
+
+	fs := &fslocal.FS{
+		Dir:               snapshotPath,
+		MaxBytesPerSecond: *maxBytesPerSecond,
+	}
+	if err := fs.Init(); err != nil {
+		return nil, fmt.Errorf("cannot initialize fs: %s", err)
+	}
+	return fs, nil
+}
+
+func newDstFS() (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(*dst)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-dst`=%q: %s", *dst, err)
+	}
+	return fs, nil
+}
+
+func newOriginFS() (common.RemoteFS, error) {
+	if len(*origin) == 0 {
+		return nil, nil
+	}
+	fs, err := actions.NewRemoteFS(*origin)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-origin`=%q: %s", *origin, err)
+	}
+	return fs, nil
+}

app/vminsert/common/insert_ctx.go

@@ -2,9 +2,11 @@ package common
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
@@ -30,11 +32,10 @@ func (ctx *InsertCtx) Reset(rowsLen int) {
 		mr.MetricNameRaw = nil
 	}
 	ctx.mrs = ctx.mrs[:0]
-
 	if n := rowsLen - cap(ctx.mrs); n > 0 {
 		ctx.mrs = append(ctx.mrs[:cap(ctx.mrs)], make([]storage.MetricRow, n)...)
 	}
-	ctx.mrs = ctx.mrs[:rowsLen]
+	ctx.mrs = ctx.mrs[:0]
 	ctx.metricNamesBuf = ctx.metricNamesBuf[:0]
 }
 
@@ -46,7 +47,7 @@ func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label)
 	return metricNameRaw[:len(metricNameRaw):len(metricNameRaw)]
 }
 
-// WriteDataPoint writes (timestamp, value) with the given prefix and lables into ctx buffer.
+// WriteDataPoint writes (timestamp, value) with the given prefix and labels into ctx buffer.
 func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompb.Label, timestamp int64, value float64) {
 	metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels)
 	ctx.addRow(metricNameRaw, timestamp, value)
@@ -77,6 +78,26 @@ func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float6
 	mr.Value = value
 }
 
+// AddLabelBytes adds (name, value) label to ctx.Labels.
+//
+// name and value must exist until ctx.Labels is used.
+func (ctx *InsertCtx) AddLabelBytes(name, value []byte) {
+	labels := ctx.Labels
+	if cap(labels) > len(labels) {
+		labels = labels[:len(labels)+1]
+	} else {
+		labels = append(labels, prompb.Label{})
+	}
+	label := &labels[len(labels)-1]
+
+	// Do not copy name and value contents for performance reasons.
+	// This reduces GC overhead on the number of objects and allocations.
+	label.Name = name
+	label.Value = value
+
+	ctx.Labels = labels
+}
+
 // AddLabel adds (name, value) label to ctx.Labels.
 //
 // name and value must exist until ctx.Labels is used.
@@ -100,7 +121,10 @@ func (ctx *InsertCtx) AddLabel(name, value string) {
 // FlushBufs flushes buffered rows to the underlying storage.
 func (ctx *InsertCtx) FlushBufs() error {
 	if err := vmstorage.AddRows(ctx.mrs); err != nil {
-		return fmt.Errorf("cannot store metrics: %s", err)
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot store metrics: %s", err),
+			StatusCode: http.StatusServiceUnavailable,
+		}
 	}
 	return nil
 }

app/vminsert/common/insert_ctx_pool.go

@@ -0,0 +1,36 @@
+package common
+
+import (
+	"runtime"
+	"sync"
+)
+
+// GetInsertCtx returns InsertCtx from the pool.
+//
+// Call PutInsertCtx for returning it to the pool.
+func GetInsertCtx() *InsertCtx {
+	select {
+	case ctx := <-insertCtxPoolCh:
+		return ctx
+	default:
+		if v := insertCtxPool.Get(); v != nil {
+			return v.(*InsertCtx)
+		}
+		return &InsertCtx{}
+	}
+}
+
+// PutInsertCtx returns ctx to the pool.
+//
+// ctx cannot be used after the call.
+func PutInsertCtx(ctx *InsertCtx) {
+	ctx.Reset(0)
+	select {
+	case insertCtxPoolCh <- ctx:
+	default:
+		insertCtxPool.Put(ctx)
+	}
+}
+
+var insertCtxPool sync.Pool
+var insertCtxPoolCh = make(chan *InsertCtx, runtime.GOMAXPROCS(-1))

app/vminsert/concurrencylimiter/concurrencylimiter.go

@@ -1,34 +0,0 @@
-package concurrencylimiter
-
-import (
-	"fmt"
-	"runtime"
-	"time"
-)
-
-var (
-	// ch is the channel for limiting concurrent inserts.
-	// Put an item into it before performing an insert and remove
-	// the item after the insert is complete.
-	ch = make(chan struct{}, runtime.GOMAXPROCS(-1)*2)
-
-	// waitDuration is the amount of time to wait until at least a single
-	// concurrent insert out of cap(Ch) inserts is complete.
-	waitDuration = time.Second * 30
-)
-
-// Do calls f with the limited concurrency.
-func Do(f func() error) error {
-	// Limit the number of conurrent inserts in order to prevent from excess
-	// memory usage and CPU trashing.
-	t := time.NewTimer(waitDuration)
-	select {
-	case ch <- struct{}{}:
-		t.Stop()
-		err := f()
-		<-ch
-		return err
-	case <-t.C:
-		return fmt.Errorf("the server is overloaded with %d concurrent inserts; either increase the number of CPUs or reduce the load", cap(ch))
-	}
-}

app/vminsert/graphite/request_handler.go

@@ -1,166 +1,44 @@
 package graphite
 
 import (
-	"bytes"
-	"fmt"
 	"io"
-	"net"
-	"runtime"
-	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
-var rowsInserted = metrics.NewCounter(`vm_rows_inserted_total{type="graphite"}`)
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="graphite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="graphite"}`)
+)
 
-// insertHandler processes remote write for graphite plaintext protocol.
+// InsertHandler processes remote write for graphite plaintext protocol.
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
-func insertHandler(r io.Reader) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(r)
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
 	})
 }
 
-func insertHandlerInternal(r io.Reader) error {
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-	for ctx.Read(r) {
-		if err := ctx.InsertRows(); err != nil {
-			return err
-		}
-	}
-	return ctx.Error()
-}
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
 
-func (ctx *pushCtx) InsertRows() error {
-	rows := ctx.Rows.Rows
-	ic := &ctx.Common
-	ic.Reset(len(rows))
+	ctx.Reset(len(rows))
 	for i := range rows {
 		r := &rows[i]
-		ic.Labels = ic.Labels[:0]
-		ic.AddLabel("", r.Metric)
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			ic.AddLabel(tag.Key, tag.Value)
+			ctx.AddLabel(tag.Key, tag.Value)
 		}
-		ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, r.Value)
+		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
 	}
 	rowsInserted.Add(len(rows))
-	return ic.FlushBufs()
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
 }
-
-const maxReadPacketSize = 4 * 1024 * 1024
-
-const flushTimeout = 3 * time.Second
-
-func (ctx *pushCtx) Read(r io.Reader) bool {
-	graphiteReadCalls.Inc()
-	if ctx.err != nil {
-		return false
-	}
-	if c, ok := r.(net.Conn); ok {
-		if err := c.SetReadDeadline(time.Now().Add(flushTimeout)); err != nil {
-			graphiteReadErrors.Inc()
-			ctx.err = fmt.Errorf("cannot set read deadline: %s", err)
-			return false
-		}
-	}
-	lr := io.LimitReader(r, maxReadPacketSize)
-	ctx.reqBuf.Reset()
-	ctx.reqBuf.B = append(ctx.reqBuf.B[:0], ctx.tailBuf...)
-	n, err := io.CopyBuffer(&ctx.reqBuf, lr, ctx.copyBuf[:])
-	if err != nil {
-		if ne, ok := err.(net.Error); ok && ne.Timeout() {
-			// Flush the read data on timeout and try reading again.
-		} else {
-			graphiteReadErrors.Inc()
-			ctx.err = fmt.Errorf("cannot read graphite plaintext protocol data: %s", err)
-			return false
-		}
-	} else if n < maxReadPacketSize {
-		// Mark the end of stream.
-		ctx.err = io.EOF
-	}
-
-	// Parse all the rows until the last newline in ctx.reqBuf.B
-	nn := bytes.LastIndexByte(ctx.reqBuf.B, '\n')
-	ctx.tailBuf = ctx.tailBuf[:0]
-	if nn >= 0 {
-		ctx.tailBuf = append(ctx.tailBuf[:0], ctx.reqBuf.B[nn+1:]...)
-		ctx.reqBuf.B = ctx.reqBuf.B[:nn]
-	}
-	if err = ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf.B)); err != nil {
-		graphiteUnmarshalErrors.Inc()
-		ctx.err = fmt.Errorf("cannot unmarshal graphite plaintext protocol data with size %d: %s", len(ctx.reqBuf.B), err)
-		return false
-	}
-
-	// Convert timestamps from seconds to milliseconds
-	for i := range ctx.Rows.Rows {
-		ctx.Rows.Rows[i].Timestamp *= 1e3
-	}
-	return true
-}
-
-type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf  bytesutil.ByteBuffer
-	tailBuf []byte
-	copyBuf [16 * 1024]byte
-
-	err error
-}
-
-func (ctx *pushCtx) Error() error {
-	if ctx.err == io.EOF {
-		return nil
-	}
-	return ctx.err
-}
-
-func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
-	ctx.Common.Reset(0)
-	ctx.reqBuf.Reset()
-	ctx.tailBuf = ctx.tailBuf[:0]
-
-	ctx.err = nil
-}
-
-var (
-	graphiteReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="graphite"}`)
-	graphiteReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="graphite"}`)
-	graphiteUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="graphite"}`)
-)
-
-func getPushCtx() *pushCtx {
-	select {
-	case ctx := <-pushCtxPoolCh:
-		return ctx
-	default:
-		if v := pushCtxPool.Get(); v != nil {
-			return v.(*pushCtx)
-		}
-		return &pushCtx{}
-	}
-}
-
-func putPushCtx(ctx *pushCtx) {
-	ctx.reset()
-	select {
-	case pushCtxPoolCh <- ctx:
-	default:
-		pushCtxPool.Put(ctx)
-	}
-}
-
-var pushCtxPool sync.Pool
-var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/influx/parser_timing_test.go

@@ -1,20 +0,0 @@
-package influx
-
-import (
-	"fmt"
-	"testing"
-)
-
-func BenchmarkRowsUnmarshal(b *testing.B) {
-	s := `cpu usage_user=1.23,usage_system=4.34,usage_iowait=0.1112 1234556768`
-	b.SetBytes(int64(len(s)))
-	b.ReportAllocs()
-	b.RunParallel(func(pb *testing.PB) {
-		var rows Rows
-		for pb.Next() {
-			if err := rows.Unmarshal(s); err != nil {
-				panic(fmt.Errorf("cannot unmarshal %q: %s", s, err))
-			}
-		}
-	})
-}

app/vminsert/influx/request_handler.go

@@ -1,222 +1,97 @@
 package influx
 
 import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"io"
+	"flag"
 	"net/http"
 	"runtime"
 	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
-var rowsInserted = metrics.NewCounter(`vm_rows_inserted_total{type="influx"}`)
+var (
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="influx"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="influx"}`)
+)
 
 // InsertHandler processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
 func InsertHandler(req *http.Request) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(req)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
 	})
 }
 
-func insertHandlerInternal(req *http.Request) error {
-	influxReadCalls.Inc()
-
-	r := req.Body
-	if req.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := getGzipReader(r)
-		if err != nil {
-			return fmt.Errorf("cannot read gzipped influx line protocol data: %s", err)
-		}
-		defer putGzipReader(zr)
-		r = zr
-	}
-
-	q := req.URL.Query()
-	tsMultiplier := int64(1e6)
-	switch q.Get("precision") {
-	case "ns":
-		tsMultiplier = 1e6
-	case "u":
-		tsMultiplier = 1e3
-	case "ms":
-		tsMultiplier = 1
-	case "s":
-		tsMultiplier = -1e3
-	case "m":
-		tsMultiplier = -1e3 * 60
-	case "h":
-		tsMultiplier = -1e3 * 3600
-	}
-
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
-	db := q.Get("db")
-
+func insertRows(db string, rows []parser.Row) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
-	for ctx.Read(r, tsMultiplier) {
-		if err := ctx.InsertRows(db); err != nil {
-			return err
-		}
-	}
-	return ctx.Error()
-}
 
-func (ctx *pushCtx) InsertRows(db string) error {
-	rows := ctx.Rows.Rows
 	rowsLen := 0
 	for i := range rows {
-		rowsLen += len(rows[i].Tags)
+		rowsLen += len(rows[i].Fields)
 	}
 	ic := &ctx.Common
 	ic.Reset(rowsLen)
+	rowsTotal := 0
 	for i := range rows {
 		r := &rows[i]
 		ic.Labels = ic.Labels[:0]
-		ic.AddLabel("db", db)
+		hasDBLabel := false
 		for j := range r.Tags {
 			tag := &r.Tags[j]
+			if tag.Key == "db" {
+				hasDBLabel = true
+			}
 			ic.AddLabel(tag.Key, tag.Value)
 		}
+		if len(db) > 0 && !hasDBLabel {
+			ic.AddLabel("db", db)
+		}
 		ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
 		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
-		ctx.metricGroupBuf = append(ctx.metricGroupBuf, '.')
+		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
+		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
+		}
 		metricGroupPrefixLen := len(ctx.metricGroupBuf)
 		for j := range r.Fields {
 			f := &r.Fields[j]
-			ctx.metricGroupBuf = append(ctx.metricGroupBuf[:metricGroupPrefixLen], f.Key...)
+			if !skipFieldKey {
+				ctx.metricGroupBuf = append(ctx.metricGroupBuf[:metricGroupPrefixLen], f.Key...)
+			}
 			metricGroup := bytesutil.ToUnsafeString(ctx.metricGroupBuf)
 			ic.Labels = ic.Labels[:0]
 			ic.AddLabel("", metricGroup)
 			ic.WriteDataPoint(ctx.metricNameBuf, ic.Labels[:1], r.Timestamp, f.Value)
 		}
-		rowsInserted.Add(len(r.Fields))
+		rowsTotal += len(r.Fields)
 	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
 	return ic.FlushBufs()
 }
 
-func getGzipReader(r io.Reader) (*gzip.Reader, error) {
-	v := gzipReaderPool.Get()
-	if v == nil {
-		return gzip.NewReader(r)
-	}
-	zr := v.(*gzip.Reader)
-	if err := zr.Reset(r); err != nil {
-		return nil, err
-	}
-	return zr, nil
-}
-
-func putGzipReader(zr *gzip.Reader) {
-	_ = zr.Close()
-	gzipReaderPool.Put(zr)
-}
-
-var gzipReaderPool sync.Pool
-
-const maxReadPacketSize = 4 * 1024 * 1024
-
-func (ctx *pushCtx) Read(r io.Reader, tsMultiplier int64) bool {
-	if ctx.err != nil {
-		return false
-	}
-	lr := io.LimitReader(r, maxReadPacketSize)
-	ctx.reqBuf.Reset()
-	ctx.reqBuf.B = append(ctx.reqBuf.B[:0], ctx.tailBuf...)
-	n, err := io.CopyBuffer(&ctx.reqBuf, lr, ctx.copyBuf[:])
-	if err != nil {
-		influxReadErrors.Inc()
-		ctx.err = fmt.Errorf("cannot read influx line protocol data: %s", err)
-		return false
-	}
-	if n < maxReadPacketSize {
-		// Mark the end of stream.
-		ctx.err = io.EOF
-	}
-
-	// Parse all the rows until the last newline in ctx.reqBuf.B
-	nn := bytes.LastIndexByte(ctx.reqBuf.B, '\n')
-	ctx.tailBuf = ctx.tailBuf[:0]
-	if nn >= 0 {
-		ctx.tailBuf = append(ctx.tailBuf[:0], ctx.reqBuf.B[nn+1:]...)
-		ctx.reqBuf.B = ctx.reqBuf.B[:nn]
-	}
-	if err = ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf.B)); err != nil {
-		influxUnmarshalErrors.Inc()
-		ctx.err = fmt.Errorf("cannot unmarshal influx line protocol data with size %d: %s", len(ctx.reqBuf.B), err)
-		return false
-	}
-
-	// Adjust timestamps according to tsMultiplier
-	currentTs := time.Now().UnixNano() / 1e6
-	if tsMultiplier >= 1 {
-		for i := range ctx.Rows.Rows {
-			row := &ctx.Rows.Rows[i]
-			if row.Timestamp == 0 {
-				row.Timestamp = currentTs
-			} else {
-				row.Timestamp /= tsMultiplier
-			}
-		}
-	} else if tsMultiplier < 0 {
-		tsMultiplier = -tsMultiplier
-		for i := range ctx.Rows.Rows {
-			row := &ctx.Rows.Rows[i]
-			if row.Timestamp == 0 {
-				row.Timestamp = currentTs
-			} else {
-				row.Timestamp *= tsMultiplier
-			}
-		}
-	}
-	return true
-}
-
-var (
-	influxReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="influx"}`)
-	influxReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="influx"}`)
-	influxUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="influx"}`)
-)
-
 type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf         bytesutil.ByteBuffer
-	tailBuf        []byte
-	copyBuf        [16 * 1024]byte
+	Common         common.InsertCtx
 	metricNameBuf  []byte
 	metricGroupBuf []byte
-
-	err error
-}
-
-func (ctx *pushCtx) Error() error {
-	if ctx.err == io.EOF {
-		return nil
-	}
-	return ctx.err
 }
 
 func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
 	ctx.Common.Reset(0)
-
-	ctx.reqBuf.Reset()
-	ctx.tailBuf = ctx.tailBuf[:0]
 	ctx.metricNameBuf = ctx.metricNameBuf[:0]
 	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
-
-	ctx.err = nil
 }
 
 func getPushCtx() *pushCtx {

app/vminsert/main.go

@@ -9,34 +9,63 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prompush"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
+	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
+	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	graphiteListenAddr   = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
-	opentsdbListenAddr   = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB put messages. Usually :4242 must be set. Doesn't work if empty")
-	maxInsertRequestSize = flag.Int("maxInsertRequestSize", 32*1024*1024, "The maximum size of a single insert request in bytes")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superflouos labels are dropped")
+)
+
+var (
+	graphiteServer     *graphiteserver.Server
+	opentsdbServer     *opentsdbserver.Server
+	opentsdbhttpServer *opentsdbhttpserver.Server
 )
 
 // Init initializes vminsert.
 func Init() {
+	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
+
+	writeconcurrencylimiter.Init()
 	if len(*graphiteListenAddr) > 0 {
-		go graphite.Serve(*graphiteListenAddr)
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		go opentsdb.Serve(*opentsdbListenAddr)
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
 	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
+	}
+	promscrape.Init(prompush.Push)
 }
 
 // Stop stops vminsert.
 func Stop() {
+	promscrape.Stop()
 	if len(*graphiteListenAddr) > 0 {
-		graphite.Stop()
+		graphiteServer.MustStop()
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		opentsdb.Stop()
+		opentsdbServer.MustStop()
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer.MustStop()
 	}
 }
 
@@ -46,13 +75,22 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch path {
 	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
-		if err := prometheus.InsertHandler(r, int64(*maxInsertRequestSize)); err != nil {
+		if err := promremotewrite.InsertHandler(r); err != nil {
 			prometheusWriteErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandler(r); err != nil {
@@ -63,7 +101,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusNoContent)
 		return true
 	case "/query":
-		// Emulate fake response for influx query
+		// Emulate fake response for influx query.
+		// This is required for TSBS benchmark.
 		influxQueryRequests.Inc()
 		fmt.Fprintf(w, `{"results":[{"series":[{"values":[]}]}]}`)
 		return true
@@ -77,6 +116,9 @@ var (
 	prometheusWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/write", protocol="prometheus"}`)
 	prometheusWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/write", protocol="prometheus"}`)
 
+	vmimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import", protocol="vm"}`)
+	vmimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import", protocol="vm"}`)
+
 	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/write", protocol="influx"}`)
 

app/vminsert/opentsdb/parser_timing_test.go

@@ -1,24 +0,0 @@
-package opentsdb
-
-import (
-	"fmt"
-	"testing"
-)
-
-func BenchmarkRowsUnmarshal(b *testing.B) {
-	s := `cpu.usage_user 1234556768 1.23 a=b
-cpu.usage_system 1234556768 23.344 a=b
-cpu.usage_iowait 1234556769 3.3443 a=b
-cpu.usage_irq 1234556768 0.34432 a=b
-`
-	b.SetBytes(int64(len(s)))
-	b.ReportAllocs()
-	b.RunParallel(func(pb *testing.PB) {
-		var rows Rows
-		for pb.Next() {
-			if err := rows.Unmarshal(s); err != nil {
-				panic(fmt.Errorf("cannot unmarshal %q: %s", s, err))
-			}
-		}
-	})
-}

app/vminsert/opentsdb/request_handler.go

@@ -1,166 +1,44 @@
 package opentsdb
 
 import (
-	"bytes"
-	"fmt"
 	"io"
-	"net"
-	"runtime"
-	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
-var rowsInserted = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdb"}`)
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdb"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="opentsdb"}`)
+)
 
-// insertHandler processes remote write for OpenTSDB put protocol.
+// InsertHandler processes remote write for OpenTSDB put protocol.
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
-func insertHandler(r io.Reader) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(r)
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
 	})
 }
 
-func insertHandlerInternal(r io.Reader) error {
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-	for ctx.Read(r) {
-		if err := ctx.InsertRows(); err != nil {
-			return err
-		}
-	}
-	return ctx.Error()
-}
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
 
-func (ctx *pushCtx) InsertRows() error {
-	rows := ctx.Rows.Rows
-	ic := &ctx.Common
-	ic.Reset(len(rows))
+	ctx.Reset(len(rows))
 	for i := range rows {
 		r := &rows[i]
-		ic.Labels = ic.Labels[:0]
-		ic.AddLabel("", r.Metric)
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			ic.AddLabel(tag.Key, tag.Value)
+			ctx.AddLabel(tag.Key, tag.Value)
 		}
-		ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, r.Value)
+		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
 	}
 	rowsInserted.Add(len(rows))
-	return ic.FlushBufs()
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
 }
-
-const maxReadPacketSize = 4 * 1024 * 1024
-
-const flushTimeout = 3 * time.Second
-
-func (ctx *pushCtx) Read(r io.Reader) bool {
-	opentsdbReadCalls.Inc()
-	if ctx.err != nil {
-		return false
-	}
-	if c, ok := r.(net.Conn); ok {
-		if err := c.SetReadDeadline(time.Now().Add(flushTimeout)); err != nil {
-			opentsdbReadErrors.Inc()
-			ctx.err = fmt.Errorf("cannot set read deadline: %s", err)
-			return false
-		}
-	}
-	lr := io.LimitReader(r, maxReadPacketSize)
-	ctx.reqBuf.Reset()
-	ctx.reqBuf.B = append(ctx.reqBuf.B[:0], ctx.tailBuf...)
-	n, err := io.CopyBuffer(&ctx.reqBuf, lr, ctx.copyBuf[:])
-	if err != nil {
-		if ne, ok := err.(net.Error); ok && ne.Timeout() {
-			// Flush the read data on timeout and try reading again.
-		} else {
-			opentsdbReadErrors.Inc()
-			ctx.err = fmt.Errorf("cannot read OpenTSDB put protocol data: %s", err)
-			return false
-		}
-	} else if n < maxReadPacketSize {
-		// Mark the end of stream.
-		ctx.err = io.EOF
-	}
-
-	// Parse all the rows until the last newline in ctx.reqBuf.B
-	nn := bytes.LastIndexByte(ctx.reqBuf.B, '\n')
-	ctx.tailBuf = ctx.tailBuf[:0]
-	if nn >= 0 {
-		ctx.tailBuf = append(ctx.tailBuf[:0], ctx.reqBuf.B[nn+1:]...)
-		ctx.reqBuf.B = ctx.reqBuf.B[:nn]
-	}
-	if err = ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf.B)); err != nil {
-		opentsdbUnmarshalErrors.Inc()
-		ctx.err = fmt.Errorf("cannot unmarshal OpenTSDB put protocol data with size %d: %s", len(ctx.reqBuf.B), err)
-		return false
-	}
-
-	// Convert timestamps from seconds to milliseconds
-	for i := range ctx.Rows.Rows {
-		ctx.Rows.Rows[i].Timestamp *= 1e3
-	}
-	return true
-}
-
-type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf  bytesutil.ByteBuffer
-	tailBuf []byte
-	copyBuf [16 * 1024]byte
-
-	err error
-}
-
-func (ctx *pushCtx) Error() error {
-	if ctx.err == io.EOF {
-		return nil
-	}
-	return ctx.err
-}
-
-func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
-	ctx.Common.Reset(0)
-	ctx.reqBuf.Reset()
-	ctx.tailBuf = ctx.tailBuf[:0]
-
-	ctx.err = nil
-}
-
-var (
-	opentsdbReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="opentsdb"}`)
-	opentsdbReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="opentsdb"}`)
-	opentsdbUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="opentsdb"}`)
-)
-
-func getPushCtx() *pushCtx {
-	select {
-	case ctx := <-pushCtxPoolCh:
-		return ctx
-	default:
-		if v := pushCtxPool.Get(); v != nil {
-			return v.(*pushCtx)
-		}
-		return &pushCtx{}
-	}
-}
-
-func putPushCtx(ctx *pushCtx) {
-	ctx.reset()
-	select {
-	case pushCtxPoolCh <- ctx:
-	default:
-		pushCtxPool.Put(ctx)
-	}
-}
-
-var pushCtxPool sync.Pool
-var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/opentsdbhttp/request_handler.go

@@ -0,0 +1,50 @@
+package opentsdbhttp
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdbhttp"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="opentsdbhttp"}`)
+)
+
+// InsertHandler processes HTTP OpenTSDB put requests.
+// See http://opentsdb.net/docs/build/html/api_http/put.html
+func InsertHandler(req *http.Request) error {
+	path := req.URL.Path
+	switch path {
+	case "/api/put":
+		return writeconcurrencylimiter.Do(func() error {
+			return parser.ParseStream(req, insertRows)
+		})
+	default:
+		return fmt.Errorf("unexpected path requested on HTTP OpenTSDB server: %q", path)
+	}
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	ctx.Reset(len(rows))
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			ctx.AddLabel(tag.Key, tag.Value)
+		}
+		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
+	}
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
+}

app/vminsert/prometheus/request_handler.go

@@ -1,106 +0,0 @@
-package prometheus
-
-import (
-	"fmt"
-	"net/http"
-	"runtime"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var rowsInserted = metrics.NewCounter(`vm_rows_inserted_total{type="prometheus"}`)
-
-// InsertHandler processes remote write for prometheus.
-func InsertHandler(r *http.Request, maxSize int64) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(r, maxSize)
-	})
-}
-
-func insertHandlerInternal(r *http.Request, maxSize int64) error {
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-	if err := ctx.Read(r, maxSize); err != nil {
-		return err
-	}
-	timeseries := ctx.req.Timeseries
-	rowsLen := 0
-	for i := range timeseries {
-		rowsLen += len(timeseries[i].Samples)
-	}
-	ic := &ctx.Common
-	ic.Reset(rowsLen)
-	for i := range timeseries {
-		ts := &timeseries[i]
-		var metricNameRaw []byte
-		for i := range ts.Samples {
-			r := &ts.Samples[i]
-			metricNameRaw = ic.WriteDataPointExt(metricNameRaw, ts.Labels, r.Timestamp, r.Value)
-		}
-		rowsInserted.Add(len(ts.Samples))
-	}
-	return ic.FlushBufs()
-}
-
-type pushCtx struct {
-	Common common.InsertCtx
-
-	req    prompb.WriteRequest
-	reqBuf []byte
-}
-
-func (ctx *pushCtx) reset() {
-	ctx.Common.Reset(0)
-	ctx.req.Reset()
-	ctx.reqBuf = ctx.reqBuf[:0]
-}
-
-func (ctx *pushCtx) Read(r *http.Request, maxSize int64) error {
-	prometheusReadCalls.Inc()
-
-	var err error
-	ctx.reqBuf, err = prompb.ReadSnappy(ctx.reqBuf[:0], r.Body, maxSize)
-	if err != nil {
-		prometheusReadErrors.Inc()
-		return fmt.Errorf("cannot read prompb.WriteRequest: %s", err)
-	}
-	if err = ctx.req.Unmarshal(ctx.reqBuf); err != nil {
-		prometheusUnmarshalErrors.Inc()
-		return fmt.Errorf("cannot unmarshal prompb.WriteRequest with size %d bytes: %s", len(ctx.reqBuf), err)
-	}
-	return nil
-}
-
-var (
-	prometheusReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="prometheus"}`)
-	prometheusReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="prometheus"}`)
-	prometheusUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="prometheus"}`)
-)
-
-func getPushCtx() *pushCtx {
-	select {
-	case ctx := <-pushCtxPoolCh:
-		return ctx
-	default:
-		if v := pushCtxPool.Get(); v != nil {
-			return v.(*pushCtx)
-		}
-		return &pushCtx{}
-	}
-}
-
-func putPushCtx(ctx *pushCtx) {
-	ctx.reset()
-	select {
-	case pushCtxPoolCh <- ctx:
-	default:
-		pushCtxPool.Put(ctx)
-	}
-}
-
-var pushCtxPool sync.Pool
-var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/prompush/push.go

@@ -0,0 +1,97 @@
+package prompush
+
+import (
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="promscrape"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promscrape"}`)
+)
+
+// Push pushes wr to to storage.
+func Push(wr *prompbmarshal.WriteRequest) {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	timeseries := wr.Timeseries
+	rowsLen := 0
+	for i := range timeseries {
+		rowsLen += len(timeseries[i].Samples)
+	}
+	ic := &ctx.Common
+	ic.Reset(rowsLen)
+	rowsTotal := 0
+	labels := ctx.labels[:0]
+	for i := range timeseries {
+		ts := &timeseries[i]
+		labels = labels[:0]
+		for j := range ts.Labels {
+			label := &ts.Labels[j]
+			labels = append(labels, prompb.Label{
+				Name:  bytesutil.ToUnsafeBytes(label.Name),
+				Value: bytesutil.ToUnsafeBytes(label.Value),
+			})
+		}
+		var metricNameRaw []byte
+		for i := range ts.Samples {
+			r := &ts.Samples[i]
+			metricNameRaw = ic.WriteDataPointExt(metricNameRaw, labels, r.Timestamp, r.Value)
+		}
+		rowsTotal += len(ts.Samples)
+	}
+	ctx.labels = labels
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	if err := ic.FlushBufs(); err != nil {
+		logger.Errorf("cannot flush promscrape data to storage: %s", err)
+	}
+}
+
+type pushCtx struct {
+	Common common.InsertCtx
+	labels []prompb.Label
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.Common.Reset(0)
+
+	for i := range ctx.labels {
+		label := &ctx.labels[i]
+		label.Name = nil
+		label.Value = nil
+	}
+	ctx.labels = ctx.labels[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/promremotewrite/request_handler.go

@@ -0,0 +1,47 @@
+package promremotewrite
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promremotewrite"}`)
+)
+
+// InsertHandler processes remote write for prometheus.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(timeseries []prompb.TimeSeries) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	rowsLen := 0
+	for i := range timeseries {
+		rowsLen += len(timeseries[i].Samples)
+	}
+	ctx.Reset(rowsLen)
+	rowsTotal := 0
+	for i := range timeseries {
+		ts := &timeseries[i]
+		var metricNameRaw []byte
+		for i := range ts.Samples {
+			r := &ts.Samples[i]
+			metricNameRaw = ctx.WriteDataPointExt(metricNameRaw, ts.Labels, r.Timestamp, r.Value)
+		}
+		rowsTotal += len(ts.Samples)
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vminsert/vmimport/request_handler.go

@@ -0,0 +1,94 @@
+package vmimport
+
+import (
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="vmimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="vmimport"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	rowsLen := 0
+	for i := range rows {
+		rowsLen += len(rows[i].Values)
+	}
+	ic := &ctx.Common
+	ic.Reset(rowsLen)
+	rowsTotal := 0
+	for i := range rows {
+		r := &rows[i]
+		ic.Labels = ic.Labels[:0]
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			ic.AddLabelBytes(tag.Key, tag.Value)
+		}
+		ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
+		values := r.Values
+		timestamps := r.Timestamps
+		_ = timestamps[len(values)-1]
+		for j, value := range values {
+			timestamp := timestamps[j]
+			ic.WriteDataPoint(ctx.metricNameBuf, nil, timestamp, value)
+		}
+		rowsTotal += len(values)
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ic.FlushBufs()
+}
+
+type pushCtx struct {
+	Common        common.InsertCtx
+	metricNameBuf []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.Common.Reset(0)
+	ctx.metricNameBuf = ctx.metricNameBuf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vmrestore/Makefile

@@ -0,0 +1,67 @@
+# All these commands must run from repository root.
+
+vmrestore:
+	APP_NAME=vmrestore $(MAKE) app-local
+
+vmrestore-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker
+
+vmrestore-pure-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-pure
+
+vmrestore-amd64-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-amd64
+
+vmrestore-arm-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-arm
+
+vmrestore-arm64-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-arm64
+
+vmrestore-ppc64le-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-ppc64le
+
+vmrestore-386-prod:
+	APP_NAME=vmrestore $(MAKE) app-via-docker-386
+
+package-vmrestore:
+	APP_NAME=vmrestore $(MAKE) package-via-docker
+
+package-vmrestore-pure:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-pure
+
+package-vmrestore-amd64:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-amd64
+
+package-vmrestore-arm:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-arm
+
+package-vmrestore-arm64:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-arm64
+
+package-vmrestore-ppc64le:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-ppc64le
+
+package-vmrestore-386:
+	APP_NAME=vmrestore $(MAKE) package-via-docker-386
+
+publish-vmrestore:
+	APP_NAME=vmrestore $(MAKE) publish-via-docker
+
+vmrestore-pure:
+	APP_NAME=vmrestore $(MAKE) app-local-pure
+
+vmrestore-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-amd64 ./app/vmrestore
+
+vmrestore-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm ./app/vmrestore
+
+vmrestore-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-arm64 ./app/vmrestore
+
+vmrestore-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-ppc64le ./app/vmrestore
+
+vmrestore-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmrestore-386 ./app/vmrestore

app/vmrestore/README.md

@@ -0,0 +1,86 @@
+## vmrestore
+
+`vmrestore` restores data from backups created by [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md).
+VictoriaMetrics `v1.29.0` and newer versions must be used for working with the restored data.
+
+Restore process can be interrupted at any time. It is automatically resumed from the inerruption point
+when restarting `vmrestore` with the same args.
+
+
+### Usage
+
+VictoriaMetrics must be stopped during the restore process.
+
+```
+vmrestore -src=gcs://<bucket>/<path/to/backup> -storageDataPath=<local/path/to/restore>
+
+```
+
+* `<bucket>` is [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets) name.
+* `<path/to/backup>` is the path to backup made with [vmbackup](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmbackup/README.md) on GCS bucket.
+* `<local/path/to/restore>` is the path to folder where data will be restored. This folder must be passed
+  to VictoriaMetrics in `-storageDataPath` command-line flag after the restore process is complete.
+
+The original `-storageDataPath` directory may contain old files. They will be susbstituted by the files from backup.
+
+
+### Troubleshooting
+
+* If `vmrestore` eats all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
+* If `vmrestore` has been interrupted due to temporary error, then just restart it with the same args. It will resume the restore process.
+
+
+### Advanced usage
+
+Run `vmrestore -help` in order to see all the available options:
+
+```
+  -concurrency int
+    	The number of concurrent workers. Higher concurrency may reduce restore duration (default 10)
+  -configFilePath string
+    	Path to file with S3 configs. Configs are loaded from default location if not set.
+    	See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -configProfile string
+    	Profile name for S3 configs (default "default")
+  -credsFilePath string
+    	Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set.
+    	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
+  -customS3Endpoint string
+    	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -loggerLevel string
+    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+  -maxBytesPerSecond int
+    	The maximum download speed. There is no limit if it is set to 0
+  -memory.allowedPercent float
+    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+  -src string
+    	Source path with backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+  -storageDataPath string
+    	Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup (default "victoria-metrics-data")
+  -version
+    	Show VictoriaMetrics version
+```
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - see `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+2. Run `make vmrestore` from the root folder of the repository.
+   It builds `vmrestore` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmrestore-prod` from the root folder of the repository.
+   It builds `vmrestore-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.

app/vmrestore/deployment/Dockerfile

@@ -0,0 +1,7 @@
+ARG certs_image
+FROM $certs_image AS certs
+FROM scratch
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG src_binary
+COPY $src_binary ./vmrestore-prod
+ENTRYPOINT ["/vmrestore-prod"]

app/vmrestore/main.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+var (
+	src = flag.String("src", "", "Source path with backup on the remote storage. "+
+		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir")
+	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Destination path where backup must be restored. "+
+		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup")
+	concurrency             = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce restore duration")
+	maxBytesPerSecond       = flag.Int("maxBytesPerSecond", 0, "The maximum download speed. There is no limit if it is set to 0")
+	skipBackupCompleteCheck = flag.Bool("skipBackupCompleteCheck", false, "Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file")
+)
+
+func main() {
+	flag.Usage = usage
+	envflag.Parse()
+	buildinfo.Init()
+
+	srcFS, err := newSrcFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	dstFS, err := newDstFS()
+	if err != nil {
+		logger.Fatalf("%s", err)
+	}
+	a := &actions.Restore{
+		Concurrency:             *concurrency,
+		Src:                     srcFS,
+		Dst:                     dstFS,
+		SkipBackupCompleteCheck: *skipBackupCompleteCheck,
+	}
+	if err := a.Run(); err != nil {
+		logger.Fatalf("cannot restore from backup: %s", err)
+	}
+}
+
+func usage() {
+	const s = `
+vmrestore restores VictoriaMetrics data from backups made by vmbackup.
+
+See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmrestore/README.md .
+`
+
+	f := flag.CommandLine.Output()
+	fmt.Fprintf(f, "%s\n", s)
+	flag.PrintDefaults()
+}
+
+func newDstFS() (*fslocal.FS, error) {
+	if len(*storageDataPath) == 0 {
+		return nil, fmt.Errorf("`-storageDataPath` cannot be empty")
+	}
+	fs := &fslocal.FS{
+		Dir:               *storageDataPath,
+		MaxBytesPerSecond: *maxBytesPerSecond,
+	}
+	if err := fs.Init(); err != nil {
+		return nil, fmt.Errorf("cannot initialize local fs: %s", err)
+	}
+	return fs, nil
+}
+
+func newSrcFS() (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(*src)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-src`=%q: %s", *src, err)
+	}
+	return fs, nil
+}

app/vmselect/main.go

@@ -2,6 +2,7 @@ package vmselect
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"runtime"
 	"strings"
@@ -14,53 +15,107 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	deleteAuthKey         = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series")
-	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", runtime.GOMAXPROCS(-1)*2, "The maximum number of concurrent search requests. It shouldn't exceed 2*vCPUs for better performance. See also -search.maxQueueDuration")
-	maxQueueDuration      = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached")
+	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
+		"It shouldn't be high, since a single request can saturate all the CPU cores. See also -search.maxQueueDuration")
+	maxQueueDuration  = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached")
+	resetCacheAuthKey = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetCache call")
 )
 
+func getDefaultMaxConcurrentRequests() int {
+	n := runtime.GOMAXPROCS(-1)
+	if n <= 4 {
+		n *= 2
+	}
+	if n > 16 {
+		// A single request can saturate all the CPU cores, so there is no sense
+		// in allowing higher number of concurrent requests - they will just contend
+		// for unavailable CPU time.
+		n = 16
+	}
+	return n
+}
+
 // Init initializes vmselect
 func Init() {
 	tmpDirPath := *vmstorage.DataPath + "/tmp"
 	fs.RemoveDirContents(tmpDirPath)
 	netstorage.InitTmpBlocksDir(tmpDirPath)
 	promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult")
+
 	concurrencyCh = make(chan struct{}, *maxConcurrentRequests)
 }
 
-var concurrencyCh chan struct{}
-
 // Stop stops vmselect
 func Stop() {
 	promql.StopRollupResultCache()
 }
 
+var concurrencyCh chan struct{}
+
+var (
+	concurrencyLimitReached = metrics.NewCounter(`vm_concurrent_select_limit_reached_total`)
+	concurrencyLimitTimeout = metrics.NewCounter(`vm_concurrent_select_limit_timeout_total`)
+
+	_ = metrics.NewGauge(`vm_concurrent_select_capacity`, func() float64 {
+		return float64(cap(concurrencyCh))
+	})
+	_ = metrics.NewGauge(`vm_concurrent_select_current`, func() float64 {
+		return float64(len(concurrencyCh))
+	})
+)
+
 // RequestHandler handles remote read API requests for Prometheus
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+	startTime := time.Now()
 	// Limit the number of concurrent queries.
-	// Sleep for a second until giving up. This should resolve short bursts in requests.
-	t := time.NewTimer(*maxQueueDuration)
 	select {
 	case concurrencyCh <- struct{}{}:
-		t.Stop()
 		defer func() { <-concurrencyCh }()
-	case <-t.C:
-		httpserver.Errorf(w, "cannot handle more than %d concurrent requests", cap(concurrencyCh))
-		return true
+	default:
+		// Sleep for a while until giving up. This should resolve short bursts in requests.
+		concurrencyLimitReached.Inc()
+		t := timerpool.Get(*maxQueueDuration)
+		select {
+		case concurrencyCh <- struct{}{}:
+			timerpool.Put(t)
+			defer func() { <-concurrencyCh }()
+		case <-t.C:
+			timerpool.Put(t)
+			concurrencyLimitTimeout.Inc()
+			err := &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("cannot handle more than %d concurrent search requests during %s; possible solutions: "+
+					"increase `-search.maxQueueDuration`, increase `-search.maxConcurrentRequests`, increase server capacity",
+					*maxConcurrentRequests, *maxQueueDuration),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, "%s", err)
+			return true
+		}
 	}
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	if path == "/internal/resetRollupResultCache" {
+		if len(*resetCacheAuthKey) > 0 && r.FormValue("authKey") != *resetCacheAuthKey {
+			sendPrometheusError(w, r, fmt.Errorf("invalid authKey=%q for %q", r.FormValue("authKey"), path))
+			return true
+		}
+		promql.ResetRollupResultCache()
+		return true
+	}
+
 	if strings.HasPrefix(path, "/api/v1/label/") {
 		s := r.URL.Path[len("/api/v1/label/"):]
 		if strings.HasSuffix(s, "/values") {
 			labelValuesRequests.Inc()
 			labelName := s[:len(s)-len("/values")]
 			httpserver.EnableCORS(w, r)
-			if err := prometheus.LabelValuesHandler(labelName, w, r); err != nil {
+			if err := prometheus.LabelValuesHandler(startTime, labelName, w, r); err != nil {
 				labelValuesErrors.Inc()
 				sendPrometheusError(w, r, err)
 				return true
@@ -73,7 +128,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/query":
 		queryRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.QueryHandler(w, r); err != nil {
+		if err := prometheus.QueryHandler(startTime, w, r); err != nil {
 			queryErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -82,7 +137,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/query_range":
 		queryRangeRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.QueryRangeHandler(w, r); err != nil {
+		if err := prometheus.QueryRangeHandler(startTime, w, r); err != nil {
 			queryRangeErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -91,7 +146,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/series":
 		seriesRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.SeriesHandler(w, r); err != nil {
+		if err := prometheus.SeriesHandler(startTime, w, r); err != nil {
 			seriesErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -100,7 +155,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/series/count":
 		seriesCountRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.SeriesCountHandler(w, r); err != nil {
+		if err := prometheus.SeriesCountHandler(startTime, w, r); err != nil {
 			seriesCountErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -109,15 +164,24 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/labels":
 		labelsRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.LabelsHandler(w, r); err != nil {
+		if err := prometheus.LabelsHandler(startTime, w, r); err != nil {
 			labelsErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
 		}
 		return true
+	case "/api/v1/labels/count":
+		labelsCountRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		if err := prometheus.LabelsCountHandler(startTime, w, r); err != nil {
+			labelsCountErrors.Inc()
+			sendPrometheusError(w, r, err)
+			return true
+		}
+		return true
 	case "/api/v1/export":
 		exportRequests.Inc()
-		if err := prometheus.ExportHandler(w, r); err != nil {
+		if err := prometheus.ExportHandler(startTime, w, r); err != nil {
 			exportErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
@@ -125,12 +189,30 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/federate":
 		federateRequests.Inc()
-		if err := prometheus.FederateHandler(w, r); err != nil {
+		if err := prometheus.FederateHandler(startTime, w, r); err != nil {
 			federateErrors.Inc()
 			httpserver.Errorf(w, "error int %q: %s", r.URL.Path, err)
 			return true
 		}
 		return true
+	case "/api/v1/rules":
+		// Return dumb placeholder
+		rulesRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":{"groups":[]}}`)
+		return true
+	case "/api/v1/alerts":
+		// Return dumb placehloder
+		alertsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
+		return true
+	case "/api/v1/metadata":
+		// Return dumb placeholder
+		metadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":{}}`)
+		return true
 	case "/api/v1/admin/tsdb/delete_series":
 		deleteRequests.Inc()
 		authKey := r.FormValue("authKey")
@@ -138,7 +220,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, "invalid authKey %q. It must match the value from -deleteAuthKey command line flag", authKey)
 			return true
 		}
-		if err := prometheus.DeleteHandler(r); err != nil {
+		if err := prometheus.DeleteHandler(startTime, r); err != nil {
 			deleteErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
@@ -151,10 +233,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
-	logger.Errorf("error in %q: %s", r.URL.Path, err)
+	logger.Errorf("error in %q: %s", r.RequestURI, err)
 
 	w.Header().Set("Content-Type", "application/json")
-	statusCode := 422
+	statusCode := http.StatusUnprocessableEntity
+	if esc, ok := err.(*httpserver.ErrorWithStatusCode); ok {
+		statusCode = esc.StatusCode
+	}
 	w.WriteHeader(statusCode)
 	prometheus.WriteErrorResponse(w, statusCode, err)
 }
@@ -178,6 +263,9 @@ var (
 	labelsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/labels"}`)
 	labelsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/labels"}`)
 
+	labelsCountRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/labels/count"}`)
+	labelsCountErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/labels/count"}`)
+
 	deleteRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/admin/tsdb/delete_series"}`)
 	deleteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/admin/tsdb/delete_series"}`)
 
@@ -186,4 +274,8 @@ var (
 
 	federateRequests = metrics.NewCounter(`vm_http_requests_total{path="/federate"}`)
 	federateErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/federate"}`)
+
+	rulesRequests    = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
+	alertsRequests   = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
+	metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
 )

app/vmselect/netstorage/fadvise_darwin.go

@@ -1,9 +0,0 @@
-package netstorage
-
-import (
-	"os"
-)
-
-func mustFadviseRandomRead(f *os.File) {
-	// Do nothing :)
-}

app/vmselect/netstorage/fadvise_linux.go

@@ -1,15 +0,0 @@
-package netstorage
-
-import (
-	"os"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"golang.org/x/sys/unix"
-)
-
-func mustFadviseRandomRead(f *os.File) {
-	fd := int(f.Fd())
-	if err := unix.Fadvise(int(fd), 0, 0, unix.FADV_RANDOM|unix.FADV_WILLNEED); err != nil {
-		logger.Panicf("FATAL: error returned from unix.Fadvise(RANDOM|WILLNEED): %s", err)
-	}
-}

app/vmselect/netstorage/netstorage.go

@@ -19,9 +19,9 @@ import (
 )
 
 var (
-	maxTagKeysPerSearch   = flag.Int("search.maxTagKeys", 10e3, "The maximum number of tag keys returned per search")
-	maxTagValuesPerSearch = flag.Int("search.maxTagValues", 10e3, "The maximum number of tag values returned per search")
-	maxMetricsPerSearch   = flag.Int("search.maxUniqueTimeseries", 100e3, "The maximum number of unique time series each search can scan")
+	maxTagKeysPerSearch   = flag.Int("search.maxTagKeys", 100e3, "The maximum number of tag keys returned per search")
+	maxTagValuesPerSearch = flag.Int("search.maxTagValues", 100e3, "The maximum number of tag values returned per search")
+	maxMetricsPerSearch   = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series each search can scan")
 )
 
 // Result is a single timeseries result.
@@ -49,8 +49,9 @@ func (r *Result) reset() {
 
 // Results holds results returned from ProcessSearchQuery.
 type Results struct {
-	tr       storage.TimeRange
-	deadline Deadline
+	tr        storage.TimeRange
+	fetchData bool
+	deadline  Deadline
 
 	tbf *tmpBlocksFile
 
@@ -71,9 +72,10 @@ func (rss *Results) Cancel() {
 // RunParallel runs in parallel f for all the results from rss.
 //
 // f shouldn't hold references to rs after returning.
+// workerID is the id of the worker goroutine that calls f.
 //
 // rss becomes unusable after the call to RunParallel.
-func (rss *Results) RunParallel(f func(rs *Result)) error {
+func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 	defer func() {
 		putTmpBlocksFile(rss.tbf)
 		rss.tbf = nil
@@ -90,38 +92,43 @@ func (rss *Results) RunParallel(f func(rs *Result)) error {
 	doneCh := make(chan error)
 
 	// Start workers.
+	rowsProcessedTotal := uint64(0)
 	for i := 0; i < workersCount; i++ {
-		go func() {
+		go func(workerID uint) {
 			rs := getResult()
 			defer putResult(rs)
 			maxWorkersCount := gomaxprocs / workersCount
 
 			var err error
+			rowsProcessed := 0
 			for pts := range workCh {
 				if time.Until(rss.deadline.Deadline) < 0 {
-					err = fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.Timeout)
+					err = fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.String())
 					break
 				}
-				if err = pts.Unpack(rss.tbf, rs, rss.tr, maxWorkersCount); err != nil {
+				if err = pts.Unpack(rss.tbf, rs, rss.tr, rss.fetchData, maxWorkersCount); err != nil {
 					break
 				}
-				if len(rs.Timestamps) == 0 {
+				if len(rs.Timestamps) == 0 && rss.fetchData {
 					// Skip empty blocks.
 					continue
 				}
-				f(rs)
+				rowsProcessed += len(rs.Values)
+				f(rs, workerID)
 			}
+			atomic.AddUint64(&rowsProcessedTotal, uint64(rowsProcessed))
 			// Drain the remaining work
 			for range workCh {
 			}
 			doneCh <- err
-		}()
+		}(uint(i))
 	}
 
 	// Feed workers with work.
 	for i := range rss.packedTimeseries {
 		workCh <- &rss.packedTimeseries[i]
 	}
+	seriesProcessedTotal := len(rss.packedTimeseries)
 	rss.packedTimeseries = rss.packedTimeseries[:0]
 	close(workCh)
 
@@ -132,6 +139,8 @@ func (rss *Results) RunParallel(f func(rs *Result)) error {
 			errors = append(errors, err)
 		}
 	}
+	perQueryRowsProcessed.Update(float64(rowsProcessedTotal))
+	perQuerySeriesProcessed.Update(float64(seriesProcessedTotal))
 	if len(errors) > 0 {
 		// Return just the first error, since other errors
 		// is likely duplicate the first error.
@@ -140,6 +149,9 @@ func (rss *Results) RunParallel(f func(rs *Result)) error {
 	return nil
 }
 
+var perQueryRowsProcessed = metrics.NewHistogram(`vm_per_query_rows_processed_count`)
+var perQuerySeriesProcessed = metrics.NewHistogram(`vm_per_query_series_processed_count`)
+
 var gomaxprocs = runtime.GOMAXPROCS(-1)
 
 type packedTimeseries struct {
@@ -148,7 +160,7 @@ type packedTimeseries struct {
 }
 
 // Unpack unpacks pts to dst.
-func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.TimeRange, maxWorkersCount int) error {
+func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.TimeRange, fetchData bool, maxWorkersCount int) error {
 	dst.reset()
 
 	if err := dst.MetricName.Unmarshal(bytesutil.ToUnsafeBytes(pts.metricName)); err != nil {
@@ -175,7 +187,7 @@ func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.
 			var err error
 			for addr := range workCh {
 				sb := getSortBlock()
-				if err = sb.unpackFrom(tbf, addr, tr); err != nil {
+				if err = sb.unpackFrom(tbf, addr, tr, fetchData); err != nil {
 					break
 				}
 
@@ -254,7 +266,7 @@ func mergeSortBlocks(dst *Result, sbh sortBlocksHeap) {
 			dst.Timestamps = append(dst.Timestamps, top.Timestamps[top.NextIdx:]...)
 			dst.Values = append(dst.Values, top.Values[top.NextIdx:]...)
 			putSortBlock(top)
-			return
+			break
 		}
 		sbNext := sbh[0]
 		tsNext := sbNext.Timestamps[sbNext.NextIdx]
@@ -275,6 +287,8 @@ func mergeSortBlocks(dst *Result, sbh sortBlocksHeap) {
 			putSortBlock(top)
 		}
 	}
+
+	dst.Timestamps, dst.Values = storage.DeduplicateSamples(dst.Timestamps, dst.Values)
 }
 
 type sortBlock struct {
@@ -294,10 +308,12 @@ func (sb *sortBlock) reset() {
 	sb.NextIdx = 0
 }
 
-func (sb *sortBlock) unpackFrom(tbf *tmpBlocksFile, addr tmpBlockAddr, tr storage.TimeRange) error {
+func (sb *sortBlock) unpackFrom(tbf *tmpBlocksFile, addr tmpBlockAddr, tr storage.TimeRange, fetchData bool) error {
 	tbf.MustReadBlockAt(&sb.b, addr)
-	if err := sb.b.UnmarshalData(); err != nil {
-		return fmt.Errorf("cannot unmarshal block: %s", err)
+	if fetchData {
+		if err := sb.b.UnmarshalData(); err != nil {
+			return fmt.Errorf("cannot unmarshal block: %s", err)
+		}
 	}
 	timestamps := sb.b.Timestamps()
 
@@ -400,6 +416,33 @@ func GetLabelValues(labelName string, deadline Deadline) ([]string, error) {
 	return labelValues, nil
 }
 
+// GetLabelEntries returns all the label entries until the given deadline.
+func GetLabelEntries(deadline Deadline) ([]storage.TagEntry, error) {
+	labelEntries, err := vmstorage.SearchTagEntries(*maxTagKeysPerSearch, *maxTagValuesPerSearch)
+	if err != nil {
+		return nil, fmt.Errorf("error during label entries request: %s", err)
+	}
+
+	// Substitute "" with "__name__"
+	for i := range labelEntries {
+		e := &labelEntries[i]
+		if e.Key == "" {
+			e.Key = "__name__"
+		}
+	}
+
+	// Sort labelEntries by the number of label values in each entry.
+	sort.Slice(labelEntries, func(i, j int) bool {
+		a, b := labelEntries[i].Values, labelEntries[j].Values
+		if len(a) != len(b) {
+			return len(a) > len(b)
+		}
+		return labelEntries[i].Key > labelEntries[j].Key
+	})
+
+	return labelEntries, nil
+}
+
 // GetSeriesCount returns the number of unique series.
 func GetSeriesCount(deadline Deadline) (uint64, error) {
 	n, err := vmstorage.GetSeriesCount()
@@ -418,18 +461,14 @@ func getStorageSearch() *storage.Search {
 }
 
 func putStorageSearch(sr *storage.Search) {
-	n := atomic.LoadUint64(&sr.MissingMetricNamesForMetricID)
-	missingMetricNamesForMetricID.Add(int(n))
 	sr.MustClose()
 	ssPool.Put(sr)
 }
 
 var ssPool sync.Pool
 
-var missingMetricNamesForMetricID = metrics.NewCounter(`vm_missing_metric_names_for_metric_id_total`)
-
 // ProcessSearchQuery performs sq on storage nodes until the given deadline.
-func ProcessSearchQuery(sq *storage.SearchQuery, deadline Deadline) (*Results, error) {
+func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadline) (*Results, error) {
 	// Setup search.
 	tfss, err := setupTfss(sq.TagFilterss)
 	if err != nil {
@@ -445,35 +484,41 @@ func ProcessSearchQuery(sq *storage.SearchQuery, deadline Deadline) (*Results, e
 
 	sr := getStorageSearch()
 	defer putStorageSearch(sr)
-	sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch)
+	sr.Init(vmstorage.Storage, tfss, tr, fetchData, *maxMetricsPerSearch)
 
 	tbf := getTmpBlocksFile()
 	m := make(map[string][]tmpBlockAddr)
+	blocksRead := 0
+	bb := tmpBufPool.Get()
+	defer tmpBufPool.Put(bb)
 	for sr.NextMetricBlock() {
-		addr, err := tbf.WriteBlock(sr.MetricBlock.Block)
+		blocksRead++
+		bb.B = storage.MarshalBlock(bb.B[:0], sr.MetricBlock.Block)
+		addr, err := tbf.WriteBlockData(bb.B)
 		if err != nil {
 			putTmpBlocksFile(tbf)
-			return nil, fmt.Errorf("cannot write data to temporary blocks file: %s", err)
+			return nil, fmt.Errorf("cannot write data block #%d to temporary blocks file: %s", blocksRead, err)
 		}
 		if time.Until(deadline.Deadline) < 0 {
 			putTmpBlocksFile(tbf)
-			return nil, fmt.Errorf("timeout exceeded while fetching data from storage: %s", deadline.Timeout)
+			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
 		}
 		metricName := sr.MetricBlock.MetricName
 		m[string(metricName)] = append(m[string(metricName)], addr)
 	}
 	if err := sr.Error(); err != nil {
 		putTmpBlocksFile(tbf)
-		return nil, fmt.Errorf("search error: %s", err)
+		return nil, fmt.Errorf("search error after reading %d data blocks: %s", blocksRead, err)
 	}
 	if err := tbf.Finalize(); err != nil {
 		putTmpBlocksFile(tbf)
-		return nil, fmt.Errorf("cannot finalize temporary blocks file: %s", err)
+		return nil, fmt.Errorf("cannot finalize temporary blocks file with %d blocks: %s", blocksRead, err)
 	}
 
 	var rss Results
 	rss.packedTimeseries = make([]packedTimeseries, len(m))
 	rss.tr = tr
+	rss.fetchData = fetchData
 	rss.deadline = deadline
 	rss.tbf = tbf
 	i := 0
@@ -483,6 +528,15 @@ func ProcessSearchQuery(sq *storage.SearchQuery, deadline Deadline) (*Results, e
 		pts.metricName = metricName
 		pts.addrs = addrs
 	}
+
+	// Sort rss.packedTimeseries by the first addr offset in order
+	// to reduce the number of disk seeks during unpacking in RunParallel.
+	// In this case tmpBlocksFile must be read almost sequentially.
+	sort.Slice(rss.packedTimeseries, func(i, j int) bool {
+		pts := rss.packedTimeseries
+		return pts[i].addrs[0].offset < pts[j].addrs[0].offset
+	})
+
 	return &rss, nil
 }
 
@@ -523,13 +577,24 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error)
 // Deadline contains deadline with the corresponding timeout for pretty error messages.
 type Deadline struct {
 	Deadline time.Time
-	Timeout  time.Duration
+
+	timeout  time.Duration
+	flagHint string
 }
 
 // NewDeadline returns deadline for the given timeout.
-func NewDeadline(timeout time.Duration) Deadline {
+//
+// flagHint must contain a hit for command-line flag, which could be used
+// in order to increase timeout.
+func NewDeadline(timeout time.Duration, flagHint string) Deadline {
 	return Deadline{
 		Deadline: time.Now().Add(timeout),
-		Timeout:  timeout,
+		timeout:  timeout,
+		flagHint: flagHint,
 	}
 }
+
+// String returns human-readable string representation for d.
+func (d *Deadline) String() string {
+	return fmt.Sprintf("%.3f seconds; the timeout can be adjusted with `%s` command-line flag", d.timeout.Seconds(), d.flagHint)
+}

app/vmselect/netstorage/tmp_blocks_file.go

@@ -1,7 +1,6 @@
 package netstorage
 
 import (
-	"bufio"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -10,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -22,9 +22,7 @@ func InitTmpBlocksDir(tmpDirPath string) {
 		tmpDirPath = os.TempDir()
 	}
 	tmpBlocksDir = tmpDirPath + "/searchResults"
-	if err := os.RemoveAll(tmpBlocksDir); err != nil {
-		logger.Panicf("FATAL: cannot remove %q: %s", tmpBlocksDir, err)
-	}
+	fs.MustRemoveAll(tmpBlocksDir)
 	if err := fs.MkdirAllIfNotExist(tmpBlocksDir); err != nil {
 		logger.Panicf("FATAL: cannot create %q: %s", tmpBlocksDir, err)
 	}
@@ -32,13 +30,27 @@ func InitTmpBlocksDir(tmpDirPath string) {
 
 var tmpBlocksDir string
 
-const maxInmemoryTmpBlocksFile = 512 * 1024
+func maxInmemoryTmpBlocksFile() int {
+	mem := memory.Allowed()
+	maxLen := mem / 1024
+	if maxLen < 64*1024 {
+		return 64 * 1024
+	}
+	if maxLen > 4*1024*1024 {
+		return 4 * 1024 * 1024
+	}
+	return maxLen
+}
+
+var _ = metrics.NewGauge(`vm_tmp_blocks_max_inmemory_file_size_bytes`, func() float64 {
+	return float64(maxInmemoryTmpBlocksFile())
+})
 
 type tmpBlocksFile struct {
 	buf []byte
 
-	f  *os.File
-	bw *bufio.Writer
+	f *os.File
+	r *fs.ReaderAt
 
 	offset uint64
 }
@@ -46,7 +58,9 @@ type tmpBlocksFile struct {
 func getTmpBlocksFile() *tmpBlocksFile {
 	v := tmpBlocksFilePool.Get()
 	if v == nil {
-		return &tmpBlocksFile{}
+		return &tmpBlocksFile{
+			buf: make([]byte, 0, maxInmemoryTmpBlocksFile()),
+		}
 	}
 	return v.(*tmpBlocksFile)
 }
@@ -55,7 +69,7 @@ func putTmpBlocksFile(tbf *tmpBlocksFile) {
 	tbf.MustClose()
 	tbf.buf = tbf.buf[:0]
 	tbf.f = nil
-	tbf.bw = nil
+	tbf.r = nil
 	tbf.offset = 0
 	tmpBlocksFilePool.Put(tbf)
 }
@@ -71,51 +85,34 @@ func (addr tmpBlockAddr) String() string {
 	return fmt.Sprintf("offset %d, size %d", addr.offset, addr.size)
 }
 
-func getBufioWriter(f *os.File) *bufio.Writer {
-	v := bufioWriterPool.Get()
-	if v == nil {
-		return bufio.NewWriterSize(f, maxInmemoryTmpBlocksFile*2)
-	}
-	bw := v.(*bufio.Writer)
-	bw.Reset(f)
-	return bw
-}
-
-func putBufioWriter(bw *bufio.Writer) {
-	bufioWriterPool.Put(bw)
-}
-
-var bufioWriterPool sync.Pool
-
 var tmpBlocksFilesCreated = metrics.NewCounter(`vm_tmp_blocks_files_created_total`)
 
-// WriteBlock writes b to tbf.
+// WriteBlockData writes b to tbf.
 //
 // It returns errors since the operation may fail on space shortage
 // and this must be handled.
-func (tbf *tmpBlocksFile) WriteBlock(b *storage.Block) (tmpBlockAddr, error) {
+func (tbf *tmpBlocksFile) WriteBlockData(b []byte) (tmpBlockAddr, error) {
 	var addr tmpBlockAddr
 	addr.offset = tbf.offset
-
-	tbfBufLen := len(tbf.buf)
-	tbf.buf = storage.MarshalBlock(tbf.buf, b)
-	addr.size = len(tbf.buf) - tbfBufLen
+	addr.size = len(b)
 	tbf.offset += uint64(addr.size)
-	if tbf.offset <= maxInmemoryTmpBlocksFile {
+	if len(tbf.buf)+len(b) <= cap(tbf.buf) {
+		// Fast path - the data fits tbf.buf
+		tbf.buf = append(tbf.buf, b...)
 		return addr, nil
 	}
 
+	// Slow path: flush the data from tbf.buf to file.
 	if tbf.f == nil {
 		f, err := ioutil.TempFile(tmpBlocksDir, "")
 		if err != nil {
 			return addr, err
 		}
 		tbf.f = f
-		tbf.bw = getBufioWriter(f)
 		tmpBlocksFilesCreated.Inc()
 	}
-	_, err := tbf.bw.Write(tbf.buf)
-	tbf.buf = tbf.buf[:0]
+	_, err := tbf.f.Write(tbf.buf)
+	tbf.buf = append(tbf.buf[:0], b...)
 	if err != nil {
 		return addr, fmt.Errorf("cannot write block to %q: %s", tbf.f.Name(), err)
 	}
@@ -126,15 +123,21 @@ func (tbf *tmpBlocksFile) Finalize() error {
 	if tbf.f == nil {
 		return nil
 	}
-
-	err := tbf.bw.Flush()
-	putBufioWriter(tbf.bw)
-	tbf.bw = nil
-	if _, err := tbf.f.Seek(0, 0); err != nil {
-		logger.Panicf("FATAL: cannot seek to the start of file: %s", err)
+	fname := tbf.f.Name()
+	if _, err := tbf.f.Write(tbf.buf); err != nil {
+		return fmt.Errorf("cannot write the remaining %d bytes to %q: %s", len(tbf.buf), fname, err)
 	}
-	mustFadviseRandomRead(tbf.f)
-	return err
+	tbf.buf = tbf.buf[:0]
+	r, err := fs.OpenReaderAt(fname)
+	if err != nil {
+		logger.Panicf("FATAL: cannot open %q: %s", fname, err)
+	}
+	// Hint the OS that the file is read almost sequentiallly.
+	// This should reduce the number of disk seeks, which is important
+	// for HDDs.
+	r.MustFadviseSequentialRead(true)
+	tbf.r = r
+	return nil
 }
 
 func (tbf *tmpBlocksFile) MustReadBlockAt(dst *storage.Block, addr tmpBlockAddr) {
@@ -145,13 +148,7 @@ func (tbf *tmpBlocksFile) MustReadBlockAt(dst *storage.Block, addr tmpBlockAddr)
 		bb := tmpBufPool.Get()
 		defer tmpBufPool.Put(bb)
 		bb.B = bytesutil.Resize(bb.B, addr.size)
-		n, err := tbf.f.ReadAt(bb.B, int64(addr.offset))
-		if err != nil {
-			logger.Panicf("FATAL: cannot read from %q at %s: %s", tbf.f.Name(), addr, err)
-		}
-		if n != len(bb.B) {
-			logger.Panicf("FATAL: too short number of bytes read at %s; got %d; want %d", addr, n, len(bb.B))
-		}
+		tbf.r.MustReadAt(bb.B, int64(addr.offset))
 		buf = bb.B
 	}
 	tail, err := storage.UnmarshalBlock(dst, buf)
@@ -169,9 +166,9 @@ func (tbf *tmpBlocksFile) MustClose() {
 	if tbf.f == nil {
 		return
 	}
-	if tbf.bw != nil {
-		putBufioWriter(tbf.bw)
-		tbf.bw = nil
+	if tbf.r != nil {
+		// tbf.r could be nil if Finalize wasn't called.
+		tbf.r.MustClose()
 	}
 	fname := tbf.f.Name()
 

app/vmselect/netstorage/tmp_blocks_file_test.go

@@ -30,7 +30,7 @@ func TestTmpBlocksFileSerial(t *testing.T) {
 }
 
 func TestTmpBlocksFileConcurrent(t *testing.T) {
-	concurrency := 4
+	concurrency := 3
 	ch := make(chan error, concurrency)
 	for i := 0; i < concurrency; i++ {
 		go func() {
@@ -69,7 +69,7 @@ func testTmpBlocksFile() error {
 		_, _, _ = b.MarshalData(0, 0)
 		return &b
 	}
-	for _, size := range []int{1024, 16 * 1024, maxInmemoryTmpBlocksFile / 2, 2 * maxInmemoryTmpBlocksFile} {
+	for _, size := range []int{1024, 16 * 1024, maxInmemoryTmpBlocksFile() / 2, 2 * maxInmemoryTmpBlocksFile()} {
 		err := func() error {
 			tbf := getTmpBlocksFile()
 			defer putTmpBlocksFile(tbf)
@@ -77,9 +77,12 @@ func testTmpBlocksFile() error {
 			// Write blocks until their summary size exceeds `size`.
 			var addrs []tmpBlockAddr
 			var blocks []*storage.Block
+			bb := tmpBufPool.Get()
+			defer tmpBufPool.Put(bb)
 			for tbf.offset < uint64(size) {
 				b := createBlock()
-				addr, err := tbf.WriteBlock(b)
+				bb.B = storage.MarshalBlock(bb.B[:0], b)
+				addr, err := tbf.WriteBlockData(bb.B)
 				if err != nil {
 					return fmt.Errorf("cannot write block at offset %d: %s", tbf.offset, err)
 				}
@@ -94,7 +97,7 @@ func testTmpBlocksFile() error {
 			}
 
 			// Read blocks in parallel and verify them
-			concurrency := 3
+			concurrency := 2
 			workCh := make(chan int)
 			doneCh := make(chan error)
 			for i := 0; i < concurrency; i++ {

app/vmselect/prometheus/export.qtpl

@@ -13,7 +13,7 @@
 	{% for i, ts := range rs.Timestamps %}
 		{%z= bb.B %}{% space %}
 		{%f= rs.Values[i] %}{% space %}
-		{%d= int(ts) %}{% newline %}
+		{%dl= ts %}{% newline %}
 	{% endfor %}
 	{% code quicktemplate.ReleaseByteBuffer(bb) %}
 {% endfunc %}
@@ -35,10 +35,10 @@
 		"timestamps":[
 			{% if len(rs.Timestamps) > 0 %}
 				{% code timestamps := rs.Timestamps %}
-				{%d= int(timestamps[0]) %}
+				{%dl= timestamps[0] %}
 				{% code timestamps = timestamps[1:] %}
 				{% for _, ts := range timestamps %}
-					,{%d= int(ts) %}
+					,{%dl= ts %}
 				{% endfor %}
 			{% endif %}
 		]

app/vmselect/prometheus/export.qtpl.go

@@ -49,7 +49,7 @@ func StreamExportPrometheusLine(qw422016 *qt422016.Writer, rs *netstorage.Result
 //line app/vmselect/prometheus/export.qtpl:15
 		qw422016.N().S(` `)
 //line app/vmselect/prometheus/export.qtpl:16
-		qw422016.N().D(int(ts))
+		qw422016.N().DL(ts)
 //line app/vmselect/prometheus/export.qtpl:16
 		qw422016.N().S(`
 `)
@@ -129,7 +129,7 @@ func StreamExportJSONLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
 		timestamps := rs.Timestamps
 
 //line app/vmselect/prometheus/export.qtpl:38
-		qw422016.N().D(int(timestamps[0]))
+		qw422016.N().DL(timestamps[0])
 //line app/vmselect/prometheus/export.qtpl:39
 		timestamps = timestamps[1:]
 
@@ -138,7 +138,7 @@ func StreamExportJSONLine(qw422016 *qt422016.Writer, rs *netstorage.Result) {
 //line app/vmselect/prometheus/export.qtpl:40
 			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/export.qtpl:41
-			qw422016.N().D(int(ts))
+			qw422016.N().DL(ts)
 //line app/vmselect/prometheus/export.qtpl:42
 		}
 //line app/vmselect/prometheus/export.qtpl:43

app/vmselect/prometheus/federate.qtpl

@@ -10,7 +10,7 @@
 	{% if len(rs.Timestamps) == 0 || len(rs.Values) == 0 %}{% return %}{% endif %}
 	{%= prometheusMetricName(&rs.MetricName) %}{% space %}
 	{%f= rs.Values[len(rs.Values)-1] %}{% space %}
-	{%d= int(rs.Timestamps[len(rs.Timestamps)-1]) %}{% newline %}
+	{%dl= rs.Timestamps[len(rs.Timestamps)-1] %}{% newline %}
 {% endfunc %}
 
 {% endstripspace %}

app/vmselect/prometheus/federate.qtpl.go

@@ -41,7 +41,7 @@ func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result) {
 //line app/vmselect/prometheus/federate.qtpl:12
 	qw422016.N().S(` `)
 //line app/vmselect/prometheus/federate.qtpl:13
-	qw422016.N().D(int(rs.Timestamps[len(rs.Timestamps)-1]))
+	qw422016.N().DL(rs.Timestamps[len(rs.Timestamps)-1])
 //line app/vmselect/prometheus/federate.qtpl:13
 	qw422016.N().S(`
 `)

app/vmselect/prometheus/labels_count_response.qtpl

@@ -0,0 +1,17 @@
+{% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" %}
+
+{% stripspace %}
+LabelsCountResponse generates response for /api/v1/labels/count .
+{% func LabelsCountResponse(labelEntries []storage.TagEntry) %}
+{
+	"status":"success",
+	"data":{
+		{% for i, e := range labelEntries %}
+			{%q= e.Key %}:{%d= len(e.Values) %}
+			{% if i+1 < len(labelEntries) %},{% endif %}
+		{% endfor %}
+	}
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/prometheus/labels_count_response.qtpl.go

@@ -0,0 +1,74 @@
+// Code generated by qtc from "labels_count_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/prometheus/labels_count_response.qtpl:1
+package prometheus
+
+//line app/vmselect/prometheus/labels_count_response.qtpl:1
+import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+
+// LabelsCountResponse generates response for /api/v1/labels/count .
+
+//line app/vmselect/prometheus/labels_count_response.qtpl:5
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/prometheus/labels_count_response.qtpl:5
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/prometheus/labels_count_response.qtpl:5
+func StreamLabelsCountResponse(qw422016 *qt422016.Writer, labelEntries []storage.TagEntry) {
+//line app/vmselect/prometheus/labels_count_response.qtpl:5
+	qw422016.N().S(`{"status":"success","data":{`)
+//line app/vmselect/prometheus/labels_count_response.qtpl:9
+	for i, e := range labelEntries {
+//line app/vmselect/prometheus/labels_count_response.qtpl:10
+		qw422016.N().Q(e.Key)
+//line app/vmselect/prometheus/labels_count_response.qtpl:10
+		qw422016.N().S(`:`)
+//line app/vmselect/prometheus/labels_count_response.qtpl:10
+		qw422016.N().D(len(e.Values))
+//line app/vmselect/prometheus/labels_count_response.qtpl:11
+		if i+1 < len(labelEntries) {
+//line app/vmselect/prometheus/labels_count_response.qtpl:11
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/labels_count_response.qtpl:11
+		}
+//line app/vmselect/prometheus/labels_count_response.qtpl:12
+	}
+//line app/vmselect/prometheus/labels_count_response.qtpl:12
+	qw422016.N().S(`}}`)
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+}
+
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+func WriteLabelsCountResponse(qq422016 qtio422016.Writer, labelEntries []storage.TagEntry) {
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+	StreamLabelsCountResponse(qw422016, labelEntries)
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+}
+
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+func LabelsCountResponse(labelEntries []storage.TagEntry) string {
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+	WriteLabelsCountResponse(qb422016, labelEntries)
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+	return qs422016
+//line app/vmselect/prometheus/labels_count_response.qtpl:15
+}

app/vmselect/prometheus/prometheus.go

@@ -6,41 +6,60 @@ import (
 	"math"
 	"net/http"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/quicktemplate"
 )
 
 var (
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum time for search query execution")
-	maxQueryLen      = flag.Int("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
+	latencyOffset = flag.Duration("search.latencyOffset", time.Second*30, "The time when data points become visible in query results after the colection. "+
+		"Too small value can result in incomplete last points for query results")
+	maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call")
+	maxQueryDuration  = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for search query execution")
+	maxQueryLen       = flag.Int("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
+	maxLookback       = flag.Duration("search.maxLookback", 0, "Synonim to -search.lookback-delta from Prometheus. "+
+		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg")
 )
 
 // Default step used if not set.
 const defaultStep = 5 * 60 * 1000
 
-// Latency for data processing pipeline, i.e. the time between data is ignested
-// into the system and the time it becomes visible to search.
-const latencyOffset = 60 * 1000
-
 // FederateHandler implements /federate . See https://prometheus.io/docs/prometheus/latest/federation/
-func FederateHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %s", err)
 	}
 	matches := r.Form["match[]"]
-	maxLookback := getDuration(r, "max_lookback", defaultStep)
-	start := getTime(r, "start", ct-maxLookback)
-	end := getTime(r, "end", ct)
-	deadline := getDeadline(r)
+	if len(matches) == 0 {
+		return fmt.Errorf("missing `match[]` arg")
+	}
+	lookbackDelta, err := getMaxLookback(r)
+	if err != nil {
+		return err
+	}
+	if lookbackDelta <= 0 {
+		lookbackDelta = defaultStep
+	}
+	start, err := getTime(r, "start", ct-lookbackDelta)
+	if err != nil {
+		return err
+	}
+	end, err := getTime(r, "end", ct)
+	if err != nil {
+		return err
+	}
+	deadline := getDeadlineForQuery(r)
 	if start >= end {
 		start = end - defaultStep
 	}
@@ -53,7 +72,7 @@ func FederateHandler(w http.ResponseWriter, r *http.Request) error {
 		MaxTimestamp: end,
 		TagFilterss:  tagFilterss,
 	}
-	rss, err := netstorage.ProcessSearchQuery(sq, deadline)
+	rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch data for %q: %s", sq, err)
 	}
@@ -61,7 +80,7 @@ func FederateHandler(w http.ResponseWriter, r *http.Request) error {
 	resultsCh := make(chan *quicktemplate.ByteBuffer)
 	doneCh := make(chan error)
 	go func() {
-		err := rss.RunParallel(func(rs *netstorage.Result) {
+		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
 			bb := quicktemplate.AcquireByteBuffer()
 			WriteFederate(bb, rs)
 			resultsCh <- bb
@@ -87,8 +106,7 @@ func FederateHandler(w http.ResponseWriter, r *http.Request) error {
 var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/federate"}`)
 
 // ExportHandler exports data in raw format from /api/v1/export.
-func ExportHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %s", err)
@@ -97,17 +115,26 @@ func ExportHandler(w http.ResponseWriter, r *http.Request) error {
 	if len(matches) == 0 {
 		// Maintain backwards compatibility
 		match := r.FormValue("match")
+		if len(match) == 0 {
+			return fmt.Errorf("missing `match[]` arg")
+		}
 		matches = []string{match}
 	}
-	start := getTime(r, "start", 0)
-	end := getTime(r, "end", ct)
+	start, err := getTime(r, "start", 0)
+	if err != nil {
+		return err
+	}
+	end, err := getTime(r, "end", ct)
+	if err != nil {
+		return err
+	}
 	format := r.FormValue("format")
-	deadline := getDeadline(r)
+	deadline := getDeadlineForExport(r)
 	if start >= end {
-		start = end - defaultStep
+		end = start + defaultStep
 	}
 	if err := exportHandler(w, matches, start, end, format, deadline); err != nil {
-		return err
+		return fmt.Errorf("error when exporting data for queries=%q on the time range (start=%d, end=%d): %s", matches, start, end, err)
 	}
 	exportDuration.UpdateDuration(startTime)
 	return nil
@@ -118,7 +145,7 @@ var exportDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 func exportHandler(w http.ResponseWriter, matches []string, start, end int64, format string, deadline netstorage.Deadline) error {
 	writeResponseFunc := WriteExportStdResponse
 	writeLineFunc := WriteExportJSONLine
-	contentType := "application/json"
+	contentType := "application/stream+json"
 	if format == "prometheus" {
 		contentType = "text/plain"
 		writeLineFunc = WriteExportPrometheusLine
@@ -136,7 +163,7 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 		MaxTimestamp: end,
 		TagFilterss:  tagFilterss,
 	}
-	rss, err := netstorage.ProcessSearchQuery(sq, deadline)
+	rss, err := netstorage.ProcessSearchQuery(sq, true, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch data for %q: %s", sq, err)
 	}
@@ -144,7 +171,7 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 	resultsCh := make(chan *quicktemplate.ByteBuffer, runtime.GOMAXPROCS(-1))
 	doneCh := make(chan error)
 	go func() {
-		err := rss.RunParallel(func(rs *netstorage.Result) {
+		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
 			bb := quicktemplate.AcquireByteBuffer()
 			writeLineFunc(bb, rs)
 			resultsCh <- bb
@@ -156,6 +183,11 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 	w.Header().Set("Content-Type", contentType)
 	writeResponseFunc(w, resultsCh)
 
+	// Consume all the data from resultsCh in the event writeResponseFunc
+	// fails to consume all the data.
+	for bb := range resultsCh {
+		quicktemplate.ReleaseByteBuffer(bb)
+	}
 	err = <-doneCh
 	if err != nil {
 		return fmt.Errorf("error during data fetching: %s", err)
@@ -166,8 +198,7 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 // DeleteHandler processes /api/v1/admin/tsdb/delete_series prometheus API request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#delete-series
-func DeleteHandler(r *http.Request) error {
-	startTime := time.Now()
+func DeleteHandler(startTime time.Time, r *http.Request) error {
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %s", err)
 	}
@@ -175,6 +206,9 @@ func DeleteHandler(r *http.Request) error {
 		return fmt.Errorf("start and end aren't supported. Remove these args from the query in order to delete all the matching metrics")
 	}
 	matches := r.Form["match[]"]
+	if len(matches) == 0 {
+		return fmt.Errorf("missing `match[]` arg")
+	}
 	tagFilterss, err := getTagFilterssFromMatches(matches)
 	if err != nil {
 		return err
@@ -198,12 +232,41 @@ var deleteDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 // LabelValuesHandler processes /api/v1/label/<labelName>/values request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
-func LabelValuesHandler(labelName string, w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
-	deadline := getDeadline(r)
-	labelValues, err := netstorage.GetLabelValues(labelName, deadline)
-	if err != nil {
-		return fmt.Errorf(`cannot obtain label values for %q: %s`, labelName, err)
+func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWriter, r *http.Request) error {
+	deadline := getDeadlineForQuery(r)
+
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %s", err)
+	}
+	var labelValues []string
+	if len(r.Form["match[]"]) == 0 && len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
+		var err error
+		labelValues, err = netstorage.GetLabelValues(labelName, deadline)
+		if err != nil {
+			return fmt.Errorf(`cannot obtain label values for %q: %s`, labelName, err)
+		}
+	} else {
+		// Extended functionality that allows filtering by label filters and time range
+		// i.e. /api/v1/label/foo/values?match[]=foobar{baz="abc"}&start=...&end=...
+		// is equivalent to `label_values(foobar{baz="abc"}, foo)` call on the selected
+		// time range in Grafana templating.
+		matches := r.Form["match[]"]
+		if len(matches) == 0 {
+			matches = []string{fmt.Sprintf("{%s!=''}", labelName)}
+		}
+		ct := currentTime()
+		end, err := getTime(r, "end", ct)
+		if err != nil {
+			return err
+		}
+		start, err := getTime(r, "start", end-defaultStep)
+		if err != nil {
+			return err
+		}
+		labelValues, err = labelValuesWithMatches(labelName, matches, start, end, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot obtain label values for %q, match[]=%q, start=%d, end=%d: %s", labelName, matches, start, end, err)
+		}
 	}
 
 	w.Header().Set("Content-Type", "application/json")
@@ -212,17 +275,111 @@ func LabelValuesHandler(labelName string, w http.ResponseWriter, r *http.Request
 	return nil
 }
 
+func labelValuesWithMatches(labelName string, matches []string, start, end int64, deadline netstorage.Deadline) ([]string, error) {
+	if len(matches) == 0 {
+		logger.Panicf("BUG: matches must be non-empty")
+	}
+	tagFilterss, err := getTagFilterssFromMatches(matches)
+	if err != nil {
+		return nil, err
+	}
+	for i, tfs := range tagFilterss {
+		// Add `labelName!=''` tag filter in order to filter out series without the labelName.
+		tagFilterss[i] = append(tfs, storage.TagFilter{
+			Key:        []byte(labelName),
+			IsNegative: true,
+		})
+	}
+	if start >= end {
+		end = start + defaultStep
+	}
+	sq := &storage.SearchQuery{
+		MinTimestamp: start,
+		MaxTimestamp: end,
+		TagFilterss:  tagFilterss,
+	}
+	rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
+	if err != nil {
+		return nil, fmt.Errorf("cannot fetch data for %q: %s", sq, err)
+	}
+
+	m := make(map[string]struct{})
+	var mLock sync.Mutex
+	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+		labelValue := rs.MetricName.GetTagValue(labelName)
+		if len(labelValue) == 0 {
+			return
+		}
+		mLock.Lock()
+		m[string(labelValue)] = struct{}{}
+		mLock.Unlock()
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error when data fetching: %s", err)
+	}
+
+	labelValues := make([]string, 0, len(m))
+	for labelValue := range m {
+		labelValues = append(labelValues, labelValue)
+	}
+	sort.Strings(labelValues)
+	return labelValues, nil
+}
+
 var labelValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/label/{}/values"}`)
 
+// LabelsCountHandler processes /api/v1/labels/count request.
+func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := getDeadlineForQuery(r)
+	labelEntries, err := netstorage.GetLabelEntries(deadline)
+	if err != nil {
+		return fmt.Errorf(`cannot obtain label entries: %s`, err)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	WriteLabelsCountResponse(w, labelEntries)
+	labelsCountDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var labelsCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/labels/count"}`)
+
 // LabelsHandler processes /api/v1/labels request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names
-func LabelsHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
-	deadline := getDeadline(r)
-	labels, err := netstorage.GetLabels(deadline)
-	if err != nil {
-		return fmt.Errorf("cannot obtain labels: %s", err)
+func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := getDeadlineForQuery(r)
+
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %s", err)
+	}
+	var labels []string
+	if len(r.Form["match[]"]) == 0 && len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
+		var err error
+		labels, err = netstorage.GetLabels(deadline)
+		if err != nil {
+			return fmt.Errorf("cannot obtain labels: %s", err)
+		}
+	} else {
+		// Extended functionality that allows filtering by label filters and time range
+		// i.e. /api/v1/labels?match[]=foobar{baz="abc"}&start=...&end=...
+		matches := r.Form["match[]"]
+		if len(matches) == 0 {
+			matches = []string{"{__name__!=''}"}
+		}
+		ct := currentTime()
+		end, err := getTime(r, "end", ct)
+		if err != nil {
+			return err
+		}
+		start, err := getTime(r, "start", end-defaultStep)
+		if err != nil {
+			return err
+		}
+		labels, err = labelsWithMatches(matches, start, end, deadline)
+		if err != nil {
+			return fmt.Errorf("cannot obtain labels for match[]=%q, start=%d, end=%d: %s", matches, start, end, err)
+		}
 	}
 
 	w.Header().Set("Content-Type", "application/json")
@@ -231,12 +388,56 @@ func LabelsHandler(w http.ResponseWriter, r *http.Request) error {
 	return nil
 }
 
+func labelsWithMatches(matches []string, start, end int64, deadline netstorage.Deadline) ([]string, error) {
+	if len(matches) == 0 {
+		logger.Panicf("BUG: matches must be non-empty")
+	}
+	tagFilterss, err := getTagFilterssFromMatches(matches)
+	if err != nil {
+		return nil, err
+	}
+	if start >= end {
+		end = start + defaultStep
+	}
+	sq := &storage.SearchQuery{
+		MinTimestamp: start,
+		MaxTimestamp: end,
+		TagFilterss:  tagFilterss,
+	}
+	rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
+	if err != nil {
+		return nil, fmt.Errorf("cannot fetch data for %q: %s", sq, err)
+	}
+
+	m := make(map[string]struct{})
+	var mLock sync.Mutex
+	err = rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
+		mLock.Lock()
+		tags := rs.MetricName.Tags
+		for i := range tags {
+			t := &tags[i]
+			m[string(t.Key)] = struct{}{}
+		}
+		m["__name__"] = struct{}{}
+		mLock.Unlock()
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error when data fetching: %s", err)
+	}
+
+	labels := make([]string, 0, len(m))
+	for label := range m {
+		labels = append(labels, label)
+	}
+	sort.Strings(labels)
+	return labels, nil
+}
+
 var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/labels"}`)
 
 // SeriesCountHandler processes /api/v1/series/count request.
-func SeriesCountHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
-	deadline := getDeadline(r)
+func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := getDeadlineForQuery(r)
 	n, err := netstorage.GetSeriesCount(deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain series count: %s", err)
@@ -252,31 +453,44 @@ var seriesCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
 // SeriesHandler processes /api/v1/series request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers
-func SeriesHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse form values: %s", err)
 	}
 	matches := r.Form["match[]"]
-	start := getTime(r, "start", ct-defaultStep)
-	end := getTime(r, "end", ct)
-	deadline := getDeadline(r)
+	if len(matches) == 0 {
+		return fmt.Errorf("missing `match[]` arg")
+	}
+	end, err := getTime(r, "end", ct)
+	if err != nil {
+		return err
+	}
+	// Do not set start to minTimeMsecs by default as Prometheus does,
+	// since this leads to fetching and scanning all the data from the storage,
+	// which can take a lot of time for big storages.
+	// It is better setting start as end-defaultStep by default.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/91
+	start, err := getTime(r, "start", end-defaultStep)
+	if err != nil {
+		return err
+	}
+	deadline := getDeadlineForQuery(r)
 
 	tagFilterss, err := getTagFilterssFromMatches(matches)
 	if err != nil {
 		return err
 	}
 	if start >= end {
-		start = end - defaultStep
+		end = start + defaultStep
 	}
 	sq := &storage.SearchQuery{
 		MinTimestamp: start,
 		MaxTimestamp: end,
 		TagFilterss:  tagFilterss,
 	}
-	rss, err := netstorage.ProcessSearchQuery(sq, deadline)
+	rss, err := netstorage.ProcessSearchQuery(sq, false, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch data for %q: %s", sq, err)
 	}
@@ -284,7 +498,7 @@ func SeriesHandler(w http.ResponseWriter, r *http.Request) error {
 	resultsCh := make(chan *quicktemplate.ByteBuffer)
 	doneCh := make(chan error)
 	go func() {
-		err := rss.RunParallel(func(rs *netstorage.Result) {
+		err := rss.RunParallel(func(rs *netstorage.Result, workerID uint) {
 			bb := quicktemplate.AcquireByteBuffer()
 			writemetricNameObject(bb, &rs.MetricName)
 			resultsCh <- bb
@@ -297,11 +511,10 @@ func SeriesHandler(w http.ResponseWriter, r *http.Request) error {
 	WriteSeriesResponse(w, resultsCh)
 
 	// Consume all the data from resultsCh in the event WriteSeriesResponse
-	// fail to consume all the data.
+	// fails to consume all the data.
 	for bb := range resultsCh {
 		quicktemplate.ReleaseByteBuffer(bb)
 	}
-
 	err = <-doneCh
 	if err != nil {
 		return fmt.Errorf("error during data fetching: %s", err)
@@ -315,57 +528,90 @@ var seriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 // QueryHandler processes /api/v1/query request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
-func QueryHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 
 	query := r.FormValue("query")
-	start := getTime(r, "time", ct)
-	step := getDuration(r, "step", latencyOffset)
-	deadline := getDeadline(r)
+	if len(query) == 0 {
+		return fmt.Errorf("missing `query` arg")
+	}
+	start, err := getTime(r, "time", ct)
+	if err != nil {
+		return err
+	}
+	queryOffset := getLatencyOffsetMilliseconds()
+	step, err := getDuration(r, "step", queryOffset)
+	if err != nil {
+		return err
+	}
+	deadline := getDeadlineForQuery(r)
+	lookbackDelta, err := getMaxLookback(r)
+	if err != nil {
+		return err
+	}
 
 	if len(query) > *maxQueryLen {
-		return fmt.Errorf(`too long query; got %d bytes; mustn't exceed %d bytes`, len(query), *maxQueryLen)
+		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), *maxQueryLen)
 	}
-	if ct-start < latencyOffset {
-		start -= latencyOffset
+	if !getBool(r, "nocache") && ct-start < queryOffset {
+		// Adjust start time only if `nocache` arg isn't set.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241
+		start = ct - queryOffset
 	}
 	if childQuery, windowStr, offsetStr := promql.IsMetricSelectorWithRollup(query); childQuery != "" {
-		var window int64
-		if len(windowStr) > 0 {
-			var err error
-			window, err = promql.DurationValue(windowStr, step)
-			if err != nil {
-				return err
-			}
+		window, err := parsePositiveDuration(windowStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse window: %s", err)
 		}
-		var offset int64
-		if len(offsetStr) > 0 {
-			var err error
-			offset, err = promql.DurationValue(offsetStr, step)
-			if err != nil {
-				return err
-			}
+		offset, err := parseDuration(offsetStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse offset: %s", err)
 		}
 		start -= offset
 		end := start
 		start = end - window
 		if err := exportHandler(w, []string{childQuery}, start, end, "promapi", deadline); err != nil {
-			return err
+			return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %s", childQuery, start, end, err)
+		}
+		queryDuration.UpdateDuration(startTime)
+		return nil
+	}
+	if childQuery, windowStr, stepStr, offsetStr := promql.IsRollup(query); childQuery != "" {
+		newStep, err := parsePositiveDuration(stepStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse step: %s", err)
+		}
+		if newStep > 0 {
+			step = newStep
+		}
+		window, err := parsePositiveDuration(windowStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse window: %s", err)
+		}
+		offset, err := parseDuration(offsetStr, step)
+		if err != nil {
+			return fmt.Errorf("cannot parse offset: %s", err)
+		}
+		start -= offset
+		end := start
+		start = end - window
+		if err := queryRangeHandler(w, childQuery, start, end, step, r, ct); err != nil {
+			return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %s", childQuery, start, end, step, err)
 		}
 		queryDuration.UpdateDuration(startTime)
 		return nil
 	}
 
 	ec := promql.EvalConfig{
-		Start:    start,
-		End:      start,
-		Step:     step,
-		Deadline: deadline,
+		Start:         start,
+		End:           start,
+		Step:          step,
+		Deadline:      deadline,
+		LookbackDelta: lookbackDelta,
 	}
-	result, err := promql.Exec(&ec, query)
+	result, err := promql.Exec(&ec, query, true)
 	if err != nil {
-		return fmt.Errorf("cannot execute %q: %s", query, err)
+		return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %s", query, start, step, err)
 	}
 
 	w.Header().Set("Content-Type", "application/json")
@@ -376,68 +622,143 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 
 var queryDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/query"}`)
 
+func parseDuration(s string, step int64) (int64, error) {
+	if len(s) == 0 {
+		return 0, nil
+	}
+	return metricsql.DurationValue(s, step)
+}
+
+func parsePositiveDuration(s string, step int64) (int64, error) {
+	if len(s) == 0 {
+		return 0, nil
+	}
+	return metricsql.PositiveDurationValue(s, step)
+}
+
 // QueryRangeHandler processes /api/v1/query_range request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
-func QueryRangeHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func QueryRangeHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 
 	query := r.FormValue("query")
-	start := getTime(r, "start", ct-defaultStep)
-	end := getTime(r, "end", ct)
-	step := getDuration(r, "step", defaultStep)
-	deadline := getDeadline(r)
+	if len(query) == 0 {
+		return fmt.Errorf("missing `query` arg")
+	}
+	start, err := getTime(r, "start", ct-defaultStep)
+	if err != nil {
+		return err
+	}
+	end, err := getTime(r, "end", ct)
+	if err != nil {
+		return err
+	}
+	step, err := getDuration(r, "step", defaultStep)
+	if err != nil {
+		return err
+	}
+	if err := queryRangeHandler(w, query, start, end, step, r, ct); err != nil {
+		return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %s", query, start, end, step, err)
+	}
+	queryRangeDuration.UpdateDuration(startTime)
+	return nil
+}
+
+func queryRangeHandler(w http.ResponseWriter, query string, start, end, step int64, r *http.Request, ct int64) error {
+	deadline := getDeadlineForQuery(r)
 	mayCache := !getBool(r, "nocache")
+	lookbackDelta, err := getMaxLookback(r)
+	if err != nil {
+		return err
+	}
 
 	// Validate input args.
 	if len(query) > *maxQueryLen {
-		return fmt.Errorf(`too long query; got %d bytes; mustn't exceed %d bytes`, len(query), *maxQueryLen)
+		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), *maxQueryLen)
 	}
 	if start > end {
-		start = end
+		end = start + defaultStep
 	}
 	if err := promql.ValidateMaxPointsPerTimeseries(start, end, step); err != nil {
 		return err
 	}
-	start, end = promql.AdjustStartEnd(start, end, step)
+	if mayCache {
+		start, end = promql.AdjustStartEnd(start, end, step)
+	}
 
 	ec := promql.EvalConfig{
-		Start:    start,
-		End:      end,
-		Step:     step,
-		Deadline: deadline,
-		MayCache: mayCache,
+		Start:         start,
+		End:           end,
+		Step:          step,
+		Deadline:      deadline,
+		MayCache:      mayCache,
+		LookbackDelta: lookbackDelta,
 	}
-	result, err := promql.Exec(&ec, query)
+	result, err := promql.Exec(&ec, query, false)
 	if err != nil {
-		return fmt.Errorf("cannot execute %q: %s", query, err)
+		return fmt.Errorf("cannot execute query: %s", err)
 	}
-	if ct-end < latencyOffset {
-		adjustLastPoints(result)
+	queryOffset := getLatencyOffsetMilliseconds()
+	if ct-end < queryOffset {
+		result = adjustLastPoints(result)
 	}
 
+	// Remove NaN values as Prometheus does.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153
+	removeNaNValuesInplace(result)
+
 	w.Header().Set("Content-Type", "application/json")
 	WriteQueryRangeResponse(w, result)
-	queryRangeDuration.UpdateDuration(startTime)
 	return nil
 }
 
+func removeNaNValuesInplace(tss []netstorage.Result) {
+	for i := range tss {
+		ts := &tss[i]
+		hasNaNs := false
+		for _, v := range ts.Values {
+			if math.IsNaN(v) {
+				hasNaNs = true
+				break
+			}
+		}
+		if !hasNaNs {
+			// Fast path: nothing to remove.
+			continue
+		}
+
+		// Slow path: remove NaNs.
+		srcTimestamps := ts.Timestamps
+		dstValues := ts.Values[:0]
+		dstTimestamps := ts.Timestamps[:0]
+		for j, v := range ts.Values {
+			if math.IsNaN(v) {
+				continue
+			}
+			dstValues = append(dstValues, v)
+			dstTimestamps = append(dstTimestamps, srcTimestamps[j])
+		}
+		ts.Values = dstValues
+		ts.Timestamps = dstTimestamps
+	}
+}
+
 var queryRangeDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/query_range"}`)
 
 // adjustLastPoints substitutes the last point values with the previous
 // point values, since the last points may contain garbage.
-func adjustLastPoints(tss []netstorage.Result) {
+func adjustLastPoints(tss []netstorage.Result) []netstorage.Result {
 	if len(tss) == 0 {
-		return
+		return nil
 	}
 
 	// Search for the last non-NaN value across all the timeseries.
 	lastNonNaNIdx := -1
 	for i := range tss {
-		r := &tss[i]
-		j := len(r.Values) - 1
-		for j >= 0 && math.IsNaN(r.Values[j]) {
+		values := tss[i].Values
+		j := len(values) - 1
+		for j >= 0 && math.IsNaN(values[j]) {
 			j--
 		}
 		if j > lastNonNaNIdx {
@@ -446,81 +767,117 @@ func adjustLastPoints(tss []netstorage.Result) {
 	}
 	if lastNonNaNIdx == -1 {
 		// All timeseries contain only NaNs.
-		return
+		return nil
 	}
 
-	// Substitute last three values starting from lastNonNaNIdx
+	// Substitute the last two values starting from lastNonNaNIdx
 	// with the previous values for each timeseries.
 	for i := range tss {
-		r := &tss[i]
-		for j := 0; j < 3; j++ {
+		values := tss[i].Values
+		for j := 0; j < 2; j++ {
 			idx := lastNonNaNIdx + j
-			if idx <= 0 || idx >= len(r.Values) {
+			if idx <= 0 || idx >= len(values) || math.IsNaN(values[idx-1]) {
 				continue
 			}
-			r.Values[idx] = r.Values[idx-1]
+			values[idx] = values[idx-1]
 		}
 	}
+	return tss
 }
 
-func getTime(r *http.Request, argKey string, defaultValue int64) int64 {
+func getTime(r *http.Request, argKey string, defaultValue int64) (int64, error) {
 	argValue := r.FormValue(argKey)
 	if len(argValue) == 0 {
-		return defaultValue
+		return defaultValue, nil
 	}
 	secs, err := strconv.ParseFloat(argValue, 64)
 	if err != nil {
 		// Try parsing string format
 		t, err := time.Parse(time.RFC3339, argValue)
 		if err != nil {
-			return defaultValue
+			// Handle Prometheus'-provided minTime and maxTime.
+			// See https://github.com/prometheus/client_golang/issues/614
+			switch argValue {
+			case prometheusMinTimeFormatted:
+				return minTimeMsecs, nil
+			case prometheusMaxTimeFormatted:
+				return maxTimeMsecs, nil
+			}
+			return 0, fmt.Errorf("cannot parse %q=%q: %s", argKey, argValue, err)
 		}
 		secs = float64(t.UnixNano()) / 1e9
 	}
 	msecs := int64(secs * 1e3)
-	if msecs < minTimeMsecs || msecs > maxTimeMsecs {
-		return defaultValue
+	if msecs < minTimeMsecs {
+		msecs = 0
 	}
-	return msecs
+	if msecs > maxTimeMsecs {
+		msecs = maxTimeMsecs
+	}
+	return msecs, nil
 }
 
+var (
+	// These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442
+	// See https://github.com/prometheus/client_golang/issues/614 for details.
+	prometheusMinTimeFormatted = time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Format(time.RFC3339Nano)
+	prometheusMaxTimeFormatted = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Format(time.RFC3339Nano)
+)
+
 const (
 	// These values prevent from overflow when storing msec-precision time in int64.
-	minTimeMsecs = int64(-1<<63) / 1e6
+	minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time
 	maxTimeMsecs = int64(1<<63-1) / 1e6
 )
 
-func getDuration(r *http.Request, argKey string, defaultValue int64) int64 {
+func getDuration(r *http.Request, argKey string, defaultValue int64) (int64, error) {
 	argValue := r.FormValue(argKey)
 	if len(argValue) == 0 {
-		return defaultValue
+		return defaultValue, nil
 	}
 	secs, err := strconv.ParseFloat(argValue, 64)
 	if err != nil {
 		// Try parsing string format
 		d, err := time.ParseDuration(argValue)
 		if err != nil {
-			return defaultValue
+			return 0, fmt.Errorf("cannot parse %q=%q: %s", argKey, argValue, err)
 		}
 		secs = d.Seconds()
 	}
 	msecs := int64(secs * 1e3)
 	if msecs <= 0 || msecs > maxDurationMsecs {
-		return defaultValue
+		return 0, fmt.Errorf("%q=%dms is out of allowed range [%d ... %d]", argKey, msecs, 0, int64(maxDurationMsecs))
 	}
-	return msecs
+	return msecs, nil
 }
 
 const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000
 
-func getDeadline(r *http.Request) netstorage.Deadline {
-	d := getDuration(r, "timeout", 0)
+func getMaxLookback(r *http.Request) (int64, error) {
+	d := int64(*maxLookback / time.Millisecond)
+	return getDuration(r, "max_lookback", d)
+}
+
+func getDeadlineForQuery(r *http.Request) netstorage.Deadline {
 	dMax := int64(maxQueryDuration.Seconds() * 1e3)
+	return getDeadlineWithMaxDuration(r, dMax, "-search.maxQueryDuration")
+}
+
+func getDeadlineForExport(r *http.Request) netstorage.Deadline {
+	dMax := int64(maxExportDuration.Seconds() * 1e3)
+	return getDeadlineWithMaxDuration(r, dMax, "-search.maxExportDuration")
+}
+
+func getDeadlineWithMaxDuration(r *http.Request, dMax int64, flagHint string) netstorage.Deadline {
+	d, err := getDuration(r, "timeout", 0)
+	if err != nil {
+		d = 0
+	}
 	if d <= 0 || d > dMax {
 		d = dMax
 	}
 	timeout := time.Duration(d) * time.Millisecond
-	return netstorage.NewDeadline(timeout)
+	return netstorage.NewDeadline(timeout, flagHint)
 }
 
 func getBool(r *http.Request, argKey string) bool {
@@ -548,3 +905,11 @@ func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error)
 	}
 	return tagFilterss, nil
 }
+
+func getLatencyOffsetMilliseconds() int64 {
+	d := int64(*latencyOffset / time.Millisecond)
+	if d <= 1000 {
+		d = 1000
+	}
+	return d
+}

app/vmselect/prometheus/prometheus_test.go

@@ -0,0 +1,115 @@
+package prometheus
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"net/url"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+)
+
+func TestRemoveNaNValuesInplace(t *testing.T) {
+	f := func(tss []netstorage.Result, tssExpected []netstorage.Result) {
+		t.Helper()
+		removeNaNValuesInplace(tss)
+		if !reflect.DeepEqual(tss, tssExpected) {
+			t.Fatalf("unexpected result; got %v; want %v", tss, tssExpected)
+		}
+	}
+
+	nan := math.NaN()
+
+	f(nil, nil)
+	f([]netstorage.Result{
+		{
+			Timestamps: []int64{100, 200, 300},
+			Values:     []float64{1, 2, 3},
+		},
+		{
+			Timestamps: []int64{100, 200, 300, 400},
+			Values:     []float64{nan, nan, 3, nan},
+		},
+	}, []netstorage.Result{
+		{
+			Timestamps: []int64{100, 200, 300},
+			Values:     []float64{1, 2, 3},
+		},
+		{
+			Timestamps: []int64{300},
+			Values:     []float64{3},
+		},
+	})
+}
+
+func TestGetTimeSuccess(t *testing.T) {
+	f := func(s string, timestampExpected int64) {
+		t.Helper()
+		urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
+		r, err := http.NewRequest("GET", urlStr, nil)
+		if err != nil {
+			t.Fatalf("unexpected error in NewRequest: %s", err)
+		}
+
+		// Verify defaultValue
+		ts, err := getTime(r, "foo", 123)
+		if err != nil {
+			t.Fatalf("unexpected error when obtaining default time from getTime(%q): %s", s, err)
+		}
+		if ts != 123 {
+			t.Fatalf("unexpected default value for getTime(%q); got %d; want %d", s, ts, 123)
+		}
+
+		// Verify timestampExpected
+		ts, err = getTime(r, "s", 123)
+		if err != nil {
+			t.Fatalf("unexpected error in getTime(%q): %s", s, err)
+		}
+		if ts != timestampExpected {
+			t.Fatalf("unexpected timestamp for getTime(%q); got %d; want %d", s, ts, timestampExpected)
+		}
+	}
+
+	f("2019-07-07T20:01:02Z", 1562529662000)
+	f("2019-07-07T20:47:40+03:00", 1562521660000)
+	f("-292273086-05-16T16:47:06Z", minTimeMsecs)
+	f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs)
+	f("1562529662.324", 1562529662324)
+	f("-9223372036.854", minTimeMsecs)
+	f("-9223372036.855", minTimeMsecs)
+	f("9223372036.855", maxTimeMsecs)
+}
+
+func TestGetTimeError(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
+		r, err := http.NewRequest("GET", urlStr, nil)
+		if err != nil {
+			t.Fatalf("unexpected error in NewRequest: %s", err)
+		}
+
+		// Verify defaultValue
+		ts, err := getTime(r, "foo", 123)
+		if err != nil {
+			t.Fatalf("unexpected error when obtaining default time from getTime(%q): %s", s, err)
+		}
+		if ts != 123 {
+			t.Fatalf("unexpected default value for getTime(%q); got %d; want %d", s, ts, 123)
+		}
+
+		// Verify timestampExpected
+		_, err = getTime(r, "s", 123)
+		if err == nil {
+			t.Fatalf("expecting non-nil error in getTime(%q)", s)
+		}
+	}
+
+	f("foo")
+	f("2019-07-07T20:01:02Zisdf")
+	f("2019-07-07T20:47:40+03:00123")
+	f("-292273086-05-16T16:47:07Z")
+	f("292277025-08-18T07:12:54.999999998Z")
+}

app/vmselect/prometheus/series_count_response.qtpl

@@ -3,7 +3,7 @@ SeriesCountResponse generates response for /api/v1/series/count .
 {% func SeriesCountResponse(n uint64) %}
 {
 	"status":"success",
-	"data":[{%d int(n) %}]
+	"data":[{%dl int64(n) %}]
 }
 {% endfunc %}
 {% endstripspace %}

app/vmselect/prometheus/series_count_response.qtpl.go

@@ -24,7 +24,7 @@ func StreamSeriesCountResponse(qw422016 *qt422016.Writer, n uint64) {
 //line app/vmselect/prometheus/series_count_response.qtpl:3
 	qw422016.N().S(`{"status":"success","data":[`)
 //line app/vmselect/prometheus/series_count_response.qtpl:6
-	qw422016.N().D(int(n))
+	qw422016.N().DL(int64(n))
 //line app/vmselect/prometheus/series_count_response.qtpl:6
 	qw422016.N().S(`]}`)
 //line app/vmselect/prometheus/series_count_response.qtpl:8

app/vmselect/promql/aggr.go

@@ -6,6 +6,12 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/histogram"
 )
 
 var aggrFuncs = map[string]aggrFunc{
@@ -22,17 +28,28 @@ var aggrFuncs = map[string]aggrFunc{
 	"topk":         newAggrFuncTopK(false),
 	"quantile":     aggrFuncQuantile,
 
-	// Extended PromQL funcs
-	"median":   aggrFuncMedian,
-	"limitk":   aggrFuncLimitK,
-	"distinct": newAggrFunc(aggrFuncDistinct),
+	// PromQL extension funcs
+	"median":         aggrFuncMedian,
+	"limitk":         aggrFuncLimitK,
+	"distinct":       newAggrFunc(aggrFuncDistinct),
+	"sum2":           newAggrFunc(aggrFuncSum2),
+	"geomean":        newAggrFunc(aggrFuncGeomean),
+	"histogram":      newAggrFunc(aggrFuncHistogram),
+	"topk_min":       newAggrFuncRangeTopK(minValue, false),
+	"topk_max":       newAggrFuncRangeTopK(maxValue, false),
+	"topk_avg":       newAggrFuncRangeTopK(avgValue, false),
+	"topk_median":    newAggrFuncRangeTopK(medianValue, false),
+	"bottomk_min":    newAggrFuncRangeTopK(minValue, true),
+	"bottomk_max":    newAggrFuncRangeTopK(maxValue, true),
+	"bottomk_avg":    newAggrFuncRangeTopK(avgValue, true),
+	"bottomk_median": newAggrFuncRangeTopK(medianValue, true),
 }
 
 type aggrFunc func(afa *aggrFuncArg) ([]*timeseries, error)
 
 type aggrFuncArg struct {
 	args [][]*timeseries
-	ae   *aggrFuncExpr
+	ae   *metricsql.AggrFuncExpr
 	ec   *EvalConfig
 }
 
@@ -41,20 +58,6 @@ func getAggrFunc(s string) aggrFunc {
 	return aggrFuncs[s]
 }
 
-func isAggrFunc(s string) bool {
-	return getAggrFunc(s) != nil
-}
-
-func isAggrFuncModifier(s string) bool {
-	s = strings.ToLower(s)
-	switch s {
-	case "by", "without":
-		return true
-	default:
-		return false
-	}
-}
-
 func newAggrFunc(afe func(tss []*timeseries) []*timeseries) aggrFunc {
 	return func(afa *aggrFuncArg) ([]*timeseries, error) {
 		args := afa.args
@@ -65,33 +68,26 @@ func newAggrFunc(afe func(tss []*timeseries) []*timeseries) aggrFunc {
 	}
 }
 
-func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeseries, modifier *modifierExpr, keepOriginal bool) ([]*timeseries, error) {
-	arg := copyTimeseriesMetricNames(argOrig)
-
-	// Filter out superflouos tags.
-	var groupTags []string
-	groupOp := "by"
-	if modifier.Op != "" {
-		groupTags = modifier.Args
-		groupOp = strings.ToLower(modifier.Op)
-	}
+func removeGroupTags(metricName *storage.MetricName, modifier *metricsql.ModifierExpr) {
+	groupOp := strings.ToLower(modifier.Op)
 	switch groupOp {
-	case "by":
-		for _, ts := range arg {
-			ts.MetricName.RemoveTagsOn(groupTags)
-		}
+	case "", "by":
+		metricName.RemoveTagsOn(modifier.Args)
 	case "without":
-		for _, ts := range arg {
-			ts.MetricName.RemoveTagsIgnoring(groupTags)
-		}
+		metricName.RemoveTagsIgnoring(modifier.Args)
 	default:
-		return nil, fmt.Errorf(`unknown modifier: %q`, groupOp)
+		logger.Panicf("BUG: unknown group modifier: %q", groupOp)
 	}
+}
+
+func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeseries, modifier *metricsql.ModifierExpr, keepOriginal bool) ([]*timeseries, error) {
+	arg := copyTimeseriesMetricNames(argOrig)
 
 	// Perform grouping.
 	m := make(map[string][]*timeseries)
 	bb := bbPool.Get()
 	for i, ts := range arg {
+		removeGroupTags(&ts.MetricName, modifier)
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
 		if keepOriginal {
 			ts = argOrig[i]
@@ -100,10 +96,18 @@ func aggrFuncExt(afe func(tss []*timeseries) []*timeseries, argOrig []*timeserie
 	}
 	bbPool.Put(bb)
 
+	srcTssCount := 0
+	dstTssCount := 0
 	rvs := make([]*timeseries, 0, len(m))
 	for _, tss := range m {
 		rv := afe(tss)
 		rvs = append(rvs, rv...)
+		srcTssCount += len(tss)
+		dstTssCount += len(rv)
+		if dstTssCount > 2000 && dstTssCount > 16*srcTssCount {
+			// This looks like count_values explosion.
+			return nil, fmt.Errorf(`too many timeseries after aggragation; got %d; want less than %d`, dstTssCount, 16*srcTssCount)
+		}
 	}
 	return rvs, nil
 }
@@ -132,6 +136,84 @@ func aggrFuncSum(tss []*timeseries) []*timeseries {
 	return tss[:1]
 }
 
+func aggrFuncSum2(tss []*timeseries) []*timeseries {
+	dst := tss[0]
+	for i := range dst.Values {
+		sum2 := float64(0)
+		count := 0
+		for _, ts := range tss {
+			v := ts.Values[i]
+			if math.IsNaN(v) {
+				continue
+			}
+			sum2 += v * v
+			count++
+		}
+		if count == 0 {
+			sum2 = nan
+		}
+		dst.Values[i] = sum2
+	}
+	return tss[:1]
+}
+
+func aggrFuncGeomean(tss []*timeseries) []*timeseries {
+	if len(tss) == 1 {
+		// Fast path - nothing to geomean.
+		return tss
+	}
+	dst := tss[0]
+	for i := range dst.Values {
+		p := 1.0
+		count := 0
+		for _, ts := range tss {
+			v := ts.Values[i]
+			if math.IsNaN(v) {
+				continue
+			}
+			p *= v
+			count++
+		}
+		if count == 0 {
+			p = nan
+		}
+		dst.Values[i] = math.Pow(p, 1/float64(count))
+	}
+	return tss[:1]
+}
+
+func aggrFuncHistogram(tss []*timeseries) []*timeseries {
+	var h metrics.Histogram
+	m := make(map[string]*timeseries)
+	for i := range tss[0].Values {
+		h.Reset()
+		for _, ts := range tss {
+			v := ts.Values[i]
+			h.Update(v)
+		}
+		h.VisitNonZeroBuckets(func(vmrange string, count uint64) {
+			ts := m[vmrange]
+			if ts == nil {
+				ts = &timeseries{}
+				ts.CopyFromShallowTimestamps(tss[0])
+				ts.MetricName.RemoveTag("vmrange")
+				ts.MetricName.AddTag("vmrange", vmrange)
+				values := ts.Values
+				for k := range values {
+					values[k] = 0
+				}
+				m[vmrange] = ts
+			}
+			ts.Values[i] = float64(count)
+		})
+	}
+	rvs := make([]*timeseries, 0, len(m))
+	for _, ts := range m {
+		rvs = append(rvs, ts)
+	}
+	return vmrangeBucketsToLE(rvs)
+}
+
 func aggrFuncMin(tss []*timeseries) []*timeseries {
 	if len(tss) == 1 {
 		// Fast path - nothing to min.
@@ -260,7 +342,11 @@ func aggrFuncCount(tss []*timeseries) []*timeseries {
 			}
 			count++
 		}
-		dst.Values[i] = float64(count)
+		v := float64(count)
+		if count == 0 {
+			v = nan
+		}
+		dst.Values[i] = v
 	}
 	return tss[:1]
 }
@@ -297,10 +383,32 @@ func aggrFuncCountValues(afa *aggrFuncArg) ([]*timeseries, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	// Remove dstLabel from grouping like Prometheus does.
+	modifier := &afa.ae.Modifier
+	switch strings.ToLower(modifier.Op) {
+	case "without":
+		modifier.Args = append(modifier.Args, dstLabel)
+	case "by":
+		dstArgs := modifier.Args[:0]
+		for _, arg := range modifier.Args {
+			if arg == dstLabel {
+				continue
+			}
+			dstArgs = append(dstArgs, arg)
+		}
+		modifier.Args = dstArgs
+	default:
+		// Do nothing
+	}
+
 	afe := func(tss []*timeseries) []*timeseries {
 		m := make(map[float64]bool)
 		for _, ts := range tss {
 			for _, v := range ts.Values {
+				if math.IsNaN(v) {
+					continue
+				}
 				m[v] = true
 			}
 		}
@@ -313,7 +421,7 @@ func aggrFuncCountValues(afa *aggrFuncArg) ([]*timeseries, error) {
 		var rvs []*timeseries
 		for _, v := range values {
 			var dst timeseries
-			dst.CopyFrom(tss[0])
+			dst.CopyFromShallowTimestamps(tss[0])
 			dst.MetricName.RemoveTag(dstLabel)
 			dst.MetricName.AddTag(dstLabel, strconv.FormatFloat(v, 'g', -1, 64))
 			for i := range dst.Values {
@@ -347,37 +455,138 @@ func newAggrFuncTopK(isReverse bool) aggrFunc {
 			return nil, err
 		}
 		afe := func(tss []*timeseries) []*timeseries {
-			rvs := tss
-			for n := range rvs[0].Values {
-				sort.Slice(rvs, func(i, j int) bool {
-					a := rvs[i].Values[n]
-					b := rvs[j].Values[n]
-					cmp := lessWithNaNs(a, b)
+			for n := range tss[0].Values {
+				sort.Slice(tss, func(i, j int) bool {
+					a := tss[i].Values[n]
+					b := tss[j].Values[n]
 					if isReverse {
-						cmp = !cmp
+						a, b = b, a
 					}
-					return cmp
+					return lessWithNaNs(a, b)
 				})
-				if math.IsNaN(ks[n]) {
-					ks[n] = 0
-				}
-				k := int(ks[n])
-				if k < 0 {
-					k = 0
-				}
-				if k > len(rvs) {
-					k = len(rvs)
-				}
-				for _, ts := range rvs[:len(rvs)-k] {
-					ts.Values[n] = nan
-				}
+				fillNaNsAtIdx(n, ks[n], tss)
 			}
-			return rvs
+			return removeNaNs(tss)
 		}
 		return aggrFuncExt(afe, args[1], &afa.ae.Modifier, true)
 	}
 }
 
+type tsWithValue struct {
+	ts    *timeseries
+	value float64
+}
+
+func newAggrFuncRangeTopK(f func(values []float64) float64, isReverse bool) aggrFunc {
+	return func(afa *aggrFuncArg) ([]*timeseries, error) {
+		args := afa.args
+		if err := expectTransformArgsNum(args, 2); err != nil {
+			return nil, err
+		}
+		ks, err := getScalar(args[0], 0)
+		if err != nil {
+			return nil, err
+		}
+		afe := func(tss []*timeseries) []*timeseries {
+			maxs := make([]tsWithValue, len(tss))
+			for i, ts := range tss {
+				value := f(ts.Values)
+				maxs[i] = tsWithValue{
+					ts:    ts,
+					value: value,
+				}
+			}
+			sort.Slice(maxs, func(i, j int) bool {
+				a := maxs[i].value
+				b := maxs[j].value
+				if isReverse {
+					a, b = b, a
+				}
+				return lessWithNaNs(a, b)
+			})
+			for i := range maxs {
+				tss[i] = maxs[i].ts
+			}
+			for i, k := range ks {
+				fillNaNsAtIdx(i, k, tss)
+			}
+			return removeNaNs(tss)
+		}
+		return aggrFuncExt(afe, args[1], &afa.ae.Modifier, true)
+	}
+}
+
+func fillNaNsAtIdx(idx int, k float64, tss []*timeseries) {
+	if math.IsNaN(k) {
+		k = 0
+	}
+	kn := int(k)
+	if kn < 0 {
+		kn = 0
+	}
+	if kn > len(tss) {
+		kn = len(tss)
+	}
+	for _, ts := range tss[:len(tss)-kn] {
+		ts.Values[idx] = nan
+	}
+}
+
+func minValue(values []float64) float64 {
+	if len(values) == 0 {
+		return nan
+	}
+	min := values[0]
+	for _, v := range values[1:] {
+		if v < min {
+			min = v
+		}
+	}
+	return min
+}
+
+func maxValue(values []float64) float64 {
+	if len(values) == 0 {
+		return nan
+	}
+	max := values[0]
+	for _, v := range values[1:] {
+		if v > max {
+			max = v
+		}
+	}
+	return max
+}
+
+func avgValue(values []float64) float64 {
+	sum := float64(0)
+	count := 0
+	for _, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		count++
+		sum += v
+	}
+	if count == 0 {
+		return nan
+	}
+	return sum / float64(count)
+}
+
+func medianValue(values []float64) float64 {
+	h := histogram.GetFast()
+	for _, v := range values {
+		if math.IsNaN(v) {
+			continue
+		}
+		h.Update(v)
+	}
+	value := h.Quantile(0.5)
+	histogram.PutFast(h)
+	return value
+}
+
 func aggrFuncLimitK(afa *aggrFuncArg) ([]*timeseries, error) {
 	args := afa.args
 	if err := expectTransformArgsNum(args, 2); err != nil {
@@ -457,6 +666,7 @@ func newAggrQuantileFunc(phis []float64) func(tss []*timeseries) []*timeseries {
 			idx := int(math.Round(float64(len(tss)-1) * phi))
 			dst.Values[n] = tss[idx].Values[n]
 		}
+		tss[0] = dst
 		return tss[:1]
 	}
 }