app/vmalert: initial remote-write support for alerts state persistence. (#442)

* app/vmalert: initial remote-write support for alerts state persistence.

If `remotewrite.url` flag is set, vmalert will send alerts state  via remote-write protocol to remote storage. The sending is asynchronous to avoid blocking calls in rules evaluation loop.

* app/vmalert: merge with master

* app/vmalert: write both `instant` and `for` alerts timeseries states in remote storage.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -26,5 +26,16 @@ $ ./victoria-metrics-prod --version
 victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
 ```
 
+**Used command-line flags**
+Command-line flags are listed as `flag{name="httpListenAddr", value=":443"} 1` lines at `/metrics` page.
+See the following docs for details:
+
+* [monitoring for single-node VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#monitoring)
+* [montioring for VictoriaMetrics cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/cluster/README.md#monitoring)
+
 **Additional context**
-Add any other context about the problem here such as error logs, `/metrics` output, screenshots from [the official Grafana dashboard for VictoriaMetrics](https://grafana.com/dashboards/10229).
+Add any other context about the problem here such as error logs from VictoriaMetrics and Prometheus,
+`/metrics` output, screenshots from the official Grafana dashboards for VictoriaMetrics:
+
+* [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
+* [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)

.github/workflows/github-pages.yml

@@ -2,7 +2,7 @@ name: github-pages
 on:
   push:
     paths:
-      - 'docs/*.md'
+      - 'docs/*'
       - 'README.md'
     branches:
       - master
@@ -17,14 +17,14 @@ jobs:
           TOKEN: ${{secrets.CI_TOKEN}}
         run: |
           git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git gpages
-          cp docs/*.md gpages
+          cp docs/* gpages
           cp README.md gpages
           cd gpages
           git config --local user.email "info@victoriametrics.com"
           git config --local user.name "Vika"
-          git add "*.md"
+          git add .
           git commit -m "update github pages"
           remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.github.io.git"
           git push "${remote_repo}"
           cd ..
-          rm -rf gpages
+          rm -rf gpages

.github/workflows/main.yml

@@ -14,18 +14,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@master
         with:
-          go-version: 1.13
+          go-version: 1.14
         id: go
-      - name: Code checkout
-        uses: actions/checkout@v1
       - name: Dependencies
         env:
-          GO111MODULE: off
+          GO111MODULE: on
         run: |
-          go get -v golang.org/x/lint/golint
+          go get -u golang.org/x/lint/golint
           go get -u github.com/kisielk/errcheck
+          go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
+      - name: Code checkout
+        uses: actions/checkout@master
       - name: Build
         env:
           GO111MODULE: on
@@ -44,7 +45,7 @@ jobs:
           GOOS=freebsd go build -mod=vendor ./app/victoria-metrics
           GOOS=darwin go build -mod=vendor ./app/victoria-metrics
       - name: Publish coverage
-        uses: codecov/codecov-action@v1.0.4
+        uses: codecov/codecov-action@v1.0.6
         with:
           token: ${{secrets.CODECOV_TOKEN}}
           file: ./coverage.txt

.github/workflows/wiki.yml

@@ -2,7 +2,7 @@ name: wiki
 on:
   push:
     paths:
-      - 'docs/*.md'
+      - 'docs/*'
     branches:
       - master
 jobs:
@@ -15,15 +15,14 @@ jobs:
         env:
           TOKEN: ${{secrets.CI_TOKEN}}
         run: |
-          cd docs
           git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git wiki
-          find ./ -name '*.md' -exec cp -prv '{}' 'wiki' ';'
+          cp docs/* wiki
           cd wiki
           git config --local user.email "info@victoriametrics.com"
           git config --local user.name "Vika"
-          git add "*.md"
+          git add .
           git commit -m "update wiki pages"
           remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git"
           git push "${remote_repo}"
           cd ..
-          rm -rf wiki
+          rm -rf wiki

.gitignore

@@ -8,6 +8,7 @@
 *.swp
 /gocache-for-docker
 /victoria-metrics-data
+/vmagent-remotewrite-data
 /vmstorage-data
 /vmselect-cache
 /package/temp-deb-*

Makefile

@@ -11,7 +11,11 @@ endif
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(shell date -u +'%Y%m%d-%H%M%S')-$(BUILDINFO_TAG)'
 
 all: \
-	victoria-metrics-prod
+	victoria-metrics-prod \
+	vmagent-prod \
+	vmalert-prot \
+	vmbackup-prod \
+	vmrestore-prod
 
 include app/*/Makefile
 include deployment/*/Makefile
@@ -21,15 +25,21 @@ clean:
 
 publish: \
 	publish-victoria-metrics \
+	publish-vmagent \
+	publish-vmalert \
 	publish-vmbackup \
 	publish-vmrestore
 
 package: \
 	package-victoria-metrics \
+	package-vmagent \
+	package-vmalert \
 	package-vmbackup \
 	package-vmrestore
 
 vmutils: \
+	vmagent \
+	vmalert \
 	vmbackup \
 	vmrestore
 
@@ -42,9 +52,11 @@ release-victoria-metrics: victoria-metrics-prod
 		sha256sum victoria-metrics-$(PKG_TAG).tar.gz > victoria-metrics-$(PKG_TAG)_checksums.txt
 
 release-vmutils: \
+	vmagent-prod \
+	vmalert-prod \
 	vmbackup-prod \
 	vmrestore-prod
-	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmbackup-prod vmrestore-prod && \
+	cd bin && tar czf vmutils-$(PKG_TAG).tar.gz vmagent-prod vmalert-prod vmbackup-prod vmrestore-prod && \
 		sha256sum vmutils-$(PKG_TAG).tar.gz > vmutils-$(PKG_TAG)_checksums.txt
 
 pprof-cpu:
@@ -70,8 +82,11 @@ errcheck: install-errcheck
 	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
 	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
 
 install-errcheck:
 	which errcheck || GO111MODULE=off go get -u github.com/kisielk/errcheck
@@ -79,16 +94,19 @@ install-errcheck:
 check-all: fmt vet lint errcheck golangci-lint
 
 test:
-	GO111MODULE=on go test -tags=integration -mod=vendor ./lib/... ./app/...
+	GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
+
+test-race:
+	GO111MODULE=on go test -mod=vendor -race ./lib/... ./app/...
 
 test-pure:
-	GO111MODULE=on CGO_ENABLED=0 go test -tags=integration -mod=vendor ./lib/... ./app/...
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
 
 test-full:
-	GO111MODULE=on go test -tags=integration -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GO111MODULE=on GOARCH=386 go test -tags=integration -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GO111MODULE=on GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
 	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...
@@ -118,7 +136,7 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	golangci-lint run --exclude '(SA4003|SA1019):' -D errcheck -D structcheck
+	golangci-lint run --exclude '(SA4003|SA1019):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
 	which golangci-lint || GO111MODULE=off go get -u github.com/golangci/golangci-lint/cmd/golangci-lint

app/victoria-metrics/Makefile

@@ -3,6 +3,9 @@
 victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) app-local
 
+victoria-metrics-race:
+	APP_NAME=victoria-metrics RACE=-race $(MAKE) app-local
+
 victoria-metrics-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker
 

app/victoria-metrics/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG certs_image
-FROM $certs_image AS certs
-FROM scratch
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG base_image
+FROM $base_image
+
+EXPOSE 8428
+
+ENTRYPOINT ["/victoria-metrics-prod"]
 ARG src_binary
 COPY $src_binary ./victoria-metrics-prod
-EXPOSE 8428
-ENTRYPOINT ["/victoria-metrics-prod"]

app/victoria-metrics/main.go

@@ -9,44 +9,55 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
-var httpListenAddr = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
+var (
+	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
+	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+
+		"This may be useful for reducing overhead when multiple identically configured Prometheus instances write data to the same VictoriaMetrics. "+
+		"Deduplication is disabled if the -dedup.minScrapeInterval is 0")
+)
 
 func main() {
-	flag.Parse()
+	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
+	storage.SetMinScrapeIntervalForDeduplication(*minScrapeInterval)
 	vmstorage.Init()
 	vmselect.Init()
 	vminsert.Init()
+	startSelfScraper()
 
 	go httpserver.Serve(*httpListenAddr, requestHandler)
-	logger.Infof("started VictoriaMetrics in %s", time.Since(startTime))
+	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
 
+	stopSelfScraper()
+
 	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
 	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
-	logger.Infof("successfully shut down the webservice in %s", time.Since(startTime))
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	vmstorage.Stop()
 	vmselect.Stop()
 
 	fs.MustStopDirRemover()
 
-	logger.Infof("the VictoriaMetrics has been stopped in %s", time.Since(startTime))
+	logger.Infof("the VictoriaMetrics has been stopped in %.3f seconds", time.Since(startTime).Seconds())
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {

app/victoria-metrics/main_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 package main
 
 import (
@@ -23,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -148,7 +147,7 @@ func setUp() {
 }
 
 func processFlags() {
-	flag.Parse()
+	envflag.Parse()
 	for _, fv := range []struct {
 		flag  string
 		value string
@@ -302,6 +301,9 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 	s := newSuite(t)
 	var tt []test
 	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
 		if filepath.Ext(path) != ".json" {
 			return nil
 		}

app/victoria-metrics/self_scraper.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"flag"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+var (
+	selfScrapeInterval = flag.Duration("selfScrapeInterval", 0, "Interval for self-scraping own metrics at /metrics page")
+	selfScrapeInstance = flag.String("selfScrapeInstance", "self", "Value for 'instance' label, which is added to self-scraped metrics")
+	selfScrapeJob      = flag.String("selfScrapeJob", "victoria-metrics", "Value for 'job' label, which is added to self-scraped metrics")
+)
+
+var selfScraperStopCh chan struct{}
+var selfScraperWG sync.WaitGroup
+
+func startSelfScraper() {
+	selfScraperStopCh = make(chan struct{})
+	selfScraperWG.Add(1)
+	go func() {
+		defer selfScraperWG.Done()
+		selfScraper(*selfScrapeInterval)
+	}()
+}
+
+func stopSelfScraper() {
+	close(selfScraperStopCh)
+	selfScraperWG.Wait()
+}
+
+func selfScraper(scrapeInterval time.Duration) {
+	if scrapeInterval <= 0 {
+		// Self-scrape is disabled.
+		return
+	}
+	logger.Infof("started self-scraping `/metrics` page with interval %.3f seconds", scrapeInterval.Seconds())
+
+	var bb bytesutil.ByteBuffer
+	var rows prometheus.Rows
+	var mrs []storage.MetricRow
+	var labels []prompb.Label
+	t := time.NewTicker(scrapeInterval)
+	var currentTimestamp int64
+	for {
+		select {
+		case <-selfScraperStopCh:
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			currentTimestamp = currentTime.UnixNano() / 1e6
+		}
+		bb.Reset()
+		httpserver.WritePrometheusMetrics(&bb)
+		s := bytesutil.ToUnsafeString(bb.B)
+		rows.Reset()
+		rows.Unmarshal(s)
+		mrs = mrs[:0]
+		for i := range rows.Rows {
+			r := &rows.Rows[i]
+			labels = labels[:0]
+			labels = addLabel(labels, "", r.Metric)
+			labels = addLabel(labels, "job", *selfScrapeJob)
+			labels = addLabel(labels, "instance", *selfScrapeInstance)
+			for j := range r.Tags {
+				t := &r.Tags[j]
+				labels = addLabel(labels, t.Key, t.Value)
+			}
+			if len(mrs) < cap(mrs) {
+				mrs = mrs[:len(mrs)+1]
+			} else {
+				mrs = append(mrs, storage.MetricRow{})
+			}
+			mr := &mrs[len(mrs)-1]
+			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
+			mr.Timestamp = currentTimestamp
+			mr.Value = r.Value
+		}
+		logger.Infof("writing %d rows at timestamp %d", len(mrs), currentTimestamp)
+		vmstorage.AddRows(mrs)
+	}
+}
+
+func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
+	if len(dst) < cap(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, prompb.Label{})
+	}
+	lb := &dst[len(dst)-1]
+	lb.Name = bytesutil.ToUnsafeBytes(key)
+	lb.Value = bytesutil.ToUnsafeBytes(value)
+	return dst
+}

app/victoria-metrics/test/prom_types.go

@@ -1,18 +1,18 @@
-// +build integration
-
-// Source https://github.com/prometheus/prometheus/blob/master/prompb/remote.pb.go . Code is copy pasted and cleaned up
 package test
 
+// Source https://github.com/prometheus/prometheus/blob/master/prompb/remote.pb.go . Code is copy pasted and cleaned up
 import (
 	"encoding/binary"
 	"math"
 	"math/bits"
 )
 
+// WriteRequest is write request
 type WriteRequest struct {
 	Timeseries []TimeSeries `protobuf:"bytes,1,rep,name=timeseries,proto3" json:"timeseries"`
 }
 
+// Size returns m size in bytes after marshaling.
 func (m *WriteRequest) Size() (n int) {
 	if m == nil {
 		return 0
@@ -31,6 +31,7 @@ func sovRemote(x uint64) (n int) {
 	return (bits.Len64(x|1) + 6) / 7
 }
 
+// Marshal marshals m.
 func (m *WriteRequest) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -41,11 +42,13 @@ func (m *WriteRequest) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA
 func (m *WriteRequest) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *WriteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	if len(m.Timeseries) > 0 {
@@ -77,11 +80,13 @@ func encodeVarintRemote(dAtA []byte, offset int, v uint64) int {
 	return base
 }
 
+// Sample is time series sample.
 type Sample struct {
 	Value     float64 `protobuf:"fixed64,1,opt,name=value,proto3" json:"value,omitempty"`
 	Timestamp int64   `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 }
 
+// Reset resets m.
 func (m *Sample) Reset() { *m = Sample{} }
 
 // TimeSeries represents samples and labels for a single time series.
@@ -90,21 +95,27 @@ type TimeSeries struct {
 	Samples []Sample `protobuf:"bytes,2,rep,name=samples,proto3" json:"samples"`
 }
 
+// Reset resets m.
 func (m *TimeSeries) Reset() { *m = TimeSeries{} }
 
+// Label is time series label.
 type Label struct {
 	Name  string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
 }
 
+// Reset resets m.
 func (m *Label) Reset() { *m = Label{} }
 
+// Labels is a set of labels.
 type Labels struct {
 	Labels []Label `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"`
 }
 
+// Reset resets m.
 func (m *Labels) Reset() { *m = Labels{} }
 
+// Marshal marshals m.
 func (m *Sample) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -115,11 +126,13 @@ func (m *Sample) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA.
 func (m *Sample) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *Sample) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	if m.Timestamp != 0 {
@@ -136,6 +149,7 @@ func (m *Sample) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+// Marshal marshals m.
 func (m *TimeSeries) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -146,11 +160,13 @@ func (m *TimeSeries) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA.
 func (m *TimeSeries) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *TimeSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	if len(m.Samples) > 0 {
@@ -184,6 +200,7 @@ func (m *TimeSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+// Marshal marshals m.
 func (m *Label) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -194,11 +211,13 @@ func (m *Label) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA.
 func (m *Label) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *Label) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	_ = i
@@ -221,6 +240,7 @@ func (m *Label) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+// Marshal marshals m.
 func (m *Labels) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -231,11 +251,13 @@ func (m *Labels) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA.
 func (m *Labels) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *Labels) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	if len(m.Labels) > 0 {
@@ -267,6 +289,7 @@ func encodeVarintTypes(dAtA []byte, offset int, v uint64) int {
 	return base
 }
 
+// Size returns the size of marshaled m.
 func (m *Sample) Size() (n int) {
 	if m == nil {
 		return 0
@@ -280,6 +303,7 @@ func (m *Sample) Size() (n int) {
 	return n
 }
 
+// Size returns the size of marshaled m.
 func (m *TimeSeries) Size() (n int) {
 	if m == nil {
 		return 0
@@ -301,6 +325,7 @@ func (m *TimeSeries) Size() (n int) {
 	return n
 }
 
+// Size returns the size of marshaled m.
 func (m *Label) Size() (n int) {
 	if m == nil {
 		return 0
@@ -318,6 +343,7 @@ func (m *Label) Size() (n int) {
 	return n
 }
 
+// Size returns the size of marshaled m.
 func (m *Labels) Size() (n int) {
 	if m == nil {
 		return 0

app/victoria-metrics/test/prom_writter.go

@@ -1,9 +1,8 @@
-// +build integration
-
 package test
 
 import "github.com/golang/snappy"
 
+// Compress marshals and compresses wr.
 func Compress(wr WriteRequest) ([]byte, error) {
 	data, err := wr.Marshal()
 	if err != nil {

app/victoria-metrics/testdata/graphite/empty-label-match.json

@@ -0,0 +1,16 @@
+{
+  "name": "empty-label-match",
+  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/395",
+  "data": [
+    "empty_label_match 1 {TIME_S-1m}",
+    "empty_label_match;foo=bar 2 {TIME_S-1m}",
+    "empty_label_match;foo=baz 3 {TIME_S-1m}"],
+  "query": ["/api/v1/query_range?query=empty_label_match{foo=~'bar|'}&start={TIME_S}&end={TIME_S}&step=60"],
+  "result_query_range": {
+    "status":"success",
+    "data":{"resultType":"matrix",
+      "result":[
+	      {"metric":{"__name__":"empty_label_match"},"values":[["{TIME_S}","1"]]},
+	      {"metric":{"__name__":"empty_label_match","foo":"bar"},"values":[["{TIME_S}","2"]]}
+      ]}}
+}

app/vmagent/Makefile

@@ -0,0 +1,76 @@
+# All these commands must run from repository root.
+
+vmagent:
+	APP_NAME=vmagent $(MAKE) app-local
+
+vmagent-race:
+	APP_NAME=vmagent RACE=-race $(MAKE) app-local
+
+vmagent-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker
+
+vmagent-pure-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-pure
+
+vmagent-amd64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-amd64
+
+vmagent-arm-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm
+
+vmagent-arm64-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-arm64
+
+vmagent-ppc64le-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-ppc64le
+
+vmagent-386-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-386
+
+package-vmagent:
+	APP_NAME=vmagent $(MAKE) package-via-docker
+
+package-vmagent-pure:
+	APP_NAME=vmagent $(MAKE) package-via-docker-pure
+
+package-vmagent-amd64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-amd64
+
+package-vmagent-arm:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm
+
+package-vmagent-arm64:
+	APP_NAME=vmagent $(MAKE) package-via-docker-arm64
+
+package-vmagent-ppc64le:
+	APP_NAME=vmagent $(MAKE) package-via-docker-ppc64le
+
+package-vmagent-386:
+	APP_NAME=vmagent $(MAKE) package-via-docker-386
+
+publish-vmagent:
+	APP_NAME=vmagent $(MAKE) publish-via-docker
+
+run-vmagent:
+	mkdir -p vmagent-data
+	DOCKER_OPTS='-v $(shell pwd)/vmagent-data:/vmagent-data' \
+	APP_NAME=vmagent \
+	$(MAKE) run-via-docker
+
+vmagent-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-amd64 ./app/vmagent
+
+vmagent-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm ./app/vmagent
+
+vmagent-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-arm64 ./app/vmagent
+
+vmagent-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-ppc64le ./app/vmagent
+
+vmagent-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmagent-386 ./app/vmagent
+
+vmagent-pure:
+	APP_NAME=vmagent $(MAKE) app-local-pure

app/vmagent/README.md

@@ -0,0 +1,241 @@
+## vmagent
+
+`vmagent` is a tiny but brave agent, which helps you collecting metrics from various sources
+and storing them to [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
+or any other Prometheus-compatible storage system that supports `remote_write` protocol.
+
+<img alt="vmagent" src="vmagent.png">
+
+
+### Motivation
+
+While VictoriaMetrics provides an efficient solution to store and observe metrics, our users needed something fast
+and RAM friendly to scrape metrics from Prometheus-compatible exporters to VictoriaMetrics.
+Also, we found that users’ infrastructure is like snowflakes - never alike, and we decided to add more flexibility
+to `vmagent` (like the ability to push metrics instead of pulling them). We did our best and plan to do even more.
+
+
+### Features
+
+* Can be used as drop-in replacement for Prometheus for scraping targets such as [node_exporter](https://github.com/prometheus/node_exporter).
+  See [Quick Start](#quick-start) for details.
+* Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
+* Accepts data via all the ingestion protocols supported by VictoriaMetrics:
+  * Influx line protocol via `http://<vmagent>:8429/write`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
+  * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
+  * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-opentsdb-compatible-agents).
+  * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
+  * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-time-series-data).
+  * Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-import-csv-data).
+* Can replicate collected metrics simultaneously to multiple remote storage systems.
+* Works in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
+  are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as connection
+  to remote storage is recovered. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
+* Uses lower amounts of RAM, CPU, disk IO and network bandwidth comparing to Prometheus.
+
+
+### Quick Start
+
+Just download `vmutils-*` archive from [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases), unpack it
+and pass the following flags to `vmagent` binary in order to start scraping Prometheus targets:
+
+* `-promscrape.config` with the path to Prometheus config file (it is usually located at `/etc/prometheus/prometheus.yml`)
+* `-remoteWrite.url` with the remote storage endpoint such as VictoriaMetrics. Multiple `-remoteWrite.url` args can be set in parallel
+  in order to replicate data concurrently to multiple remote storage systems.
+
+Example command line:
+
+```
+/path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+If you need collecting only Influx data, then the following command line would be enough:
+
+```
+/path/to/vmagent -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
+```
+
+Then send Influx data to `http://vmagent-host:8429`. See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for more details.
+
+`vmagent` is also available in [docker images](https://hub.docker.com/r/victoriametrics/vmagent/).
+
+Pass `-help` to `vmagent` in order to see the full list of supported command-line flags with their descriptions.
+
+
+### Use cases
+
+
+#### IoT and Edge monitoring
+
+`vmagent` can run and collect metrics in IoT and industrial networks with unreliable or scheduled connections to the remote storage.
+It buffers the collected data in local files until the connection to remote storage becomes available and then sends the buffered
+data to the remote storage. It re-tries sending the data to remote storage on any errors.
+The maximum buffer size can be limited with `-remoteWrite.maxDiskUsagePerURL`.
+
+`vmagent` works on various architectures from IoT world - 32-bit arm, 64-bit arm, ppc64, 386, amd64.
+See [the corresponding Makefile rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmagent/Makefile) for details.
+
+
+#### Drop-in replacement for Prometheus
+
+If you use Prometheus only for scraping metrics from various targets and forwarding these metrics to remote storage,
+then `vmagent` can replace such Prometheus setup. Usually `vmagent` requires lower amounts of RAM, CPU and network bandwidth comparing to Prometheus for such setup.
+See [these docs](#how-to-collect-metrics-in-prometheus-format) for details.
+
+
+#### Replication and high availability
+
+`vmagent` replicates the collected metrics among multiple remote storage instances configured via `-remoteWrite.url` args.
+If a single remote storage instance temporarily goes out of service, then the collected data remains available in another remote storage instances.
+`vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
+Then it sends the buffered data to the remote storage in order to prevent data gaps in the remote storage.
+
+
+#### Relabeling and filtering
+
+`vmagent` can add, remove or update labels on the collected data before sending it to remote storage. Additionally,
+it can remove unneeded samples via Prometheus-like relabeling before sending the collected data to remote storage.
+See [these docs](#relabeling) for details.
+
+
+#### Splitting data streams among multiple systems
+
+`vmagent` supports splitting of the collected data among muliple destinations with the help of `-remoteWrite.urlRelabelConfig`,
+which is applied independently for each configured `-remoteWrite.url` destination. For instance, it is possible to replicate or split
+data among long-term remote storage, short-term remote storage and real-time analytical system [built on top of Kafka](https://github.com/Telefonica/prometheus-kafka-adapter).
+Note that each destination can receive its own subset of the collected data thanks to per-destination relabeling via `-remoteWrite.urlRelabelConfig`.
+
+
+#### Prometheus remote_write proxy
+
+`vmagent` may be used as a proxy for Prometheus data sent via Prometheus `remote_write` protocol. It can accept data via `remote_write` API
+at `/api/v1/write` endpoint, apply relabeling and filtering and then proxy it to another `remote_write` systems.
+The `vmagent` can be configured to encrypt the incoming `remote_write` requests with `-tls*` command-line flags.
+Additionally, Basic Auth can be enabled for the incoming `remote_write` requests with `-httpAuth.*` command-line flags.
+
+
+
+### How to collect metrics in Prometheus format
+
+Pass the path to `prometheus.yml` to `-promscrape.config` command-line flag. `vmagent` takes into account the following
+sections from [Prometheus config file](https://prometheus.io/docs/prometheus/latest/configuration/configuration/):
+
+* `global`
+* `scrape_configs`
+
+All the other sections are ignored, including [remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) section.
+Use `-remoteWrite.*` command-line flags instead for configuring remote write settings.
+
+The following scrape types in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) section are supported:
+
+* `static_configs` - for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details.
+* `file_sd_configs` - for scraping targets defined in external files aka file-based service discover.
+  See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details.
+* `kubernetes_sd_configs` - for scraping targets in Kubernetes (k8s).
+  See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
+* `ec2_sd_configs` - for scraping targets in Amazone EC2.
+  See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details.
+  `vmagent` doesn't support `role_arn` config param yet.
+* `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
+  See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
+  `vmagent` provides the following additional functionality `gce_sd_config`:
+  * if `project` arg is missing, then `vmagent` uses the project for the instance where it runs;
+  * if `zone` arg is missing, then `vmagent` uses the zone for the instance where it runs;
+  * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
+  * `zone` may contain arbitrary number of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
+
+The following service discovery mechanisms will be added to `vmagent` soon:
+
+* [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)
+* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config)
+
+
+File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
+
+
+### Adding labels to metrics
+
+Labels can be added to metrics via the following mechanisms:
+
+* Via `global -> external_labels` section in `-promscrape.config` file. These labels are added only to metrics scraped from targets configured in `-promscrape.config` file.
+* Via `-remoteWrite.label` command-line flag. These labels are added to all the collected metrics before sending them to `-remoteWrite.url`.
+
+
+### Relabeling
+
+`vmagent` supports [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config).
+Additionally it provides the following extra actions:
+
+* `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`.
+* `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`.
+
+The relabeling can be defined in the following places:
+
+* At `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to targets when parsing the file during `vmagent` startup
+  or during config reload after sending `SIGHUP` signal to `vmagent`  via `kill -HUP`.
+* At `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to metrics after each scrape for the configured targets.
+* At `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage.
+* At `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`.
+
+Read more about relabeling in the following articles:
+
+* [Life of a label](https://www.robustperception.io/life-of-a-label)
+* [Discarding targets and timeseries with relabeling](https://www.robustperception.io/relabelling-can-discard-targets-timeseries-and-alerts)
+* [Dropping labels at scrape time](https://www.robustperception.io/dropping-metrics-at-scrape-time-with-prometheus)
+* [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names)
+* [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs)
+
+
+### Monitoring
+
+`vmagent` exports various metrics in Prometheus exposition format at `http://vmagent-host:8429/metrics` page. It is recommended setting up regular scraping of this page
+either via `vmagent` itself or via Prometheus, so the exported metrics could be analyzed later.
+
+`vmagent` also exports target statuses at `http://vmagent-host:8429/targets` page in plaintext format. This page also exports information on improperly configured scrape configs.
+
+
+### Troubleshooting
+
+* It is recommended increasing the maximum number of open files in the system (`ulimit -n`) when scraping big number of targets,
+  since `vmagent` establishes at least a single TCP connection per each target.
+
+* When `vmagent` scrapes many unreliable targets, it can flood error log with scrape errors. These errors can be suppressed
+  by passing `-promscrape.suppressScrapeErrors` command-line flag to `vmagent`. The most recent scrape error per each target can be observed at `http://vmagent-host:8429/targets`.
+
+* It is recommended increasing `-remoteWrite.queues` if `vmagent` collects more than 100K samples per second
+  and `vmagent_remotewrite_pending_data_bytes` metric exported by `vmagent` at `/metrics` page constantly grows.
+
+* `vmagent` buffers scraped data at `-remoteWrite.tmpDataPath` directory until it is sent to `-remoteWrite.url`.
+  The directory can grow big when remote storage is unavailable during extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
+  If you don't want sending all the data from the directory to remote storage, just stop `vmagent` and delete the directory.
+
+
+### How to build from sources
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) - `vmagent` is located in `vmutils-*` archives there.
+
+
+#### Development build
+
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
+2. Run `make vmagent` from the root folder of the repository.
+   It builds `vmagent` binary and puts it into the `bin` folder.
+
+#### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+2. Run `make vmagent-prod` from the root folder of the repository.
+   It builds `vmagent-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-vmagent`. It builds `victoriametrics/vmagent:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmagent`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmagent
+```

app/vmagent/common/push_ctx.go

@@ -0,0 +1,70 @@
+package common
+
+import (
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// PushCtx is a context used for populating WriteRequest.
+type PushCtx struct {
+	WriteRequest prompbmarshal.WriteRequest
+
+	// Labels contains flat list of all the labels used in WriteRequest.
+	Labels []prompbmarshal.Label
+
+	// Samples contains flat list of all the samples used in WriteRequest.
+	Samples []prompbmarshal.Sample
+}
+
+// Reset resets ctx.
+func (ctx *PushCtx) Reset() {
+	tss := ctx.WriteRequest.Timeseries
+	for i := range tss {
+		ts := &tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
+
+	labels := ctx.Labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	ctx.Labels = ctx.Labels[:0]
+
+	ctx.Samples = ctx.Samples[:0]
+}
+
+// GetPushCtx returns PushCtx from pool.
+//
+// Call PutPushCtx when the ctx is no longer needed.
+func GetPushCtx() *PushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*PushCtx)
+		}
+		return &PushCtx{}
+	}
+}
+
+// PutPushCtx returns ctx to the pool.
+//
+// ctx mustn't be used after returning to the pool.
+func PutPushCtx(ctx *PushCtx) {
+	ctx.Reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *PushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/csvimport/request_handler.go

@@ -0,0 +1,63 @@
+package csvimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="csvimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="csvimport"}`)
+)
+
+// InsertHandler processes csv data from req.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8429
+
+ENTRYPOINT ["/vmagent-prod"]
+ARG src_binary
+COPY $src_binary ./vmagent-prod

app/vmagent/graphite/request_handler.go

@@ -0,0 +1,65 @@
+package graphite
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="graphite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="graphite"}`)
+)
+
+// InsertHandler processes remote write for graphite plaintext protocol.
+//
+// See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/influx/request_handler.go

@@ -0,0 +1,167 @@
+package influx
+
+import (
+	"flag"
+	"io"
+	"net/http"
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="influx"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="influx"}`)
+)
+
+// InsertHandlerForReader processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
+func InsertHandlerForReader(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, false, "", "", insertRows)
+	})
+}
+
+// InsertHandlerForHTTP processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
+func InsertHandlerForHTTP(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		q := req.URL.Query()
+		precision := q.Get("precision")
+		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+		db := q.Get("db")
+		return parser.ParseStream(req.Body, isGzipped, precision, db, insertRows)
+	})
+}
+
+func insertRows(db string, rows []parser.Row) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.ctx.Labels[:0]
+	samples := ctx.ctx.Samples[:0]
+	commonLabels := ctx.commonLabels[:0]
+	buf := ctx.buf[:0]
+	for i := range rows {
+		r := &rows[i]
+		commonLabels = commonLabels[:0]
+		hasDBLabel := false
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			if tag.Key == "db" {
+				hasDBLabel = true
+			}
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		if len(db) > 0 && !hasDBLabel {
+			commonLabels = append(commonLabels, prompbmarshal.Label{
+				Name:  "db",
+				Value: db,
+			})
+		}
+		ctx.metricGroupBuf = append(ctx.metricGroupBuf[:0], r.Measurement...)
+		skipFieldKey := len(r.Fields) == 1 && *skipSingleField
+		if len(ctx.metricGroupBuf) > 0 && !skipFieldKey {
+			ctx.metricGroupBuf = append(ctx.metricGroupBuf, *measurementFieldSeparator...)
+		}
+		for j := range r.Fields {
+			f := &r.Fields[j]
+			bufLen := len(buf)
+			buf = append(buf, ctx.metricGroupBuf...)
+			if !skipFieldKey {
+				buf = append(buf, f.Key...)
+			}
+			metricGroup := bytesutil.ToUnsafeString(buf[bufLen:])
+			labelsLen := len(labels)
+			labels = append(labels, prompbmarshal.Label{
+				Name:  "__name__",
+				Value: metricGroup,
+			})
+			labels = append(labels, commonLabels...)
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: r.Timestamp,
+				Value:     f.Value,
+			})
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
+				Labels:  labels[labelsLen:],
+				Samples: samples[len(samples)-1:],
+			})
+		}
+		rowsTotal += len(r.Fields)
+	}
+	ctx.buf = buf
+	ctx.ctx.WriteRequest.Timeseries = tssDst
+	ctx.ctx.Labels = labels
+	ctx.ctx.Samples = samples
+	ctx.commonLabels = commonLabels
+	remotewrite.Push(&ctx.ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+
+	return nil
+}
+
+type pushCtx struct {
+	ctx            common.PushCtx
+	commonLabels   []prompbmarshal.Label
+	metricGroupBuf []byte
+	buf            []byte
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.ctx.Reset()
+
+	commonLabels := ctx.commonLabels
+	for i := range commonLabels {
+		label := &commonLabels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+
+	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
+	ctx.buf = ctx.buf[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vmagent/main.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
+	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
+	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
+	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
+		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
+	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
+		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+)
+
+var (
+	influxServer       *influxserver.Server
+	graphiteServer     *graphiteserver.Server
+	opentsdbServer     *opentsdbserver.Server
+	opentsdbhttpServer *opentsdbhttpserver.Server
+)
+
+func main() {
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
+	startTime := time.Now()
+	remotewrite.Init()
+	writeconcurrencylimiter.Init()
+	if len(*influxListenAddr) > 0 {
+		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
+	}
+
+	promscrape.Init(remotewrite.Push)
+
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, requestHandler)
+	}
+	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+
+	startTime = time.Now()
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+	}
+
+	promscrape.Stop()
+
+	if len(*influxListenAddr) > 0 {
+		influxServer.MustStop()
+	}
+	if len(*graphiteListenAddr) > 0 {
+		graphiteServer.MustStop()
+	}
+	if len(*opentsdbListenAddr) > 0 {
+		opentsdbServer.MustStop()
+	}
+	if len(*opentsdbHTTPListenAddr) > 0 {
+		opentsdbhttpServer.MustStop()
+	}
+	remotewrite.Stop()
+
+	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	switch path {
+	case "/api/v1/write":
+		prometheusWriteRequests.Inc()
+		if err := promremotewrite.InsertHandler(r); err != nil {
+			prometheusWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import":
+		vmimportRequests.Inc()
+		if err := vmimport.InsertHandler(r); err != nil {
+			vmimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/write", "/api/v2/write":
+		influxWriteRequests.Inc()
+		if err := influx.InsertHandlerForHTTP(r); err != nil {
+			influxWriteErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/query":
+		// Emulate fake response for influx query.
+		// This is required for TSBS benchmark.
+		influxQueryRequests.Inc()
+		fmt.Fprintf(w, `{"results":[{"series":[{"values":[]}]}]}`)
+		return true
+	case "/targets":
+		promscrapeTargetsRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain")
+		promscrape.WriteHumanReadableTargetsStatus(w)
+		return true
+	}
+	return false
+}
+
+var (
+	prometheusWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/write", protocol="promremotewrite"}`)
+	prometheusWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/write", protocol="promremotewrite"}`)
+
+	vmimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import", protocol="vmimport"}`)
+	vmimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import", protocol="vmimport"}`)
+
+	csvimportRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/import/csv", protocol="csvimport"}`)
+	csvimportErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/api/v1/import/csv", protocol="csvimport"}`)
+
+	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/write", protocol="influx"}`)
+	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/write", protocol="influx"}`)
+
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/query", protocol="influx"}`)
+
+	promscrapeTargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
+)

app/vmagent/opentsdb/request_handler.go

@@ -0,0 +1,65 @@
+package opentsdb
+
+import (
+	"io"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdb"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdb"}`)
+)
+
+// InsertHandler processes remote write for OpenTSDB put protocol.
+//
+// See http://opentsdb.net/docs/build/html/api_telnet/put.html
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/opentsdbhttp/request_handler.go

@@ -0,0 +1,64 @@
+package opentsdbhttp
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentsdbhttp"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentsdbhttp"}`)
+)
+
+// InsertHandler processes HTTP OpenTSDB put requests.
+// See http://opentsdb.net/docs/build/html/api_http/put.html
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: r.Metric,
+		})
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  tag.Key,
+				Value: tag.Value,
+			})
+		}
+		samples = append(samples, prompbmarshal.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[len(samples)-1:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return nil
+}

app/vmagent/promremotewrite/request_handler.go

@@ -0,0 +1,67 @@
+package promremotewrite
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="promremotewrite"}`)
+)
+
+// InsertHandler processes remote write for prometheus.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(timeseries []prompb.TimeSeries) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range timeseries {
+		ts := &timeseries[i]
+		labelsLen := len(labels)
+		for i := range ts.Labels {
+			label := &ts.Labels[i]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(label.Name),
+				Value: bytesutil.ToUnsafeString(label.Value),
+			})
+		}
+		samplesLen := len(samples)
+		for i := range ts.Samples {
+			sample := &ts.Samples[i]
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     sample.Value,
+				Timestamp: sample.Timestamp,
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+		rowsTotal += len(ts.Samples)
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/remotewrite/client.go

@@ -0,0 +1,281 @@
+package remotewrite
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fasthttp"
+)
+
+var (
+	sendTimeout = flag.Duration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to -remoteWrite.url")
+
+	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
+	tlsCertFile           = flag.String("remoteWrite.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url")
+	tlsKeyFile            = flag.String("remoteWrite.tlsKeyFile", "", "Optional path to client-side TLS certificate key to use when connecting to -remoteWrite.url")
+	tlsCAFile             = flag.String("remoteWrite.tlsCAFile", "", "Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. "+
+		"By default system CA is used")
+
+	basicAuthUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username to use for -remoteWrite.url")
+	basicAuthPassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password to use for -remoteWrite.url")
+	bearerToken       = flag.String("remoteWrite.bearerToken", "", "Optional bearer auth token to use for -remoteWrite.url")
+)
+
+type client struct {
+	urlLabelValue  string
+	remoteWriteURL string
+	host           string
+	requestURI     string
+	authHeader     string
+	fq             *persistentqueue.FastQueue
+	hc             *fasthttp.HostClient
+
+	requestDuration *metrics.Histogram
+	requestsOKCount *metrics.Counter
+	errorsCount     *metrics.Counter
+	retriesCount    *metrics.Counter
+
+	wg     sync.WaitGroup
+	stopCh chan struct{}
+}
+
+func newClient(remoteWriteURL, urlLabelValue string, fq *persistentqueue.FastQueue, concurrency int) *client {
+	authHeader := ""
+	if len(*basicAuthUsername) > 0 || len(*basicAuthPassword) > 0 {
+		// See https://en.wikipedia.org/wiki/Basic_access_authentication
+		token := *basicAuthUsername + ":" + *basicAuthPassword
+		token64 := base64.StdEncoding.EncodeToString([]byte(token))
+		authHeader = "Basic " + token64
+	}
+	if len(*bearerToken) > 0 {
+		if authHeader != "" {
+			logger.Panicf("FATAL: `-remoteWrite.bearerToken`=%q cannot be set when `-remoteWrite.basicAuth.*` flags are set", *bearerToken)
+		}
+		authHeader = "Bearer " + *bearerToken
+	}
+
+	readTimeout := *sendTimeout
+	if readTimeout <= 0 {
+		readTimeout = time.Minute
+	}
+	writeTimeout := readTimeout
+	var u fasthttp.URI
+	u.Update(remoteWriteURL)
+	scheme := string(u.Scheme())
+	switch scheme {
+	case "http", "https":
+	default:
+		logger.Panicf("FATAL: unsupported scheme in -remoteWrite.url=%q: %q. It must be http or https", remoteWriteURL, scheme)
+	}
+	host := string(u.Host())
+	if len(host) == 0 {
+		logger.Panicf("FATAL: invalid -remoteWrite.url=%q: host cannot be empty. Make sure the url looks like `http://host:port/path`", remoteWriteURL)
+	}
+	requestURI := string(u.RequestURI())
+	isTLS := scheme == "https"
+	var tlsCfg *tls.Config
+	if isTLS {
+		var err error
+		tlsCfg, err = getTLSConfig()
+		if err != nil {
+			logger.Panicf("FATAL: cannot initialize TLS config: %s", err)
+		}
+	}
+	if !strings.Contains(host, ":") {
+		if isTLS {
+			host += ":443"
+		} else {
+			host += ":80"
+		}
+	}
+	maxConns := 2 * concurrency
+	hc := &fasthttp.HostClient{
+		Addr:                host,
+		Name:                "vmagent",
+		Dial:                statDial,
+		DialDualStack:       netutil.TCP6Enabled(),
+		IsTLS:               isTLS,
+		TLSConfig:           tlsCfg,
+		MaxConns:            maxConns,
+		MaxIdleConnDuration: 10 * readTimeout,
+		ReadTimeout:         readTimeout,
+		WriteTimeout:        writeTimeout,
+		MaxResponseBodySize: 1024 * 1024,
+	}
+	c := &client{
+		urlLabelValue:  urlLabelValue,
+		remoteWriteURL: remoteWriteURL,
+		host:           host,
+		requestURI:     requestURI,
+		authHeader:     authHeader,
+		fq:             fq,
+		hc:             hc,
+		stopCh:         make(chan struct{}),
+	}
+	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.urlLabelValue))
+	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.urlLabelValue))
+	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_errors_total{url=%q}`, c.urlLabelValue))
+	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.urlLabelValue))
+	for i := 0; i < concurrency; i++ {
+		c.wg.Add(1)
+		go func() {
+			defer c.wg.Done()
+			c.runWorker()
+		}()
+	}
+	logger.Infof("initialized client for -remoteWrite.url=%q", c.remoteWriteURL)
+	return c
+}
+
+func (c *client) MustStop() {
+	close(c.stopCh)
+	c.wg.Wait()
+	logger.Infof("stopped client for -remoteWrite.url=%q", c.remoteWriteURL)
+}
+
+func getTLSConfig() (*tls.Config, error) {
+	var tlsRootCA *x509.CertPool
+	var tlsCertificate *tls.Certificate
+	if *tlsCertFile != "" || *tlsKeyFile != "" {
+		cert, err := tls.LoadX509KeyPair(*tlsCertFile, *tlsKeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load TLS certificate for -remoteWrite.tlsCertFile=%q and -remoteWrite.tlsKeyFile=%q: %s", *tlsCertFile, *tlsKeyFile, err)
+		}
+		tlsCertificate = &cert
+	}
+	if *tlsCAFile != "" {
+		data, err := ioutil.ReadFile(*tlsCAFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read -remoteWrite.tlsCAFile=%q: %s", *tlsCAFile, err)
+		}
+		tlsRootCA = x509.NewCertPool()
+		if !tlsRootCA.AppendCertsFromPEM(data) {
+			return nil, fmt.Errorf("cannot parse data -remoteWrite.tlsCAFile=%q", *tlsCAFile)
+		}
+	}
+	tlsCfg := &tls.Config{
+		RootCAs:            tlsRootCA,
+		ClientSessionCache: tls.NewLRUClientSessionCache(0),
+	}
+	if tlsCertificate != nil {
+		tlsCfg.Certificates = []tls.Certificate{*tlsCertificate}
+	}
+	tlsCfg.InsecureSkipVerify = *tlsInsecureSkipVerify
+	return tlsCfg, nil
+}
+
+func (c *client) runWorker() {
+	var ok bool
+	var block []byte
+	ch := make(chan struct{})
+	for {
+		block, ok = c.fq.MustReadBlock(block[:0])
+		if !ok {
+			return
+		}
+		go func() {
+			c.sendBlock(block)
+			ch <- struct{}{}
+		}()
+		select {
+		case <-ch:
+			// The block has been sent successfully
+			continue
+		case <-c.stopCh:
+			// c must be stopped. Wait for a while in the hope the block will be sent.
+			graceDuration := 5 * time.Second
+			select {
+			case <-ch:
+				// The block has been sent successfully.
+			case <-time.After(graceDuration):
+				logger.Errorf("couldn't sent block with size %d bytes to %q in %.3f seconds during shutdown; dropping it",
+					len(block), c.remoteWriteURL, graceDuration.Seconds())
+			}
+			return
+		}
+	}
+}
+
+func (c *client) sendBlock(block []byte) {
+	req := fasthttp.AcquireRequest()
+	req.SetRequestURI(c.requestURI)
+	req.SetHost(c.host)
+	req.Header.SetMethod("POST")
+	req.Header.Add("Content-Type", "application/x-protobuf")
+	req.Header.Add("Content-Encoding", "snappy")
+	req.Header.Add("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if c.authHeader != "" {
+		req.Header.Set("Authorization", c.authHeader)
+	}
+	req.SetBody(block)
+
+	retryDuration := time.Second
+	resp := fasthttp.AcquireResponse()
+
+again:
+	select {
+	case <-c.stopCh:
+		fasthttp.ReleaseRequest(req)
+		fasthttp.ReleaseResponse(resp)
+		return
+	default:
+	}
+
+	startTime := time.Now()
+	err := doRequestWithPossibleRetry(c.hc, req, resp)
+	c.requestDuration.UpdateDuration(startTime)
+	if err != nil {
+		c.errorsCount.Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
+		}
+		logger.Errorf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, err, retryDuration.Seconds())
+		time.Sleep(retryDuration)
+		c.retriesCount.Inc()
+		goto again
+	}
+	statusCode := resp.StatusCode()
+	if statusCode/100 != 2 {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.urlLabelValue, statusCode)).Inc()
+		retryDuration *= 2
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
+		}
+		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q: %d; response body=%q; re-sending the block in %.3f seconds",
+			len(block), c.remoteWriteURL, statusCode, resp.Body(), retryDuration.Seconds())
+		time.Sleep(retryDuration)
+		c.retriesCount.Inc()
+		goto again
+	}
+	c.requestsOKCount.Inc()
+
+	// The block has been successfully sent to the remote storage.
+	fasthttp.ReleaseResponse(resp)
+	fasthttp.ReleaseRequest(req)
+}
+
+func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response) error {
+	// There is no need in calling DoTimeout, since the timeout must be already set in hc.ReadTimeout.
+	err := hc.Do(req, resp)
+	if err == nil {
+		return nil
+	}
+	if err != fasthttp.ErrConnectionClosed {
+		return err
+	}
+	// Retry request if the server closed the keep-alive connection during the first attempt.
+	return hc.Do(req, resp)
+}

app/vmagent/remotewrite/pendingseries.go

@@ -0,0 +1,199 @@
+package remotewrite
+
+import (
+	"flag"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
+)
+
+var (
+	flushInterval = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
+		"Higher value reduces network bandwidth usage at the cost of delayed push of scraped data to remote storage")
+	maxUnpackedBlockSize = flag.Int("remoteWrite.maxBlockSize", 32*1024*1024, "The maximum size in bytes of unpacked request to send to remote storage. "+
+		"It shouldn't exceed -maxInsertRequestSize from VictoriaMetrics")
+)
+
+// the maximum number of rows to send per each block.
+const maxRowsPerBlock = 10000
+
+type pendingSeries struct {
+	mu sync.Mutex
+	wr writeRequest
+
+	stopCh            chan struct{}
+	periodicFlusherWG sync.WaitGroup
+}
+
+func newPendingSeries(pushBlock func(block []byte)) *pendingSeries {
+	var ps pendingSeries
+	ps.wr.pushBlock = pushBlock
+	ps.stopCh = make(chan struct{})
+	ps.periodicFlusherWG.Add(1)
+	go func() {
+		defer ps.periodicFlusherWG.Done()
+		ps.periodicFlusher()
+	}()
+	return &ps
+}
+
+func (ps *pendingSeries) MustStop() {
+	close(ps.stopCh)
+	ps.periodicFlusherWG.Wait()
+}
+
+func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
+	ps.mu.Lock()
+	ps.wr.push(tss)
+	ps.mu.Unlock()
+}
+
+func (ps *pendingSeries) periodicFlusher() {
+	ticker := time.NewTicker(*flushInterval)
+	defer ticker.Stop()
+	mustStop := false
+	for !mustStop {
+		select {
+		case <-ps.stopCh:
+			mustStop = true
+		case <-ticker.C:
+			if time.Since(ps.wr.lastFlushTime) < *flushInterval/2 {
+				continue
+			}
+		}
+		ps.mu.Lock()
+		ps.wr.flush()
+		ps.mu.Unlock()
+	}
+}
+
+type writeRequest struct {
+	wr            prompbmarshal.WriteRequest
+	pushBlock     func(block []byte)
+	lastFlushTime time.Time
+
+	tss []prompbmarshal.TimeSeries
+
+	labels  []prompbmarshal.Label
+	samples []prompbmarshal.Sample
+	buf     []byte
+}
+
+func (wr *writeRequest) reset() {
+	wr.wr.Timeseries = nil
+
+	for i := range wr.tss {
+		ts := &wr.tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	wr.tss = wr.tss[:0]
+
+	for i := range wr.labels {
+		label := &wr.labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	wr.labels = wr.labels[:0]
+
+	wr.samples = wr.samples[:0]
+	wr.buf = wr.buf[:0]
+}
+
+func (wr *writeRequest) flush() {
+	wr.wr.Timeseries = wr.tss
+	wr.lastFlushTime = time.Now()
+	pushWriteRequest(&wr.wr, wr.pushBlock)
+	wr.reset()
+}
+
+func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
+	tssDst := wr.tss
+	for i := range src {
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		dst := &tssDst[len(tssDst)-1]
+		wr.copyTimeSeries(dst, &src[i])
+		if len(wr.tss) >= maxRowsPerBlock {
+			wr.flush()
+			tssDst = wr.tss
+		}
+	}
+	wr.tss = tssDst
+}
+
+func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
+	labelsDst := wr.labels
+	labelsLen := len(wr.labels)
+	samplesDst := wr.samples
+	buf := wr.buf
+	for i := range src.Labels {
+		labelsDst = append(labelsDst, prompbmarshal.Label{})
+		dstLabel := &labelsDst[len(labelsDst)-1]
+		srcLabel := &src.Labels[i]
+
+		buf = append(buf, srcLabel.Name...)
+		dstLabel.Name = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Name):])
+		buf = append(buf, srcLabel.Value...)
+		dstLabel.Value = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Value):])
+	}
+	dst.Labels = labelsDst[labelsLen:]
+
+	samplesDst = append(samplesDst, prompbmarshal.Sample{})
+	dstSample := &samplesDst[len(samplesDst)-1]
+	if len(src.Samples) != 1 {
+		logger.Panicf("BUG: unexpected number of samples in time series; got %d; want 1", len(src.Samples))
+	}
+	*dstSample = src.Samples[0]
+	dst.Samples = samplesDst[len(samplesDst)-1:]
+
+	wr.samples = samplesDst
+	wr.labels = labelsDst
+	wr.buf = buf
+}
+
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte)) {
+	if len(wr.Timeseries) == 0 {
+		// Nothing to push
+		return
+	}
+	bb := writeRequestBufPool.Get()
+	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
+	if len(bb.B) <= *maxUnpackedBlockSize {
+		zb := snappyBufPool.Get()
+		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+		writeRequestBufPool.Put(bb)
+		if len(zb.B) <= persistentqueue.MaxBlockSize {
+			pushBlock(zb.B)
+			blockSizeRows.Update(float64(len(wr.Timeseries)))
+			blockSizeBytes.Update(float64(len(zb.B)))
+			snappyBufPool.Put(zb)
+			return
+		}
+		snappyBufPool.Put(zb)
+	} else {
+		writeRequestBufPool.Put(bb)
+	}
+
+	// Too big block. Recursively split it into smaller parts.
+	timeseries := wr.Timeseries
+	n := len(timeseries) / 2
+	wr.Timeseries = timeseries[:n]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries[n:]
+	pushWriteRequest(wr, pushBlock)
+	wr.Timeseries = timeseries
+}
+
+var (
+	blockSizeBytes = metrics.NewHistogram(`vmagent_remotewrite_block_size_bytes`)
+	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
+)
+
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/relabel.go

@@ -0,0 +1,113 @@
+package remotewrite
+
+import (
+	"flag"
+	"strings"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+)
+
+var (
+	unparsedLabelsGlobal = flagutil.NewArray("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
+		"Pass multiple -remoteWrite.label flags in order to add multiple flags to metrics before sending them to remote storage")
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+
+		"before sending them to -remoteWrite.url. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config for details")
+)
+
+var labelsGlobal []prompbmarshal.Label
+var prcsGlobal []promrelabel.ParsedRelabelConfig
+
+// initRelabelGlobal must be called after parsing command-line flags.
+func initRelabelGlobal() {
+	// Init labelsGlobal
+	labelsGlobal = nil
+	for _, s := range *unparsedLabelsGlobal {
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Panicf("FATAL: missing '=' in `-remoteWrite.label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labelsGlobal = append(labelsGlobal, prompbmarshal.Label{
+			Name:  s[:n],
+			Value: s[n+1:],
+		})
+	}
+
+	// Init prcsGlobal
+	prcsGlobal = nil
+	if len(*relabelConfigPathGlobal) > 0 {
+		var err error
+		prcsGlobal, err = promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		if err != nil {
+			logger.Panicf("FATAL: cannot load relabel configs from -remoteWrite.relabelConfig=%q: %s", *relabelConfigPathGlobal, err)
+		}
+	}
+}
+
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, prcs []promrelabel.ParsedRelabelConfig) []prompbmarshal.TimeSeries {
+	if len(extraLabels) == 0 && len(prcs) == 0 {
+		// Nothing to change.
+		return tss
+	}
+	tssDst := tss[:0]
+	labels := rctx.labels[:0]
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
+		for j := range extraLabels {
+			extraLabel := &extraLabels[j]
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, *extraLabel)
+			}
+		}
+		labels = promrelabel.ApplyRelabelConfigs(labels, labelsLen, prcs, true)
+		if len(labels) == labelsLen {
+			// Drop the current time series, since relabeling removed all the labels.
+			continue
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: ts.Samples,
+		})
+	}
+	rctx.labels = labels
+	return tssDst
+}
+
+type relabelCtx struct {
+	// pool for labels, which are used during the relabeling.
+	labels []prompbmarshal.Label
+}
+
+func (rctx *relabelCtx) reset() {
+	labels := rctx.labels
+	for i := range labels {
+		label := &labels[i]
+		label.Name = ""
+		label.Value = ""
+	}
+	rctx.labels = rctx.labels[:0]
+}
+
+var relabelCtxPool = &sync.Pool{
+	New: func() interface{} {
+		return &relabelCtx{}
+	},
+}
+
+func getRelabelCtx() *relabelCtx {
+	return relabelCtxPool.Get().(*relabelCtx)
+}
+
+func putRelabelCtx(rctx *relabelCtx) {
+	rctx.labels = rctx.labels[:0]
+	relabelCtxPool.Put(rctx)
+}

app/vmagent/remotewrite/remotewrite.go

@@ -0,0 +1,195 @@
+package remotewrite
+
+import (
+	"flag"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/metrics"
+	xxhash "github.com/cespare/xxhash/v2"
+)
+
+var (
+	remoteWriteURLs = flagutil.NewArray("remoteWrite.url", "Remote storage URL to write data to. It must support Prometheus remote_write API. "+
+		"It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
+		"Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems")
+	relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url")
+	tmpDataPath        = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored")
+	queues             = flag.Int("remoteWrite.queues", 1, "The number of concurrent queues to each -remoteWrite.url. Set more queues if a single queue "+
+		"isn't enough for sending high volume of collected data to remote storage")
+	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensistive auth info")
+	maxPendingBytesPerURL = flag.Int("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
+		"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
+		"Disk usage is unlimited if the value is set to 0")
+)
+
+var rwctxs []*remoteWriteCtx
+
+// Init initializes remotewrite.
+//
+// It must be called after flag.Parse().
+//
+// Stop must be called for graceful shutdown.
+func Init() {
+	if len(*remoteWriteURLs) == 0 {
+		logger.Panicf("FATAL: at least one `-remoteWrite.url` must be set")
+	}
+
+	if !*showRemoteWriteURL {
+		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
+		httpserver.RegisterSecretFlag("remoteWrite.url")
+	}
+	initRelabelGlobal()
+
+	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / maxRowsPerBlock / 100
+	if maxInmemoryBlocks > 200 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 200
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
+	for i, remoteWriteURL := range *remoteWriteURLs {
+		relabelConfigPath := ""
+		if i < len(*relabelConfigPaths) {
+			relabelConfigPath = (*relabelConfigPaths)[i]
+		}
+		urlLabelValue := fmt.Sprintf("secret-url-%d", i+1)
+		if *showRemoteWriteURL {
+			urlLabelValue = remoteWriteURL
+		}
+		rwctx := newRemoteWriteCtx(remoteWriteURL, relabelConfigPath, maxInmemoryBlocks, urlLabelValue)
+		rwctxs = append(rwctxs, rwctx)
+	}
+}
+
+// Stop stops remotewrite.
+//
+// It is expected that nobody calls Push during and after the call to this func.
+func Stop() {
+	for _, rwctx := range rwctxs {
+		rwctx.MustStop()
+	}
+	rwctxs = nil
+}
+
+// Push sends wr to remote storage systems set via `-remoteWrite.url`.
+//
+// Each timeseries in wr.Timeseries must contain one sample.
+func Push(wr *prompbmarshal.WriteRequest) {
+	var rctx *relabelCtx
+	if len(prcsGlobal) > 0 || len(labelsGlobal) > 0 {
+		rctx = getRelabelCtx()
+	}
+	tss := wr.Timeseries
+	for len(tss) > 0 {
+		// Process big tss in smaller blocks in order to reduce maxmimum memory usage
+		tssBlock := tss
+		if len(tssBlock) > maxRowsPerBlock {
+			tssBlock = tss[:maxRowsPerBlock]
+			tss = tss[maxRowsPerBlock:]
+		} else {
+			tss = nil
+		}
+		if rctx != nil {
+			tssBlockLen := len(tssBlock)
+			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, prcsGlobal)
+			globalRelabelMetricsDropped.Add(tssBlockLen - len(tssBlock))
+		}
+		for _, rwctx := range rwctxs {
+			rwctx.Push(tssBlock)
+		}
+		if rctx != nil {
+			rctx.reset()
+		}
+	}
+	if rctx != nil {
+		putRelabelCtx(rctx)
+	}
+}
+
+var globalRelabelMetricsDropped = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
+
+type remoteWriteCtx struct {
+	fq         *persistentqueue.FastQueue
+	c          *client
+	prcs       []promrelabel.ParsedRelabelConfig
+	pss        []*pendingSeries
+	pssNextIdx uint64
+
+	relabelMetricsDropped *metrics.Counter
+}
+
+func newRemoteWriteCtx(remoteWriteURL, relabelConfigPath string, maxInmemoryBlocks int, urlLabelValue string) *remoteWriteCtx {
+	h := xxhash.Sum64([]byte(remoteWriteURL))
+	path := fmt.Sprintf("%s/persistent-queue/%016X", *tmpDataPath, h)
+	fq := persistentqueue.MustOpenFastQueue(path, remoteWriteURL, maxInmemoryBlocks, *maxPendingBytesPerURL)
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+		return float64(fq.GetPendingBytes())
+	})
+	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, path, urlLabelValue), func() float64 {
+		return float64(fq.GetInmemoryQueueLen())
+	})
+	c := newClient(remoteWriteURL, urlLabelValue, fq, *queues)
+	var prcs []promrelabel.ParsedRelabelConfig
+	if len(relabelConfigPath) > 0 {
+		var err error
+		prcs, err = promrelabel.LoadRelabelConfigs(relabelConfigPath)
+		if err != nil {
+			logger.Panicf("FATAL: cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %s", relabelConfigPath, err)
+		}
+	}
+	pss := make([]*pendingSeries, *queues)
+	for i := range pss {
+		pss[i] = newPendingSeries(fq.MustWriteBlock)
+	}
+	return &remoteWriteCtx{
+		fq:   fq,
+		c:    c,
+		prcs: prcs,
+		pss:  pss,
+
+		relabelMetricsDropped: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, path, urlLabelValue)),
+	}
+}
+
+func (rwctx *remoteWriteCtx) MustStop() {
+	for _, ps := range rwctx.pss {
+		ps.MustStop()
+	}
+	rwctx.pss = nil
+	rwctx.fq.MustClose()
+	rwctx.fq = nil
+	rwctx.prcs = nil
+	rwctx.c.MustStop()
+	rwctx.c = nil
+
+	rwctx.relabelMetricsDropped = nil
+}
+
+func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
+	var rctx *relabelCtx
+	if len(rwctx.prcs) > 0 {
+		rctx = getRelabelCtx()
+		tssLen := len(tss)
+		tss = rctx.applyRelabeling(tss, nil, rwctx.prcs)
+		rwctx.relabelMetricsDropped.Add(tssLen - len(tss))
+	}
+	pss := rwctx.pss
+	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	pss[idx].Push(tss)
+	if rctx != nil {
+		putRelabelCtx(rctx)
+	}
+}

app/vmagent/remotewrite/statconn.go

@@ -0,0 +1,71 @@
+package remotewrite
+
+import (
+	"net"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fasthttp"
+)
+
+func statDial(addr string) (net.Conn, error) {
+	conn, err := fasthttp.Dial(addr)
+	dialsTotal.Inc()
+	if err != nil {
+		dialErrors.Inc()
+		return nil, err
+	}
+	conns.Inc()
+	sc := &statConn{
+		Conn: conn,
+	}
+	return sc, nil
+}
+
+var (
+	dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
+	dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
+	conns      = metrics.NewCounter(`vmagent_remotewrite_conns`)
+)
+
+type statConn struct {
+	closed uint64
+	net.Conn
+}
+
+func (sc *statConn) Read(p []byte) (int, error) {
+	n, err := sc.Conn.Read(p)
+	connReadsTotal.Inc()
+	if err != nil {
+		connReadErrors.Inc()
+	}
+	connBytesRead.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Write(p []byte) (int, error) {
+	n, err := sc.Conn.Write(p)
+	connWritesTotal.Inc()
+	if err != nil {
+		connWriteErrors.Inc()
+	}
+	connBytesWritten.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Close() error {
+	err := sc.Conn.Close()
+	if atomic.AddUint64(&sc.closed, 1) == 1 {
+		conns.Dec()
+	}
+	return err
+}
+
+var (
+	connReadsTotal   = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
+	connWritesTotal  = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
+	connReadErrors   = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
+	connWriteErrors  = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
+	connBytesRead    = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
+	connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
+)

app/vmagent/vmimport/request_handler.go

@@ -0,0 +1,70 @@
+package vmimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vmagent_rows_inserted_total{type="vmimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="vmimport"}`)
+)
+
+// InsertHandler processes `/api/v1/import` request.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+		labelsLen := len(labels)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+		values := r.Values
+		timestamps := r.Timestamps
+		_ = timestamps[len(values)-1]
+		samplesLen := len(samples)
+		for j, value := range values {
+			samples = append(samples, prompbmarshal.Sample{
+				Value:     value,
+				Timestamp: timestamps[j],
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+		rowsTotal += len(values)
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(&ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmalert/Makefile

@@ -0,0 +1,78 @@
+# All these commands must run from repository root.
+
+vmalert:
+	APP_NAME=vmalert $(MAKE) app-local
+
+vmalert-race:
+	APP_NAME=vmalert RACE=-race $(MAKE) app-local
+
+vmalert-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker
+
+vmalert-pure-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-pure
+
+vmalert-amd64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-amd64
+
+vmalert-arm-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-arm
+
+vmalert-arm64-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-arm64
+
+vmalert-ppc64le-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-ppc64le
+
+vmalert-386-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-386
+
+package-vmalert:
+	APP_NAME=vmalert $(MAKE) package-via-docker
+
+package-vmalert-pure:
+	APP_NAME=vmalert $(MAKE) package-via-docker-pure
+
+package-vmalert-amd64:
+	APP_NAME=vmalert $(MAKE) package-via-docker-amd64
+
+package-vmalert-arm:
+	APP_NAME=vmalert $(MAKE) package-via-docker-arm
+
+package-vmalert-arm64:
+	APP_NAME=vmalert $(MAKE) package-via-docker-arm64
+
+package-vmalert-ppc64le:
+	APP_NAME=vmalert $(MAKE) package-via-docker-ppc64le
+
+package-vmalert-386:
+	APP_NAME=vmalert $(MAKE) package-via-docker-386
+
+publish-vmalert:
+	APP_NAME=vmalert $(MAKE) publish-via-docker
+
+test-vmalert:
+	go test -race -cover ./app/vmalert
+
+run-vmalert: vmalert
+	./bin/vmalert -rule=app/vmalert/testdata/rules0-good.rules \
+		-datasource.url=http://localhost:8428 -notifier.url=http://localhost:9093 \
+		-evaluationInterval=3s
+
+vmalert-amd64:
+	CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-amd64 ./app/vmalert
+
+vmalert-arm:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm ./app/vmalert
+
+vmalert-arm64:
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-arm64 ./app/vmalert
+
+vmalert-ppc64le:
+	CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-ppc64le ./app/vmalert
+
+vmalert-386:
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 GO111MODULE=on go build -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/vmalert-386 ./app/vmalert
+
+vmalert-pure:
+	APP_NAME=vmalert $(MAKE) app-local-pure

app/vmalert/README.md

@@ -0,0 +1,95 @@
+## VM Alert
+
+`vmalert` executes a list of given MetricsQL expressions (rules) and
+sends alerts to [Alert Manager](https://github.com/prometheus/alertmanager).   
+
+NOTE: `vmalert` is in early alpha and wasn't tested in production systems yet.
+
+### Features:
+* Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB;
+* VictoriaMetrics [MetricsQL](https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/MetricsQL)
+ expressions validation;
+* Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
+ support;
+* Integration with [Alertmanager](https://github.com/prometheus/alertmanager);
+* Lightweight without extra dependencies.
+
+### TODO:
+* Persist alerts state as timeseries in TSDB. Currently, alerts state is stored
+in process memory only and will be lost on restart;
+* Configuration hot reload.
+
+### QuickStart
+
+To build `vmalert` from sources:
+```
+git clone https://github.com/VictoriaMetrics/VictoriaMetrics
+cd VictoriaMetrics
+make vmalert
+```
+The build binary will be placed to `VictoriaMetrics/bin` folder.
+
+To start using `vmalert` you will need the following things:
+* list of alert rules - PromQL/MetricsQL expressions to execute;
+* datasource address - reachable VictoriaMetrics instance for rules execution;
+* notifier address - reachable Alertmanager instance for processing, 
+aggregating alerts and sending notifications.
+
+Then configure `vmalert` accordingly:
+```
+./bin/vmalert -rule=alert.rules \
+		-datasource.url=http://localhost:8428 \
+        -notifier.url=http://localhost:9093
+```
+
+Example for `.rules` file may be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/testdata/rules0-good.rules)
+
+`vmalert` runs evaluation for every group in a separate goroutine.
+Rules in group evaluated one-by-one sequentially. 
+
+`vmalert` also runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
+* `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
+* `http://<vmalert-addr>/api/v1/<groupName>/<alertID>/status" ` - get alert status by ID.
+Used as alert source in AlertManager.
+* `http://<vmalert-addr>/metrics` - application metrics.
+
+### Configuration
+
+The shortlist of configuration flags is the following:
+```
+Usage of vmalert:
+  -datasource.url string
+        Victoria Metrics or VMSelect url. Required parameter. e.g. http://127.0.0.1:8428
+  -datasource.basicAuth.password string
+        Optional basic auth password to use for -datasource.url
+  -datasource.basicAuth.username string
+        Optional basic auth username to use for -datasource.url
+  -evaluationInterval duration
+        How often to evaluate the rules. Default 1m (default 1m0s)
+  -external.url string
+        External URL is used as alert's source for sent alerts to the notifier
+  -httpListenAddr string
+        Address to listen for http connections (default ":8880")
+  -notifier.url string
+        Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093
+  -remotewrite.url string
+        Optional URL to remote-write compatible storage where to write timeseriesbased on active alerts. E.g. http://127.0.0.1:8428
+  -rule value
+        Path to the file with alert rules. 
+        Supports patterns. Flag can be specified multiple times. 
+        Examples:
+         -rule /path/to/file. Path to a single file with alerting rules
+         -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+        absolute path to all .yaml files in root.
+  -rule.validateTemplates
+        Indicates to validate annotation and label templates (default true)
+```
+
+Pass `-help` to `vmalert` in order to see the full list of supported 
+command-line flags with their descriptions.
+
+### Contributing
+
+`vmalert` is mostly designed and built by VictoriaMetrics community.
+Feel free to share your experience and ideas for improving this 
+software. Please keep simplicity as the main priority.

app/vmalert/config.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+// Parse parses rule configs from given file patterns
+func Parse(pathPatterns []string, validateAnnotations bool) ([]Group, error) {
+	var fp []string
+	for _, pattern := range pathPatterns {
+		matches, err := filepath.Glob(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("error reading file patther %s:%v", pattern, err)
+		}
+		fp = append(fp, matches...)
+	}
+	var groups []Group
+	for _, file := range fp {
+		groupsNames := map[string]struct{}{}
+		gr, err := parseFile(file)
+		if err != nil {
+			return nil, fmt.Errorf("file %s: %w", file, err)
+		}
+		for _, group := range gr {
+			if _, ok := groupsNames[group.Name]; ok {
+				return nil, fmt.Errorf("one file can not contain groups with the same name %s, filepath:%s", file, group.Name)
+			}
+			groupsNames[group.Name] = struct{}{}
+			for _, rule := range group.Rules {
+				if err = rule.Validate(); err != nil {
+					return nil, fmt.Errorf("invalid rule filepath:%s, group %s:%w", file, group.Name, err)
+				}
+				// TODO: this init looks weird here
+				rule.alerts = make(map[uint64]*notifier.Alert)
+				if validateAnnotations {
+					if err = notifier.ValidateTemplates(rule.Annotations); err != nil {
+						return nil, fmt.Errorf("invalid annotations filepath:%s, group %s:%w", file, group.Name, err)
+					}
+					if err = notifier.ValidateTemplates(rule.Labels); err != nil {
+						return nil, fmt.Errorf("invalid labels filepath:%s, group %s:%w", file, group.Name, err)
+					}
+				}
+				rule.group = &group
+			}
+		}
+		groups = append(groups, gr...)
+	}
+	if len(groups) < 1 {
+		return nil, fmt.Errorf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
+	return groups, nil
+}
+
+func parseFile(path string) ([]Group, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading alert rule file: %w", err)
+	}
+	g := struct {
+		Groups []Group `yaml:"groups"`
+	}{}
+	err = yaml.Unmarshal(data, &g)
+	return g.Groups, err
+}

app/vmalert/config_test.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"net/url"
+	"os"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+func TestMain(m *testing.M) {
+	u, _ := url.Parse("https://victoriametrics.com/path")
+	notifier.InitTemplateFunc(u)
+	os.Exit(m.Run())
+}
+
+func TestParseGood(t *testing.T) {
+	if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true); err != nil {
+		t.Errorf("error parsing files %s", err)
+	}
+}
+
+func TestParseBad(t *testing.T) {
+	if _, err := Parse([]string{"testdata/rules0-bad.rules"}, true); err == nil {
+		t.Errorf("expected syntaxt error")
+	}
+	if _, err := Parse([]string{"testdata/dir/rules0-bad.rules"}, true); err == nil {
+		t.Errorf("expected template annotation error")
+	}
+	if _, err := Parse([]string{"testdata/dir/rules1-bad.rules"}, true); err == nil {
+		t.Errorf("expected same group error")
+	}
+	if _, err := Parse([]string{"testdata/dir/rules2-bad.rules"}, true); err == nil {
+		t.Errorf("expected template label error")
+	}
+	if _, err := Parse([]string{"testdata/*.yaml"}, true); err == nil {
+		t.Errorf("expected empty group")
+	}
+}

app/vmalert/datasource/datasource.go

@@ -0,0 +1,24 @@
+package datasource
+
+import "context"
+
+// Querier interface wraps Query method which
+// executes given query and returns list of Metrics
+// as result
+type Querier interface {
+	Query(ctx context.Context, query string) ([]Metric, error)
+}
+
+// Metric is the basic entity which should be return by datasource
+// It represents single data point with full list of labels
+type Metric struct {
+	Labels    []Label
+	Timestamp int64
+	Value     float64
+}
+
+// Label represents metric's label
+type Label struct {
+	Name  string
+	Value string
+}

app/vmalert/datasource/vm.go

@@ -0,0 +1,103 @@
+package datasource
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+type response struct {
+	Status string `json:"status"`
+	Data   struct {
+		ResultType string `json:"resultType"`
+		Result     []struct {
+			Labels map[string]string `json:"metric"`
+			TV     [2]interface{}    `json:"value"`
+		} `json:"result"`
+	} `json:"data"`
+	ErrorType string `json:"errorType"`
+	Error     string `json:"error"`
+}
+
+func (r response) metrics() ([]Metric, error) {
+	var ms []Metric
+	var m Metric
+	var f float64
+	var err error
+	for i, res := range r.Data.Result {
+		f, err = strconv.ParseFloat(res.TV[1].(string), 64)
+		if err != nil {
+			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %s", res, res.TV[1], err)
+		}
+		m.Labels = nil
+		for k, v := range r.Data.Result[i].Labels {
+			m.Labels = append(m.Labels, Label{Name: k, Value: v})
+		}
+		m.Timestamp = int64(res.TV[0].(float64))
+		m.Value = f
+		ms = append(ms, m)
+	}
+	return ms, nil
+}
+
+const queryPath = "/api/v1/query?query="
+
+// VMStorage represents vmstorage entity with ability to read and write metrics
+type VMStorage struct {
+	c                            *http.Client
+	queryURL                     string
+	basicAuthUser, basicAuthPass string
+}
+
+// NewVMStorage is a constructor for VMStorage
+func NewVMStorage(baseURL, basicAuthUser, basicAuthPass string, c *http.Client) *VMStorage {
+	return &VMStorage{
+		c:             c,
+		basicAuthUser: basicAuthUser,
+		basicAuthPass: basicAuthPass,
+		queryURL:      strings.TrimSuffix(baseURL, "/") + queryPath,
+	}
+}
+
+// Query reads metrics from datasource by given query
+func (s *VMStorage) Query(ctx context.Context, query string) ([]Metric, error) {
+	const (
+		statusSuccess, statusError, rtVector = "success", "error", "vector"
+	)
+	req, err := http.NewRequest("POST", s.queryURL+url.QueryEscape(query), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if s.basicAuthPass != "" {
+		req.SetBasicAuth(s.basicAuthUser, s.basicAuthPass)
+	}
+	resp, err := s.c.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, fmt.Errorf("error getting response from %s:%s", req.URL, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return nil, fmt.Errorf("datasource returns unxeprected response code %d for %s with err %s. Reponse body %s", resp.StatusCode, req.URL, err, body)
+	}
+	r := &response{}
+	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
+		return nil, fmt.Errorf("error parsing metrics for %s:%s", req.URL, err)
+	}
+	if r.Status == statusError {
+		return nil, fmt.Errorf("response error, query: %s, errorType: %s, error: %s", req.URL, r.ErrorType, r.Error)
+	}
+	if r.Status != statusSuccess {
+		return nil, fmt.Errorf("unkown status:%s, Expected success or error ", r.Status)
+	}
+	if r.Data.ResultType != rtVector {
+		return nil, fmt.Errorf("unkown restul type:%s. Expected vector", r.Data.ResultType)
+	}
+	return r.metrics()
+}

app/vmalert/datasource/vm_test.go

@@ -0,0 +1,93 @@
+package datasource
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+var (
+	ctx           = context.Background()
+	basicAuthName = "foo"
+	basicAuthPass = "bar"
+	query         = "vm_rows"
+)
+
+func TestVMSelectQuery(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+		}
+		if r.URL.Query().Get("query") != query {
+			t.Errorf("exptected %s in query param, got %s", query, r.URL.Query().Get("query"))
+		}
+		switch c {
+		case 0:
+			conn, _, _ := w.(http.Hijacker).Hijack()
+			_ = conn.Close()
+		case 1:
+			w.WriteHeader(500)
+		case 2:
+			w.Write([]byte("[]"))
+		case 3:
+			w.Write([]byte(`{"status":"error", "errorType":"type:", "error":"some error msg"}`))
+		case 4:
+			w.Write([]byte(`{"status":"unknown"}`))
+		case 5:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
+		case 6:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]}]}}`))
+		}
+	})
+
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+	am := NewVMStorage(srv.URL, basicAuthName, basicAuthPass, srv.Client())
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected connection error got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected invalid response status error got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected response body error got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected error status got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected unkown status got nil")
+	}
+	if _, err := am.Query(ctx, query); err == nil {
+		t.Fatalf("expected non-vector resultType error  got nil")
+	}
+	m, err := am.Query(ctx, query)
+	if err != nil {
+		t.Fatalf("unexpected %s", err)
+	}
+	if len(m) != 1 {
+		t.Fatalf("exptected 1 metric  got %d in %+v", len(m), m)
+	}
+	expected := Metric{
+		Labels:    []Label{{Value: "vm_rows", Name: "__name__"}},
+		Timestamp: 1583786142,
+		Value:     13763,
+	}
+	if m[0].Timestamp != expected.Timestamp &&
+		m[0].Value != expected.Value &&
+		m[0].Labels[0].Value != expected.Labels[0].Value &&
+		m[0].Labels[0].Name != expected.Labels[0].Name {
+		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
+	}
+
+}

app/vmalert/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8880
+
+ENTRYPOINT ["/vmalert-prod"]
+ARG src_binary
+COPY $src_binary ./vmalert-prod

app/vmalert/main.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rulePath = flagutil.NewArray("rule", `Path to the file with alert rules. 
+Supports patterns. Flag can be specified multiple times. 
+Examples:
+ -rule /path/to/file. Path to a single file with alerting rules
+ -rule dir/*.yaml -rule /*.yaml. Relative path to all .yaml files in "dir" folder, 
+absolute path to all .yaml files in root.`)
+	validateTemplates = flag.Bool("rule.validateTemplates", true, "Indicates to validate annotation and label templates")
+	httpListenAddr    = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+	datasourceURL     = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter. e.g. http://127.0.0.1:8428")
+	basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username to use for -datasource.url")
+	basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password to use for -datasource.url")
+	remoteWriteURL    = flag.String("remotewrite.url", "", "Optional URL to remote-write compatible storage where to write timeseries"+
+		"based on active alerts. E.g. http://127.0.0.1:8428")
+	evaluationInterval = flag.Duration("evaluationInterval", 1*time.Minute, "How often to evaluate the rules. Default 1m")
+	notifierURL        = flag.String("notifier.url", "", "Prometheus alertmanager URL. Required parameter. e.g. http://127.0.0.1:9093")
+	externalURL        = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
+)
+
+// TODO: hot configuration reload
+// TODO: alerts state persistence
+func main() {
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+	checkFlags()
+	ctx, cancel := context.WithCancel(context.Background())
+	eu, err := getExternalURL(*externalURL, *httpListenAddr, httpserver.IsTLS())
+	if err != nil {
+		logger.Fatalf("can not get external url:%s ", err)
+	}
+	notifier.InitTemplateFunc(eu)
+
+	logger.Infof("reading alert rules configuration file from %s", strings.Join(*rulePath, ";"))
+	groups, err := Parse(*rulePath, *validateTemplates)
+	if err != nil {
+		logger.Fatalf("cannot parse configuration file: %s", err)
+	}
+
+	w := &watchdog{
+		storage: datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{}),
+		alertProvider: notifier.NewAlertManager(*notifierURL, func(group, name string) string {
+			return fmt.Sprintf("%s/api/v1/%s/%s/status", eu, group, name)
+		}, &http.Client{}),
+	}
+
+	if *remoteWriteURL != "" {
+		c, err := remotewrite.NewClient(ctx, remotewrite.Config{
+			Addr:          *remoteWriteURL,
+			FlushInterval: *evaluationInterval,
+		})
+		if err != nil {
+			logger.Fatalf("failed to init remotewrite client: %s", err)
+		}
+		w.rw = c
+	}
+
+	wg := sync.WaitGroup{}
+	for i := range groups {
+		wg.Add(1)
+		go func(group Group) {
+			w.run(ctx, group, *evaluationInterval)
+			wg.Done()
+		}(groups[i])
+	}
+
+	go httpserver.Serve(*httpListenAddr, (&requestHandler{groups: groups}).handler)
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("service received signal %s", sig)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	cancel()
+	if w.rw != nil {
+		err := w.rw.Close()
+		if err != nil {
+			logger.Fatalf("cannot stop the remotewrite: %s", err)
+		}
+	}
+	wg.Wait()
+}
+
+type watchdog struct {
+	storage       *datasource.VMStorage
+	alertProvider notifier.Notifier
+	rw            *remotewrite.Client
+}
+
+var (
+	iterationTotal    = metrics.NewCounter(`vmalert_iteration_total`)
+	iterationDuration = metrics.NewSummary(`vmalert_iteration_duration_seconds`)
+
+	execTotal    = metrics.NewCounter(`vmalert_execution_total`)
+	execErrors   = metrics.NewCounter(`vmalert_execution_errors_total`)
+	execDuration = metrics.NewSummary(`vmalert_execution_duration_seconds`)
+
+	alertsFired      = metrics.NewCounter(`vmalert_alerts_fired_total`)
+	alertsSent       = metrics.NewCounter(`vmalert_alerts_sent_total`)
+	alertsSendErrors = metrics.NewCounter(`vmalert_alerts_send_errors_total`)
+
+	remoteWriteSent   = metrics.NewCounter(`vmalert_remotewrite_sent_total`)
+	remoteWriteErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
+)
+
+func (w *watchdog) run(ctx context.Context, group Group, evaluationInterval time.Duration) {
+	logger.Infof("watchdog for %s has been started", group.Name)
+	t := time.NewTicker(evaluationInterval)
+	defer t.Stop()
+	for {
+
+		select {
+		case <-t.C:
+			iterationTotal.Inc()
+			iterationStart := time.Now()
+			for _, rule := range group.Rules {
+				execTotal.Inc()
+
+				execStart := time.Now()
+				err := rule.Exec(ctx, w.storage)
+				execDuration.UpdateDuration(execStart)
+
+				if err != nil {
+					execErrors.Inc()
+					logger.Errorf("failed to execute rule %q.%q: %s", group.Name, rule.Name, err)
+					continue
+				}
+
+				var alertsToSend []notifier.Alert
+				for _, a := range rule.alerts {
+					if a.State != notifier.StatePending {
+						alertsToSend = append(alertsToSend, *a)
+					}
+					if a.State == notifier.StateInactive || w.rw == nil {
+						continue
+					}
+					tss := rule.AlertToTimeSeries(a, execStart)
+					for _, ts := range tss {
+						remoteWriteSent.Inc()
+						if err := w.rw.Push(ts); err != nil {
+							remoteWriteErrors.Inc()
+							logger.Errorf("failed to push timeseries to remotewrite: %s", err)
+						}
+					}
+				}
+				alertsSent.Add(len(alertsToSend))
+				if err := w.alertProvider.Send(alertsToSend); err != nil {
+					alertsSendErrors.Inc()
+					logger.Errorf("failed to send alert for rule %q.%q: %s", group.Name, rule.Name, err)
+				}
+			}
+			iterationDuration.UpdateDuration(iterationStart)
+		case <-ctx.Done():
+			logger.Infof("%s received stop signal", group.Name)
+			return
+		}
+	}
+}
+
+func getExternalURL(externalURL, httpListenAddr string, isSecure bool) (*url.URL, error) {
+	if externalURL != "" {
+		return url.Parse(externalURL)
+	}
+	hname, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+	port := ""
+	if ipport := strings.Split(httpListenAddr, ":"); len(ipport) > 1 {
+		port = ":" + ipport[1]
+	}
+	schema := "http://"
+	if isSecure {
+		schema = "https://"
+	}
+	return url.Parse(fmt.Sprintf("%s%s%s", schema, hname, port))
+}
+
+func checkFlags() {
+	if *notifierURL == "" {
+		flag.PrintDefaults()
+		logger.Fatalf("notifier.url is empty")
+	}
+	if *datasourceURL == "" {
+		flag.PrintDefaults()
+		logger.Fatalf("datasource.url is empty")
+	}
+}

app/vmalert/notifier/alert.go

@@ -0,0 +1,105 @@
+package notifier
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"text/template"
+	"time"
+)
+
+// Alert the triggered alert
+// TODO: Looks like alert name isn't unique
+type Alert struct {
+	Group       string
+	Name        string
+	Labels      map[string]string
+	Annotations map[string]string
+	State       AlertState
+
+	Start time.Time
+	End   time.Time
+	Value float64
+	ID    uint64
+}
+
+// AlertState type indicates the Alert state
+type AlertState int
+
+const (
+	// StateInactive is the state of an alert that is neither firing nor pending.
+	StateInactive AlertState = iota
+	// StatePending is the state of an alert that has been active for less than
+	// the configured threshold duration.
+	StatePending
+	// StateFiring is the state of an alert that has been active for longer than
+	// the configured threshold duration.
+	StateFiring
+)
+
+// String stringer for AlertState
+func (as AlertState) String() string {
+	switch as {
+	case StateFiring:
+		return "firing"
+	case StatePending:
+		return "pending"
+	}
+	return "inactive"
+}
+
+type alertTplData struct {
+	Labels map[string]string
+	Value  float64
+}
+
+const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}`
+
+// ExecTemplate executes the Alert template for give
+// map of annotations.
+func (a *Alert) ExecTemplate(annotations map[string]string) (map[string]string, error) {
+	tplData := alertTplData{Value: a.Value, Labels: a.Labels}
+	return templateAnnotations(annotations, tplHeader, tplData)
+}
+
+// ValidateTemplates validate annotations for possible template error, uses empty data for template population
+func ValidateTemplates(annotations map[string]string) error {
+	_, err := templateAnnotations(annotations, tplHeader, alertTplData{
+		Labels: map[string]string{},
+		Value:  0,
+	})
+	return err
+}
+
+func templateAnnotations(annotations map[string]string, header string, data alertTplData) (map[string]string, error) {
+	var builder strings.Builder
+	var buf bytes.Buffer
+	eg := errGroup{}
+	r := make(map[string]string, len(annotations))
+	for key, text := range annotations {
+		r[key] = text
+		buf.Reset()
+		builder.Reset()
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
+		builder.WriteString(text)
+		if err := templateAnnotation(&buf, builder.String(), data); err != nil {
+			eg.errs = append(eg.errs, fmt.Sprintf("key %s, template %s:%s", key, text, err))
+			continue
+		}
+		r[key] = buf.String()
+	}
+	return r, eg.err()
+}
+
+func templateAnnotation(dst io.Writer, text string, data alertTplData) error {
+	tpl, err := template.New("").Funcs(tmplFunc).Option("missingkey=zero").Parse(text)
+	if err != nil {
+		return fmt.Errorf("error parsing annotation:%w", err)
+	}
+	if err = tpl.Execute(dst, data); err != nil {
+		return fmt.Errorf("error evaluating annotation template:%w", err)
+	}
+	return nil
+}

app/vmalert/notifier/alert_test.go

@@ -0,0 +1,65 @@
+package notifier
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestAlert_ExecTemplate(t *testing.T) {
+	testCases := []struct {
+		alert       *Alert
+		annotations map[string]string
+		expTpl      map[string]string
+	}{
+		{
+			alert:       &Alert{},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{
+				"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}}",
+			},
+			expTpl: map[string]string{
+				"summary":     "Too high connection number for localhost for job staging",
+				"description": "It is 10000 connections for localhost",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			tpl, err := tc.alert.ExecTemplate(tc.annotations)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(tpl) != len(tc.expTpl) {
+				t.Fatalf("expected %d elements; got %d", len(tc.expTpl), len(tpl))
+			}
+			for k := range tc.expTpl {
+				got, exp := tpl[k], tc.expTpl[k]
+				if got != exp {
+					t.Fatalf("expected %q=%q; got %q=%q", k, exp, k, got)
+				}
+			}
+		})
+	}
+}

app/vmalert/notifier/alertmanager.go

@@ -0,0 +1,51 @@
+package notifier
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// AlertManager represents integration provider with Prometheus alert manager
+// https://github.com/prometheus/alertmanager
+type AlertManager struct {
+	alertURL string
+	argFunc  AlertURLGenerator
+	client   *http.Client
+}
+
+// Send an alert or resolve message
+func (am *AlertManager) Send(alerts []Alert) error {
+	b := &bytes.Buffer{}
+	writeamRequest(b, alerts, am.argFunc)
+	resp, err := am.client.Post(am.alertURL, "application/json", b)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("failed to read response from %q: %s", am.alertURL, err)
+		}
+		return fmt.Errorf("invalid SC %d from %q; response body: %s", resp.StatusCode, am.alertURL, string(body))
+	}
+	return nil
+}
+
+// AlertURLGenerator returns URL to single alert by given name
+type AlertURLGenerator func(group, id string) string
+
+const alertManagerPath = "/api/v2/alerts"
+
+// NewAlertManager is a constructor for AlertManager
+func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, c *http.Client) *AlertManager {
+	return &AlertManager{
+		alertURL: strings.TrimSuffix(alertManagerURL, "/") + alertManagerPath,
+		argFunc:  fn,
+		client:   c,
+	}
+}

app/vmalert/notifier/alertmanager_request.qtpl

@@ -0,0 +1,34 @@
+{% import (
+    "strconv"
+    "time"
+) %}
+{% stripspace %}
+
+{% func amRequest(alerts []Alert, generatorURL func(string, string) string) %}
+[
+{% for i, alert := range alerts %}
+{
+	"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
+	"generatorURL": {%q= generatorURL(alert.Group, strconv.FormatUint(alert.ID, 10)) %},
+    {% if !alert.End.IsZero() %}
+    "endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
+    {% endif %}
+    "labels": {
+        "alertname":{%q= alert.Name %}
+        {% for k,v := range alert.Labels %}
+            ,{%q= k %}:{%q= v %}
+        {% endfor %}
+    },
+    "annotations": {
+        {% code c := len(alert.Annotations) %}
+        {% for k,v := range alert.Annotations %}
+          {% code c = c-1 %}
+          {%q= k %}:{%q= v %}{% if c > 0 %},{% endif %}
+        {% endfor %}
+    }
+}
+{% if i != len(alerts)-1 %},{% endif %}
+{% endfor %}
+]
+{% endfunc %}
+{% endstripspace %}

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -0,0 +1,131 @@
+// Code generated by qtc from "alertmanager_request.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:1
+package notifier
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:1
+import (
+	"strconv"
+	"time"
+)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
+func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:7
+	qw422016.N().S(`[`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:9
+	for i, alert := range alerts {
+//line app/vmalert/notifier/alertmanager_request.qtpl:9
+		qw422016.N().S(`{"startsAt":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:11
+		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
+//line app/vmalert/notifier/alertmanager_request.qtpl:11
+		qw422016.N().S(`,"generatorURL":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
+		qw422016.N().Q(generatorURL(alert.Group, strconv.FormatUint(alert.ID, 10)))
+//line app/vmalert/notifier/alertmanager_request.qtpl:12
+		qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
+		if !alert.End.IsZero() {
+//line app/vmalert/notifier/alertmanager_request.qtpl:13
+			qw422016.N().S(`"endsAt":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
+			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
+//line app/vmalert/notifier/alertmanager_request.qtpl:14
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:15
+		qw422016.N().S(`"labels": {"alertname":`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:17
+		qw422016.N().Q(alert.Name)
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+		for k, v := range alert.Labels {
+//line app/vmalert/notifier/alertmanager_request.qtpl:18
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
+			qw422016.N().Q(k)
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
+			qw422016.N().S(`:`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
+			qw422016.N().Q(v)
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:20
+		qw422016.N().S(`},"annotations": {`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:23
+		c := len(alert.Annotations)
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:24
+		for k, v := range alert.Annotations {
+//line app/vmalert/notifier/alertmanager_request.qtpl:25
+			c = c - 1
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			qw422016.N().Q(k)
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			qw422016.N().S(`:`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			qw422016.N().Q(v)
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			if c > 0 {
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+				qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:26
+			}
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:27
+		qw422016.N().S(`}}`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
+		if i != len(alerts)-1 {
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
+			qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:30
+		}
+//line app/vmalert/notifier/alertmanager_request.qtpl:31
+	}
+//line app/vmalert/notifier/alertmanager_request.qtpl:31
+	qw422016.N().S(`]`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+}
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	streamamRequest(qw422016, alerts, generatorURL)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+}
+
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+func amRequest(alerts []Alert, generatorURL func(string, string) string) string {
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	writeamRequest(qb422016, alerts, generatorURL)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qs422016 := string(qb422016.B)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+	return qs422016
+//line app/vmalert/notifier/alertmanager_request.qtpl:33
+}

app/vmalert/notifier/alertmanager_test.go

@@ -0,0 +1,80 @@
+package notifier
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestAlertManager_Send(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Errorf("should not be called")
+	})
+	c := -1
+	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
+		c++
+		if r.Method != http.MethodPost {
+			t.Errorf("expected POST method got %s", r.Method)
+		}
+		switch c {
+		case 0:
+			conn, _, _ := w.(http.Hijacker).Hijack()
+			_ = conn.Close()
+		case 1:
+			w.WriteHeader(500)
+		case 2:
+			var a []struct {
+				Labels       map[string]string `json:"labels"`
+				StartsAt     time.Time         `json:"startsAt"`
+				EndAt        time.Time         `json:"endsAt"`
+				Annotations  map[string]string `json:"annotations"`
+				GeneratorURL string            `json:"generatorURL"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
+				t.Errorf("can not unmarshal data into alert %s", err)
+				t.FailNow()
+			}
+			if len(a) != 1 {
+				t.Errorf("expected 1 alert in array got %d", len(a))
+			}
+			if a[0].GeneratorURL != "group0" {
+				t.Errorf("exptected alert0 as generatorURL got %s", a[0].GeneratorURL)
+			}
+			if a[0].Labels["alertname"] != "alert0" {
+				t.Errorf("exptected alert0 as alert name got %s", a[0].Labels["alertname"])
+			}
+			if a[0].StartsAt.IsZero() {
+				t.Errorf("exptected non-zero start time")
+			}
+			if a[0].EndAt.IsZero() {
+				t.Errorf("exptected non-zero end time")
+			}
+		}
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+	am := NewAlertManager(srv.URL, func(group, name string) string {
+		return group + name
+	}, srv.Client())
+	if err := am.Send([]Alert{{}, {}}); err == nil {
+		t.Error("expected connection error got nil")
+	}
+	if err := am.Send([]Alert{}); err == nil {
+		t.Error("expected wrong http code error got nil")
+	}
+	if err := am.Send([]Alert{{
+		Group:       "group",
+		Name:        "alert0",
+		Start:       time.Now().UTC(),
+		End:         time.Now().UTC(),
+		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
+	}}); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+	if c != 2 {
+		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
+	}
+}

app/vmalert/notifier/notifier.go

@@ -0,0 +1,6 @@
+package notifier
+
+// Notifier is common interface for alert manager provider
+type Notifier interface {
+	Send(alerts []Alert) error
+}

app/vmalert/notifier/template_func.go

@@ -0,0 +1,171 @@
+// Copyright 2013 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package notifier
+
+import (
+	"fmt"
+	html_template "html/template"
+	"math"
+	"net/url"
+	"regexp"
+	"strings"
+	text_template "text/template"
+	"time"
+)
+
+var tmplFunc text_template.FuncMap
+
+// InitTemplateFunc returns template helper functions
+func InitTemplateFunc(externalURL *url.URL) {
+	tmplFunc = text_template.FuncMap{
+		"args": func(args ...interface{}) map[string]interface{} {
+			result := make(map[string]interface{})
+			for i, a := range args {
+				result[fmt.Sprintf("arg%d", i)] = a
+			}
+			return result
+		},
+		"reReplaceAll": func(pattern, repl, text string) string {
+			re := regexp.MustCompile(pattern)
+			return re.ReplaceAllString(text, repl)
+		},
+		"safeHtml": func(text string) html_template.HTML {
+			return html_template.HTML(text)
+		},
+		"match":   regexp.MatchString,
+		"title":   strings.Title,
+		"toUpper": strings.ToUpper,
+		"toLower": strings.ToLower,
+		"humanize": func(v float64) string {
+			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			if math.Abs(v) >= 1 {
+				prefix := ""
+				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
+					if math.Abs(v) < 1000 {
+						break
+					}
+					prefix = p
+					v /= 1000
+				}
+				return fmt.Sprintf("%.4g%s", v, prefix)
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%s", v, prefix)
+		},
+		"humanize1024": func(v float64) string {
+			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			prefix := ""
+			for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
+				if math.Abs(v) < 1024 {
+					break
+				}
+				prefix = p
+				v /= 1024
+			}
+			return fmt.Sprintf("%.4g%s", v, prefix)
+		},
+		"humanizeDuration": func(v float64) string {
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			if v == 0 {
+				return fmt.Sprintf("%.4gs", v)
+			}
+			if math.Abs(v) >= 1 {
+				sign := ""
+				if v < 0 {
+					sign = "-"
+					v = -v
+				}
+				seconds := int64(v) % 60
+				minutes := (int64(v) / 60) % 60
+				hours := (int64(v) / 60 / 60) % 24
+				days := int64(v) / 60 / 60 / 24
+				// For days to minutes, we display seconds as an integer.
+				if days != 0 {
+					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds)
+				}
+				if hours != 0 {
+					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds)
+				}
+				if minutes != 0 {
+					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds)
+				}
+				// For seconds, we display 4 significant digits.
+				return fmt.Sprintf("%s%.4gs", sign, v)
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%ss", v, prefix)
+		},
+		"humanizePercentage": func(v float64) string {
+			return fmt.Sprintf("%.4g%%", v*100)
+		},
+		"humanizeTimestamp": func(v float64) string {
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v)
+			}
+			t := TimeFromUnixNano(int64(v * 1e9)).Time().UTC()
+			return fmt.Sprint(t)
+		},
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+		"externalURL": func() string {
+			return externalURL.String()
+		},
+	}
+}
+
+// Time is the number of milliseconds since the epoch
+// (1970-01-01 00:00 UTC) excluding leap seconds.
+type Time int64
+
+// TimeFromUnixNano returns the Time equivalent to the Unix Time
+// t provided in nanoseconds.
+func TimeFromUnixNano(t int64) Time {
+	return Time(t / nanosPerTick)
+}
+
+// The number of nanoseconds per minimum tick.
+const nanosPerTick = int64(minimumTick / time.Nanosecond)
+
+// MinimumTick is the minimum supported time resolution. This has to be
+// at least time.Second in order for the code below to work.
+const minimumTick = time.Millisecond
+
+// second is the Time duration equivalent to one second.
+const second = int64(time.Second / minimumTick)
+
+// Time returns the time.Time representation of t.
+func (t Time) Time() time.Time {
+	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
+}

app/vmalert/notifier/utils.go

@@ -0,0 +1,21 @@
+package notifier
+
+import (
+	"fmt"
+	"strings"
+)
+
+type errGroup struct {
+	errs []string
+}
+
+func (eg *errGroup) err() error {
+	if eg == nil || len(eg.errs) == 0 {
+		return nil
+	}
+	return eg
+}
+
+func (eg *errGroup) Error() string {
+	return fmt.Sprintf("errors:%s", strings.Join(eg.errs, "\n"))
+}

app/vmalert/remotewrite/remotewrite.go

@@ -0,0 +1,187 @@
+package remotewrite
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
+)
+
+// Client is an asynchronous HTTP client for writing
+// timeseries via remote write protocol.
+type Client struct {
+	addr           string
+	c              *http.Client
+	input          chan prompbmarshal.TimeSeries
+	baUser, baPass string
+	flushInterval  time.Duration
+	maxBatchSize   int
+	maxQueueSize   int
+
+	wg     sync.WaitGroup
+	doneCh chan struct{}
+}
+
+type Config struct {
+	// Addr of remote storage
+	Addr string
+
+	BasicAuthUser string
+	BasicAuthPass string
+
+	// MaxBatchSize defines max number of timeseries
+	// to be flushed at once
+	MaxBatchSize int
+	// MaxQueueSize defines max length of input queue
+	// populated by Push method
+	MaxQueueSize int
+	// FlushInterval defines time interval for flushing batches
+	FlushInterval time.Duration
+	// WriteTimeout defines timeout for HTTP write request
+	// to remote storage
+	WriteTimeout time.Duration
+}
+
+const (
+	defaultMaxBatchSize  = 1e3
+	defaultMaxQueueSize  = 100
+	defaultFlushInterval = 5 * time.Second
+	defaultWriteTimeout  = 30 * time.Second
+)
+
+const writePath = "/api/v1/write"
+
+// NewClient returns asynchronous client for
+// writing timeseries via remotewrite protocol.
+func NewClient(ctx context.Context, cfg Config) (*Client, error) {
+	if cfg.Addr == "" {
+		return nil, fmt.Errorf("config.Addr can't be empty")
+	}
+	if cfg.MaxBatchSize == 0 {
+		cfg.MaxBatchSize = defaultMaxBatchSize
+	}
+	if cfg.MaxQueueSize == 0 {
+		cfg.MaxQueueSize = defaultMaxQueueSize
+	}
+	if cfg.FlushInterval == 0 {
+		cfg.FlushInterval = defaultFlushInterval
+	}
+	if cfg.WriteTimeout == 0 {
+		cfg.WriteTimeout = defaultWriteTimeout
+	}
+	c := &Client{
+		c: &http.Client{
+			Timeout: cfg.WriteTimeout,
+		},
+		addr:          strings.TrimSuffix(cfg.Addr, "/") + writePath,
+		baUser:        cfg.BasicAuthUser,
+		baPass:        cfg.BasicAuthPass,
+		flushInterval: cfg.FlushInterval,
+		maxBatchSize:  cfg.MaxBatchSize,
+		doneCh:        make(chan struct{}),
+		input:         make(chan prompbmarshal.TimeSeries, cfg.MaxQueueSize),
+	}
+	c.run(ctx)
+	return c, nil
+}
+
+// Push adds timeseries into queue for writing into remote storage.
+// Push returns and error if client is stopped or if queue is full.
+func (c *Client) Push(s prompbmarshal.TimeSeries) error {
+	select {
+	case <-c.doneCh:
+		return fmt.Errorf("client is closed")
+	case c.input <- s:
+		return nil
+	default:
+		return fmt.Errorf("failed to push timeseries - queue is full (%d entries)",
+			c.maxQueueSize)
+	}
+}
+
+// Close stops the client and waits for all goroutines
+// to exit.
+func (c *Client) Close() error {
+	if c.doneCh == nil {
+		return fmt.Errorf("client is already closed")
+	}
+	close(c.input)
+	close(c.doneCh)
+	c.wg.Wait()
+	return nil
+}
+
+func (c *Client) run(ctx context.Context) {
+	ticker := time.NewTicker(c.flushInterval)
+	wr := prompbmarshal.WriteRequest{}
+	shutdown := func() {
+		for ts := range c.input {
+			wr.Timeseries = append(wr.Timeseries, ts)
+		}
+		lastCtx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		c.flush(lastCtx, wr)
+		cancel()
+	}
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
+		defer ticker.Stop()
+		for {
+			select {
+			case <-c.doneCh:
+				shutdown()
+				return
+			case <-ctx.Done():
+				shutdown()
+				return
+			case <-ticker.C:
+				c.flush(ctx, wr)
+				wr = prompbmarshal.WriteRequest{}
+			case ts := <-c.input:
+				wr.Timeseries = append(wr.Timeseries, ts)
+				if len(wr.Timeseries) >= c.maxBatchSize {
+					c.flush(ctx, wr)
+					wr = prompbmarshal.WriteRequest{}
+				}
+			}
+		}
+	}()
+}
+
+func (c *Client) flush(ctx context.Context, wr prompbmarshal.WriteRequest) {
+	if len(wr.Timeseries) < 1 {
+		return
+	}
+	data, err := wr.Marshal()
+	if err != nil {
+		logger.Errorf("failed to marshal WriteRequest: %s", err)
+		return
+	}
+	req, err := http.NewRequest("POST", c.addr, bytes.NewReader(snappy.Encode(nil, data)))
+	if err != nil {
+		logger.Errorf("failed to create new HTTP request: %s", err)
+		return
+	}
+	if c.baPass != "" {
+		req.SetBasicAuth(c.baUser, c.baPass)
+	}
+	resp, err := c.c.Do(req.WithContext(ctx))
+	if err != nil {
+		logger.Errorf("error getting response from %s:%s", req.URL, err)
+		return
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusNoContent {
+		body, _ := ioutil.ReadAll(resp.Body)
+		logger.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, req.URL, body)
+		return
+	}
+}

app/vmalert/rule.go

@@ -0,0 +1,269 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+// Group grouping array of alert
+type Group struct {
+	Name  string
+	Rules []*Rule
+}
+
+// Rule is basic alert entity
+type Rule struct {
+	Name        string            `yaml:"alert"`
+	Expr        string            `yaml:"expr"`
+	For         time.Duration     `yaml:"for"`
+	Labels      map[string]string `yaml:"labels"`
+	Annotations map[string]string `yaml:"annotations"`
+
+	group *Group
+
+	// guard status fields
+	mu sync.RWMutex
+	// stores list of active alerts
+	alerts map[uint64]*notifier.Alert
+	// stores last moment of time Exec was called
+	lastExecTime time.Time
+	// stores last error that happened in Exec func
+	// resets on every successful Exec
+	// may be used as Health state
+	lastExecError error
+}
+
+// Validate validates rule
+func (r *Rule) Validate() error {
+	if r.Name == "" {
+		return errors.New("rule name can not be empty")
+	}
+	if r.Expr == "" {
+		return fmt.Errorf("expression for rule %q can't be empty", r.Name)
+	}
+	if _, err := metricsql.Parse(r.Expr); err != nil {
+		return fmt.Errorf("invalid expression for rule %q: %w", r.Name, err)
+	}
+	return nil
+}
+
+// Exec executes Rule expression via the given Querier.
+// Based on the Querier results Rule maintains notifier.Alerts
+func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
+	qMetrics, err := q.Query(ctx, r.Expr)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	r.lastExecError = err
+	r.lastExecTime = time.Now()
+	if err != nil {
+		return fmt.Errorf("failed to execute query %q: %s", r.Expr, err)
+	}
+
+	for h, a := range r.alerts {
+		// cleanup inactive alerts from previous Eval
+		if a.State == notifier.StateInactive {
+			delete(r.alerts, h)
+		}
+	}
+
+	updated := make(map[uint64]struct{})
+	// update list of active alerts
+	for _, m := range qMetrics {
+		h := hash(m)
+		updated[h] = struct{}{}
+		if _, ok := r.alerts[h]; ok {
+			continue
+		}
+		a, err := r.newAlert(m)
+		if err != nil {
+			r.lastExecError = err
+			return fmt.Errorf("failed to create alert: %s", err)
+		}
+		a.ID = h
+		a.State = notifier.StatePending
+		r.alerts[h] = a
+	}
+
+	for h, a := range r.alerts {
+		// if alert wasn't updated in this iteration
+		// means it is resolved already
+		if _, ok := updated[h]; !ok {
+			a.State = notifier.StateInactive
+			// set endTime to last execution time
+			// so it can be sent by notifier on next step
+			a.End = r.lastExecTime
+			continue
+		}
+		if a.State == notifier.StatePending && time.Since(a.Start) >= r.For {
+			a.State = notifier.StateFiring
+			alertsFired.Inc()
+		}
+		if a.State == notifier.StateFiring {
+			a.End = r.lastExecTime.Add(3 * *evaluationInterval)
+		}
+	}
+	return nil
+}
+
+// TODO: consider hashing algorithm in VM
+func hash(m datasource.Metric) uint64 {
+	hash := fnv.New64a()
+	labels := m.Labels
+	sort.Slice(labels, func(i, j int) bool {
+		return labels[i].Name < labels[j].Name
+	})
+	for _, l := range labels {
+		hash.Write([]byte(l.Name))
+		hash.Write([]byte(l.Value))
+		hash.Write([]byte("\xff"))
+	}
+	return hash.Sum64()
+}
+
+func (r *Rule) newAlert(m datasource.Metric) (*notifier.Alert, error) {
+	a := &notifier.Alert{
+		Group:  r.group.Name,
+		Name:   r.Name,
+		Labels: map[string]string{},
+		Value:  m.Value,
+		Start:  time.Now(),
+		// TODO: support End time
+	}
+
+	// 1. use data labels
+	for _, l := range m.Labels {
+		a.Labels[l.Name] = l.Value
+	}
+
+	// 2. template rule labels with data labels
+	rLabels, err := a.ExecTemplate(r.Labels)
+	if err != nil {
+		return a, err
+	}
+
+	// 3. merge data labels and rule labels
+	// metric labels may be overridden by
+	// rule labels
+	for k, v := range rLabels {
+		a.Labels[k] = v
+	}
+
+	// 4. template merged labels
+	a.Labels, err = a.ExecTemplate(a.Labels)
+	if err != nil {
+		return a, err
+	}
+
+	a.Annotations, err = a.ExecTemplate(r.Annotations)
+	return a, err
+}
+
+// AlertAPI generates APIAlert object from alert by its id(hash)
+func (r *Rule) AlertAPI(id uint64) *APIAlert {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	a, ok := r.alerts[id]
+	if !ok {
+		return nil
+	}
+	return r.newAlertAPI(*a)
+}
+
+// AlertsAPI generates list of APIAlert objects from existing alerts
+func (r *Rule) AlertsAPI() []*APIAlert {
+	var alerts []*APIAlert
+	r.mu.RLock()
+	for _, a := range r.alerts {
+		alerts = append(alerts, r.newAlertAPI(*a))
+	}
+	r.mu.RUnlock()
+	return alerts
+}
+
+func (r *Rule) newAlertAPI(a notifier.Alert) *APIAlert {
+	return &APIAlert{
+		ID:          a.ID,
+		Name:        a.Name,
+		Group:       a.Group,
+		Expression:  r.Expr,
+		Labels:      a.Labels,
+		Annotations: a.Annotations,
+		State:       a.State.String(),
+		ActiveAt:    a.Start,
+		Value:       strconv.FormatFloat(a.Value, 'e', -1, 64),
+	}
+}
+
+const (
+	// AlertMetricName is the metric name for synthetic alert timeseries.
+	alertMetricName = "ALERTS"
+	// AlertForStateMetricName is the metric name for 'for' state of alert.
+	alertForStateMetricName = "ALERTS_FOR_STATE"
+
+	// AlertNameLabel is the label name indicating the name of an alert.
+	alertNameLabel = "alertname"
+	// AlertStateLabel is the label name indicating the state of an alert.
+	alertStateLabel = "alertstate"
+)
+
+func (r *Rule) AlertToTimeSeries(a *notifier.Alert, timestamp time.Time) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, alertToTimeSeries(r.Name, a, timestamp))
+	if r.For > 0 {
+		tss = append(tss, alertForToTimeSeries(r.Name, a, timestamp))
+	}
+	return tss
+}
+
+func alertToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertMetricName
+	labels[alertNameLabel] = name
+	labels[alertStateLabel] = a.State.String()
+	return newTimeSeries(1, labels, timestamp)
+}
+
+func alertForToTimeSeries(name string, a *notifier.Alert, timestamp time.Time) prompbmarshal.TimeSeries {
+	labels := make(map[string]string)
+	for k, v := range a.Labels {
+		labels[k] = v
+	}
+	labels["__name__"] = alertForStateMetricName
+	labels[alertNameLabel] = name
+	return newTimeSeries(float64(a.Start.Unix()), labels, timestamp)
+}
+
+func newTimeSeries(value float64, labels map[string]string, timestamp time.Time) prompbmarshal.TimeSeries {
+	ts := prompbmarshal.TimeSeries{}
+	ts.Samples = append(ts.Samples, prompbmarshal.Sample{
+		Value:     value,
+		Timestamp: timestamp.UnixNano() / 1e6,
+	})
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		ts.Labels = append(ts.Labels, prompbmarshal.Label{
+			Name:  key,
+			Value: labels[key],
+		})
+	}
+	return ts
+}

app/vmalert/rule_test.go

@@ -0,0 +1,399 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func TestRule_Validate(t *testing.T) {
+	if err := (&Rule{}).Validate(); err == nil {
+		t.Errorf("exptected empty name error")
+	}
+	if err := (&Rule{Name: "alert"}).Validate(); err == nil {
+		t.Errorf("exptected empty expr error")
+	}
+	if err := (&Rule{Name: "alert", Expr: "test{"}).Validate(); err == nil {
+		t.Errorf("exptected invalid expr error")
+	}
+	if err := (&Rule{Name: "alert", Expr: "test>0"}).Validate(); err != nil {
+		t.Errorf("exptected valid rule got %s", err)
+	}
+}
+
+func TestRule_AlertToTimeSeries(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		rule  *Rule
+		alert *notifier.Alert
+		expTS []prompbmarshal.TimeSeries
+	}{
+		{
+			newTestRule("instant", 0),
+			&notifier.Alert{State: notifier.StateFiring},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("instant extra labels", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				"job":      "foo",
+				"instance": "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant extra labels",
+					"job":           "foo",
+					"instance":      "bar",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("instant labels override", 0),
+			&notifier.Alert{State: notifier.StateFiring, Labels: map[string]string{
+				alertStateLabel: "foo",
+				"__name__":      "bar",
+			}},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "instant labels override",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("for", time.Second),
+			&notifier.Alert{State: notifier.StateFiring, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StateFiring.String(),
+					alertNameLabel:  "for",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for",
+				}, timestamp),
+			},
+		},
+		{
+			newTestRule("for pending", 10*time.Second),
+			&notifier.Alert{State: notifier.StatePending, Start: timestamp.Add(time.Second)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries(1, map[string]string{
+					"__name__":      alertMetricName,
+					alertStateLabel: notifier.StatePending.String(),
+					alertNameLabel:  "for pending",
+				}, timestamp),
+				newTimeSeries(float64(timestamp.Add(time.Second).Unix()), map[string]string{
+					"__name__":     alertForStateMetricName,
+					alertNameLabel: "for pending",
+				}, timestamp),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			tss := tc.rule.AlertToTimeSeries(tc.alert, timestamp)
+			if len(tc.expTS) != len(tss) {
+				t.Fatalf("expected number of timeseries %d; got %d", len(tc.expTS), len(tss))
+			}
+			for i := range tc.expTS {
+				expTS, gotTS := tc.expTS[i], tss[i]
+				if len(expTS.Samples) != len(gotTS.Samples) {
+					t.Fatalf("expected number of samples %d; got %d", len(expTS.Samples), len(gotTS.Samples))
+				}
+				for i, exp := range expTS.Samples {
+					got := gotTS.Samples[i]
+					if got.Value != exp.Value {
+						t.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
+					}
+					if got.Timestamp != exp.Timestamp {
+						t.Errorf("expected timestamp %d; got %d", exp.Timestamp, got.Timestamp)
+					}
+				}
+				if len(expTS.Labels) != len(gotTS.Labels) {
+					t.Fatalf("expected number of labels %d; got %d", len(expTS.Labels), len(gotTS.Labels))
+				}
+				for i, exp := range expTS.Labels {
+					got := gotTS.Labels[i]
+					if got.Name != exp.Name {
+						t.Errorf("expected label name %q; got %q", exp.Name, got.Name)
+					}
+					if got.Value != exp.Value {
+						t.Errorf("expected label value %q; got %q", exp.Value, got.Value)
+					}
+				}
+			}
+		})
+	}
+}
+
+func newTestRule(name string, waitFor time.Duration) *Rule {
+	return &Rule{Name: name, alerts: make(map[uint64]*notifier.Alert), For: waitFor}
+}
+
+func TestRule_Exec(t *testing.T) {
+	testCases := []struct {
+		rule      *Rule
+		steps     [][]datasource.Metric
+		expAlerts map[uint64]*notifier.Alert
+	}{
+		{
+			newTestRule("empty", 0),
+			[][]datasource.Metric{},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestRule("single-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("single-firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestRule("single-firing=>inactive=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{},
+				{metricWithLabels(t, "__name__", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("single-firing=>inactive=>firing=>inactive", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{},
+				{metricWithLabels(t, "__name__", "foo")},
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestRule("single-firing=>inactive=>firing=>inactive=>empty", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{},
+				{metricWithLabels(t, "__name__", "foo")},
+				{},
+				{},
+			},
+			map[uint64]*notifier.Alert{},
+		},
+		{
+			newTestRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{},
+				{metricWithLabels(t, "__name__", "foo")},
+				{},
+				{},
+				{metricWithLabels(t, "__name__", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("multiple-firing", 0),
+			[][]datasource.Metric{
+				{
+					metricWithLabels(t, "__name__", "foo"),
+					metricWithLabels(t, "__name__", "foo1"),
+					metricWithLabels(t, "__name__", "foo2"),
+				},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")):  {State: notifier.StateFiring},
+				hash(metricWithLabels(t, "__name__", "foo1")): {State: notifier.StateFiring},
+				hash(metricWithLabels(t, "__name__", "foo2")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("multiple-steps-firing", 0),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{metricWithLabels(t, "__name__", "foo1")},
+				{metricWithLabels(t, "__name__", "foo2")},
+			},
+			// 1: fire first alert
+			// 2: fire second alert, set first inactive
+			// 3: fire third alert, set second inactive, delete first one
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo1")): {State: notifier.StateInactive},
+				hash(metricWithLabels(t, "__name__", "foo2")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("duplicate", 0),
+			[][]datasource.Metric{
+				{
+					// metrics with the same labelset should result in one alert
+					metricWithLabels(t, "__name__", "foo", "type", "bar"),
+					metricWithLabels(t, "type", "bar", "__name__", "foo"),
+				},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo", "type", "bar")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("for-pending", time.Minute),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StatePending},
+			},
+		},
+		{
+			newTestRule("for-fired", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{metricWithLabels(t, "__name__", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StateFiring},
+			},
+		},
+		{
+			newTestRule("for-pending=>inactive", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{metricWithLabels(t, "__name__", "foo")},
+				// empty step to reset pending alerts
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestRule("for-pending=>firing=>inactive", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{metricWithLabels(t, "__name__", "foo")},
+				// empty step to reset pending alerts
+				{},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StateInactive},
+			},
+		},
+		{
+			newTestRule("for-pending=>firing=>inactive=>pending", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{metricWithLabels(t, "__name__", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "__name__", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StatePending},
+			},
+		},
+		{
+			newTestRule("for-pending=>firing=>inactive=>pending=>firing", time.Millisecond),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "__name__", "foo")},
+				{metricWithLabels(t, "__name__", "foo")},
+				// empty step to reset pending alerts
+				{},
+				{metricWithLabels(t, "__name__", "foo")},
+				{metricWithLabels(t, "__name__", "foo")},
+			},
+			map[uint64]*notifier.Alert{
+				hash(metricWithLabels(t, "__name__", "foo")): {State: notifier.StateFiring},
+			},
+		},
+	}
+	fakeGroup := &Group{Name: "TestRule_Exec"}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &fakeQuerier{}
+			tc.rule.group = fakeGroup
+			for _, step := range tc.steps {
+				fq.reset()
+				fq.add(t, step...)
+				if err := tc.rule.Exec(context.TODO(), fq); err != nil {
+					t.Fatalf("unexpected err: %s", err)
+				}
+				// artificial delay between applying steps
+				time.Sleep(time.Millisecond)
+			}
+			if len(tc.rule.alerts) != len(tc.expAlerts) {
+				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
+			}
+			for key, exp := range tc.expAlerts {
+				got, ok := tc.rule.alerts[key]
+				if !ok {
+					t.Fatalf("expected to have key %d", key)
+				}
+				if got.State != exp.State {
+					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				}
+			}
+		})
+	}
+}
+
+func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	m := datasource.Metric{}
+	for i := 0; i < len(labels); i += 2 {
+		m.Labels = append(m.Labels, datasource.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return m
+}
+
+type fakeQuerier struct {
+	metrics []datasource.Metric
+}
+
+func (fq *fakeQuerier) reset() {
+	fq.metrics = fq.metrics[:0]
+}
+
+func (fq *fakeQuerier) add(t *testing.T, metrics ...datasource.Metric) {
+	fq.metrics = append(fq.metrics, metrics...)
+}
+
+func (fq fakeQuerier) Query(ctx context.Context, query string) ([]datasource.Metric, error) {
+	return fq.metrics, nil
+}

app/vmalert/testdata/dir/rules0-bad.rules

@@ -0,0 +1,19 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidAnnotations
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }"
+          description: "{{$labels}}"
+      - alert: UnkownAnnotationsFunction
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ value|query }}"
+          description: "{{$labels}}"

app/vmalert/testdata/dir/rules0-good.rules

@@ -0,0 +1,13 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value|humanize }}"
+          description: "{{$labels}}"
+
+

app/vmalert/testdata/dir/rules1-bad.rules

@@ -0,0 +1,22 @@
+groups:
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/testdata/dir/rules1-good.rules

@@ -0,0 +1,13 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+
+

app/vmalert/testdata/dir/rules2-bad.rules

@@ -0,0 +1,11 @@
+groups:
+  - name: group
+    rules:
+      - alert: UnkownLabelFunction
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          summary: "{{ value|query }}"
+        annotations:
+          description: "{{$labels}}"

app/vmalert/testdata/rules0-bad.rules

@@ -0,0 +1,28 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidExpr
+        for: 5m
+        expr: vm_rows{ > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: EmptyExpr
+        for: 5m
+        expr: ""
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: ""
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: foo
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/testdata/rules0-good.rules

@@ -0,0 +1,23 @@
+groups:
+  - name: groupGorSingleAlert
+    rules:
+      - alert: VMRows
+        for: 10s
+        expr: vm_rows > 0
+        labels:
+          label: bar
+          template: "{{ $value|humanize }}"
+        annotations:
+          summary: "{{ $value|humanize }}"
+          description: "{{$labels}}"
+
+  - name: TestGroup
+    rules:
+      - alert: Conns
+        expr: sum(vm_tcplistener_conns) by(instance) > 1
+        annotations:
+          summary: "Too high connection number for {{$labels.instance}}"
+          description: "It is {{ $value }} connections for {{$labels.instance}}"
+      - alert: ExampleAlertAlwaysFiring
+        expr: sum by(job)
+          (up == 1)

app/vmalert/web.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+)
+
+// APIAlert has info for an alert.
+type APIAlert struct {
+	ID          uint64            `json:"id"`
+	Name        string            `json:"name"`
+	Group       string            `json:"group"`
+	Expression  string            `json:"expression"`
+	State       string            `json:"state"`
+	Value       string            `json:"value"`
+	Labels      map[string]string `json:"labels"`
+	Annotations map[string]string `json:"annotations"`
+	ActiveAt    time.Time         `json:"activeAt"`
+}
+
+type requestHandler struct {
+	groups []Group
+}
+
+var pathList = [][]string{
+	{"/api/v1/alerts", "list all active alerts"},
+	{"/api/v1/groupName/alertID/status", "get alert status by ID"},
+	// /metrics is served by httpserver by default
+	{"/metrics", "list of application metrics"},
+}
+
+func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
+	resph := responseHandler{w}
+	switch r.URL.Path {
+	case "/":
+		for _, path := range pathList {
+			p, doc := path[0], path[1]
+			fmt.Fprintf(w, "<a href='%s'>%q</a> - %s<br/>", p, p, doc)
+		}
+		return true
+	case "/api/v1/alerts":
+		resph.handle(rh.list())
+		return true
+	default:
+		// /api/v1/<groupName>/<alertID>/status
+		if strings.HasSuffix(r.URL.Path, "/status") {
+			resph.handle(rh.alert(r.URL.Path))
+			return true
+		}
+		return false
+	}
+}
+
+type listAlertsResponse struct {
+	Data struct {
+		Alerts []*APIAlert `json:"alerts"`
+	} `json:"data"`
+	Status string `json:"status"`
+}
+
+func (rh *requestHandler) list() ([]byte, error) {
+	lr := listAlertsResponse{Status: "success"}
+	for _, g := range rh.groups {
+		for _, r := range g.Rules {
+			lr.Data.Alerts = append(lr.Data.Alerts, r.AlertsAPI()...)
+		}
+	}
+
+	// sort list of alerts for deterministic output
+	sort.Slice(lr.Data.Alerts, func(i, j int) bool {
+		return lr.Data.Alerts[i].Name < lr.Data.Alerts[j].Name
+	})
+
+	b, err := json.Marshal(lr)
+	if err != nil {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`error encoding list of active alerts: %s`, err),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+	return b, nil
+}
+
+func (rh *requestHandler) alert(path string) ([]byte, error) {
+	parts := strings.SplitN(strings.TrimPrefix(path, "/api/v1/"), "/", 3)
+	if len(parts) != 3 {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/group/alert/status"`, path),
+			StatusCode: http.StatusBadRequest,
+		}
+	}
+	group := strings.TrimRight(parts[0], "/")
+	idStr := strings.TrimRight(parts[1], "/")
+	id, err := strconv.ParseUint(idStr, 10, 0)
+	if err != nil {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`cannot parse int from %q`, idStr),
+			StatusCode: http.StatusBadRequest,
+		}
+	}
+	for _, g := range rh.groups {
+		if g.Name != group {
+			continue
+		}
+		for i := range g.Rules {
+			if apiAlert := g.Rules[i].AlertAPI(id); apiAlert != nil {
+				return json.Marshal(apiAlert)
+			}
+		}
+	}
+	return nil, &httpserver.ErrorWithStatusCode{
+		Err:        fmt.Errorf(`cannot find alert %s in %q`, idStr, group),
+		StatusCode: http.StatusNotFound,
+	}
+}
+
+// responseHandler wrapper on http.ResponseWriter with sugar
+type responseHandler struct{ http.ResponseWriter }
+
+func (w responseHandler) handle(b []byte, err error) {
+	if err != nil {
+		httpserver.Errorf(w, "%s", err)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(b)
+}

app/vmalert/web_test.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+func TestHandler(t *testing.T) {
+	rule := &Rule{
+		Name: "alert",
+		alerts: map[uint64]*notifier.Alert{
+			0: {},
+		},
+	}
+	rh := &requestHandler{
+		groups: []Group{{
+			Name:  "group",
+			Rules: []*Rule{rule},
+		}},
+	}
+	getResp := func(url string, to interface{}, code int) {
+		t.Helper()
+		resp, err := http.Get(url)
+		if err != nil {
+			t.Errorf("unexpected err %s", err)
+		}
+		if code != resp.StatusCode {
+			t.Errorf("unexpected status code %d want %d", resp.StatusCode, code)
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				t.Errorf("err closing body %s", err)
+			}
+		}()
+		if to != nil {
+			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
+				t.Errorf("unexpected err %s", err)
+			}
+		}
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rh.handler(w, r) }))
+	defer ts.Close()
+	t.Run("/api/v1/alerts", func(t *testing.T) {
+		lr := listAlertsResponse{}
+		getResp(ts.URL+"/api/v1/alerts", &lr, 200)
+		if length := len(lr.Data.Alerts); length != 1 {
+			t.Errorf("expected 1 alert got %d", length)
+		}
+	})
+	t.Run("/api/v1/group/0/status", func(t *testing.T) {
+		alert := &APIAlert{}
+		getResp(ts.URL+"/api/v1/group/0/status", alert, 200)
+		expAlert := rule.newAlertAPI(*rule.alerts[0])
+		if !reflect.DeepEqual(alert, expAlert) {
+			t.Errorf("expected %v is equal to %v", alert, expAlert)
+		}
+	})
+	t.Run("/api/v1/group/1/status", func(t *testing.T) {
+		getResp(ts.URL+"/api/v1/group/1/status", nil, 404)
+	})
+	t.Run("/api/v1/unknown-group/0/status", func(t *testing.T) {
+		getResp(ts.URL+"/api/v1/unknown-group/0/status", nil, 404)
+	})
+	t.Run("/", func(t *testing.T) {
+		getResp(ts.URL, nil, 200)
+	})
+}

app/vmbackup/Makefile

@@ -3,6 +3,9 @@
 vmbackup:
 	APP_NAME=vmbackup $(MAKE) app-local
 
+vmbackup-race:
+	APP_NAME=vmbackup RACE=-race $(MAKE) app-local
+
 vmbackup-prod:
 	APP_NAME=vmbackup $(MAKE) app-via-docker
 

app/vmbackup/README.md

@@ -140,14 +140,28 @@ Run `vmbackup -help` in order to see all the available options:
   -dst string
     	Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
     	-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
+  -fs.disableMmap
+    	Whether to use pread() instead of mmap() for reading data files
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
-    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
   -maxBytesPerSecond int
     	The maximum upload speed. There is no limit if it is set to 0
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
   -origin string
     	Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups
+  -snapshot.createURL string
+    	VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup.Example: http://victoriametrics:8428/snaphsot/create
+  -snapshot.deleteURL string
+    	VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted.Example: http://victoriametrics:8428/snaphsot/delete
   -snapshotName string
     	Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots
   -storageDataPath string
@@ -164,7 +178,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 
 #### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
 2. Run `make vmbackup` from the root folder of the repository.
    It builds `vmbackup` binary and puts it into the `bin` folder.
 
@@ -179,3 +193,10 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 Run `make package-vmbackup`. It builds `victoriametrics/vmbackup:<PKG_TAG>` docker image locally.
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmbackup`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmbackup
+```

app/vmbackup/deployment/Dockerfile

@@ -1,7 +1,6 @@
-ARG certs_image
-FROM $certs_image AS certs
-FROM scratch
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG base_image
+FROM $base_image
+
+ENTRYPOINT ["/vmbackup-prod"]
 ARG src_binary
 COPY $src_binary ./vmbackup-prod
-ENTRYPOINT ["/vmbackup-prod"]

app/vmbackup/main.go

@@ -4,18 +4,25 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmbackup/snapshot"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/actions"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 var (
-	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
-	snapshotName    = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots")
-	dst             = flag.String("dst", "", "Where to put the backup on the remote storage. "+
+	storageDataPath   = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
+	snapshotName      = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#how-to-work-with-snapshots")
+	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup."+
+		"Example: http://victoriametrics:8428/snaphsot/create")
+	snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from snapshotCreateURL if not provided. All created snaphosts will be automatically deleted."+
+		"Example: http://victoriametrics:8428/snaphsot/delete")
+	dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+
 		"Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+
 		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
 	origin            = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
@@ -25,9 +32,37 @@ var (
 
 func main() {
 	flag.Usage = usage
-	flag.Parse()
+	envflag.Parse()
 	buildinfo.Init()
 
+	if len(*snapshotCreateURL) > 0 {
+		logger.Infof("%s", "Snapshots enabled")
+		logger.Infof("Snapshot create url %s", *snapshotCreateURL)
+		if len(*snapshotDeleteURL) <= 0 {
+			err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
+			if err != nil {
+				logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err)
+			}
+		}
+		logger.Infof("Snapshot delete url %s", *snapshotDeleteURL)
+
+		name, err := snapshot.Create(*snapshotCreateURL)
+		if err != nil {
+			logger.Fatalf("%s", err)
+		}
+		err = flag.Set("snapshotName", name)
+		if err != nil {
+			logger.Fatalf("Failed to set snapshotName flag: %v", err)
+		}
+
+		defer func() {
+			err := snapshot.Delete(*snapshotDeleteURL, name)
+			if err != nil {
+				logger.Fatalf("%s", err)
+			}
+		}()
+	}
+
 	srcFS, err := newSrcFS()
 	if err != nil {
 		logger.Fatalf("%s", err)
@@ -66,7 +101,7 @@ See the docs at https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/a
 
 func newSrcFS() (*fslocal.FS, error) {
 	if len(*snapshotName) == 0 {
-		return nil, fmt.Errorf("`-snapshotName` cannot be empty")
+		return nil, fmt.Errorf("`-snapshotName` or `-snapshot.createURL` must be provided")
 	}
 	snapshotPath := *storageDataPath + "/snapshots/" + *snapshotName
 

app/vmbackup/snapshot/snapshot.go

@@ -0,0 +1,91 @@
+package snapshot
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+type snapshot struct {
+	Status   string `json:"status"`
+	Snapshot string `json:"snapshot"`
+	Msg      string `json:"msg"`
+}
+
+// Create creates a snapshot and the provided api endpoint and returns
+// the snapshot name
+func Create(createSnapshotURL string) (string, error) {
+	logger.Infof("%s", "Creating snapshot")
+	u, err := url.Parse(createSnapshotURL)
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := http.Get(u.String())
+	if err != nil {
+		return "", err
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	snap := snapshot{}
+	err = json.Unmarshal(body, &snap)
+	if err != nil {
+		return "", err
+	}
+
+	if snap.Status == "ok" {
+		logger.Infof("Snapshot %s created", snap.Snapshot)
+		return snap.Snapshot, nil
+	} else if snap.Status == "error" {
+		return "", errors.New(snap.Msg)
+	} else {
+		return "", fmt.Errorf("Unkown status: %v", snap.Status)
+	}
+}
+
+// Delete deletes a snapshot and the provided api endpoint returns any failure
+func Delete(deleteSnapshotURL string, snapshotName string) error {
+	logger.Infof("Deleting snapshot %s", snapshotName)
+	formData := url.Values{
+		"snapshot": {snapshotName},
+	}
+
+	u, err := url.Parse(deleteSnapshotURL)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.PostForm(u.String(), formData)
+	if err != nil {
+		return err
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	snap := snapshot{}
+	err = json.Unmarshal(body, &snap)
+	if err != nil {
+		return err
+	}
+
+	if snap.Status == "ok" {
+		logger.Infof("Snapshot %s deleted", snapshotName)
+		return nil
+	} else if snap.Status == "error" {
+		return errors.New(snap.Msg)
+	} else {
+		return fmt.Errorf("Unkown status: %v", snap.Status)
+	}
+}

app/vmbackup/snapshot/snapshot_test.go

@@ -0,0 +1,106 @@
+package snapshot
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestCreateSnapshot(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/create" {
+			_, err := io.WriteString(w, `{"status":"ok","snapshot":"mysnapshot"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	snapshotName, err := Create(server.URL + "/snapshot/create")
+	if err != nil {
+		t.Fatalf("Failed taking snapshot: %v", err)
+	}
+
+	if snapshotName != "mysnapshot" {
+		t.Fatalf("Snapshot name is not correct, got %v", snapshotName)
+	}
+}
+
+func TestCreateSnapshotFailed(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/create" {
+			_, err := io.WriteString(w, `{"status":"error","msg":"I am unwell"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	snapshotName, err := Create(server.URL + "/snapshot/create")
+	if err == nil {
+		t.Fatalf("Snapshot did not fail, got snapshot: %v", snapshotName)
+	}
+}
+
+func TestDeleteSnapshot(t *testing.T) {
+	snapshotName := "mysnapshot"
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/delete" {
+			_, err := io.WriteString(w, `{"status":"ok"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+		if r.FormValue("snapshot") != snapshotName {
+			t.Fatalf("Invalid snapshot name, got %v", snapshotName)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	err := Delete(server.URL+"/snapshot/delete", snapshotName)
+	if err != nil {
+		t.Fatalf("Failed to delete snapshot: %v", err)
+	}
+}
+
+func TestDeleteSnapshotFailed(t *testing.T) {
+	snapshotName := "mysnapshot"
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/snapshot/delete" {
+			_, err := io.WriteString(w, `{"status":"error", "msg":"failed to delete"}`)
+			if err != nil {
+				t.Fatalf("Failed to write response output: %v", err)
+			}
+		} else {
+			t.Fatalf("Invalid path, got %v", r.URL.Path)
+		}
+		if r.FormValue("snapshot") != snapshotName {
+			t.Fatalf("Invalid snapshot name, got %v", snapshotName)
+		}
+	})
+
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	defer server.Close()
+
+	err := Delete(server.URL+"/snapshot/delete", snapshotName)
+	if err == nil {
+		t.Fatalf("Snapshot should have failed, got: %v", err)
+	}
+}

app/vminsert/common/insert_ctx_pool.go

@@ -0,0 +1,36 @@
+package common
+
+import (
+	"runtime"
+	"sync"
+)
+
+// GetInsertCtx returns InsertCtx from the pool.
+//
+// Call PutInsertCtx for returning it to the pool.
+func GetInsertCtx() *InsertCtx {
+	select {
+	case ctx := <-insertCtxPoolCh:
+		return ctx
+	default:
+		if v := insertCtxPool.Get(); v != nil {
+			return v.(*InsertCtx)
+		}
+		return &InsertCtx{}
+	}
+}
+
+// PutInsertCtx returns ctx to the pool.
+//
+// ctx cannot be used after the call.
+func PutInsertCtx(ctx *InsertCtx) {
+	ctx.Reset(0)
+	select {
+	case insertCtxPoolCh <- ctx:
+	default:
+		insertCtxPool.Put(ctx)
+	}
+}
+
+var insertCtxPool sync.Pool
+var insertCtxPoolCh = make(chan *InsertCtx, runtime.GOMAXPROCS(-1))

app/vminsert/csvimport/request_handler.go

@@ -0,0 +1,44 @@
+package csvimport
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="csvimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="csvimport"}`)
+)
+
+// InsertHandler processes /api/v1/import/csv requests.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(rows)
+		})
+	})
+}
+
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	ctx.Reset(len(rows))
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			ctx.AddLabel(tag.Key, tag.Value)
+		}
+		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
+	}
+	rowsInserted.Add(len(rows))
+	rowsPerInsert.Update(float64(len(rows)))
+	return ctx.FlushBufs()
+}

app/vminsert/graphite/request_handler.go

@@ -1,161 +1,44 @@
 package graphite
 
 import (
-	"fmt"
 	"io"
-	"net"
-	"runtime"
-	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="graphite"}`)
-	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="graphite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="graphite"}`)
 )
 
-// insertHandler processes remote write for graphite plaintext protocol.
+// InsertHandler processes remote write for graphite plaintext protocol.
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
-func insertHandler(r io.Reader) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(r)
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
 	})
 }
 
-func insertHandlerInternal(r io.Reader) error {
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-	for ctx.Read(r) {
-		if err := ctx.InsertRows(); err != nil {
-			return err
-		}
-	}
-	return ctx.Error()
-}
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
 
-func (ctx *pushCtx) InsertRows() error {
-	rows := ctx.Rows.Rows
-	ic := &ctx.Common
-	ic.Reset(len(rows))
+	ctx.Reset(len(rows))
 	for i := range rows {
 		r := &rows[i]
-		ic.Labels = ic.Labels[:0]
-		ic.AddLabel("", r.Metric)
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			ic.AddLabel(tag.Key, tag.Value)
+			ctx.AddLabel(tag.Key, tag.Value)
 		}
-		ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, r.Value)
+		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
 	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
-	return ic.FlushBufs()
+	return ctx.FlushBufs()
 }
-
-const flushTimeout = 3 * time.Second
-
-func (ctx *pushCtx) Read(r io.Reader) bool {
-	readCalls.Inc()
-	if ctx.err != nil {
-		return false
-	}
-	if c, ok := r.(net.Conn); ok {
-		if err := c.SetReadDeadline(time.Now().Add(flushTimeout)); err != nil {
-			readErrors.Inc()
-			ctx.err = fmt.Errorf("cannot set read deadline: %s", err)
-			return false
-		}
-	}
-	ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlock(r, ctx.reqBuf, ctx.tailBuf)
-	if ctx.err != nil {
-		if ne, ok := ctx.err.(net.Error); ok && ne.Timeout() {
-			// Flush the read data on timeout and try reading again.
-			ctx.err = nil
-		} else {
-			if ctx.err != io.EOF {
-				readErrors.Inc()
-				ctx.err = fmt.Errorf("cannot read graphite plaintext protocol data: %s", ctx.err)
-			}
-			return false
-		}
-	}
-	ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf))
-
-	// Fill missing timestamps with the current timestamp rounded to seconds.
-	currentTimestamp := time.Now().Unix()
-	rows := ctx.Rows.Rows
-	for i := range rows {
-		r := &rows[i]
-		if r.Timestamp == 0 {
-			r.Timestamp = currentTimestamp
-		}
-	}
-
-	// Convert timestamps from seconds to milliseconds.
-	for i := range rows {
-		rows[i].Timestamp *= 1e3
-	}
-
-	return true
-}
-
-type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf  []byte
-	tailBuf []byte
-
-	err error
-}
-
-func (ctx *pushCtx) Error() error {
-	if ctx.err == io.EOF {
-		return nil
-	}
-	return ctx.err
-}
-
-func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
-	ctx.Common.Reset(0)
-	ctx.reqBuf = ctx.reqBuf[:0]
-	ctx.tailBuf = ctx.tailBuf[:0]
-
-	ctx.err = nil
-}
-
-var (
-	readCalls  = metrics.NewCounter(`vm_read_calls_total{name="graphite"}`)
-	readErrors = metrics.NewCounter(`vm_read_errors_total{name="graphite"}`)
-)
-
-func getPushCtx() *pushCtx {
-	select {
-	case ctx := <-pushCtxPoolCh:
-		return ctx
-	default:
-		if v := pushCtxPool.Get(); v != nil {
-			return v.(*pushCtx)
-		}
-		return &pushCtx{}
-	}
-}
-
-func putPushCtx(ctx *pushCtx) {
-	ctx.reset()
-	select {
-	case pushCtxPoolCh <- ctx:
-	default:
-		pushCtxPool.Put(ctx)
-	}
-}
-
-var pushCtxPool sync.Pool
-var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/influx/request_handler.go

@@ -2,84 +2,56 @@ package influx
 
 import (
 	"flag"
-	"fmt"
 	"io"
 	"net/http"
 	"runtime"
 	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for `{measurement}{separator}{field_name}` metric name when inserted via Influx line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses `{measurement}` instead of `{measurement}{separator}{field_name}` for metic name if Influx line contains only a single field")
+	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via Influx line protocol")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if Influx line contains only a single field")
 )
 
 var (
 	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="influx"}`)
-	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="influx"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="influx"}`)
 )
 
-// InsertHandler processes remote write for influx line protocol.
+// InsertHandlerForReader processes remote write for influx line protocol.
 //
-// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
-func InsertHandler(req *http.Request) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(req)
+// See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
+func InsertHandlerForReader(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, false, "", "", insertRows)
 	})
 }
 
-func insertHandlerInternal(req *http.Request) error {
-	readCalls.Inc()
-
-	r := req.Body
-	if req.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(r)
-		if err != nil {
-			return fmt.Errorf("cannot read gzipped influx line protocol data: %s", err)
-		}
-		defer common.PutGzipReader(zr)
-		r = zr
-	}
-
-	q := req.URL.Query()
-	tsMultiplier := int64(1e6)
-	switch q.Get("precision") {
-	case "ns":
-		tsMultiplier = 1e6
-	case "u":
-		tsMultiplier = 1e3
-	case "ms":
-		tsMultiplier = 1
-	case "s":
-		tsMultiplier = -1e3
-	case "m":
-		tsMultiplier = -1e3 * 60
-	case "h":
-		tsMultiplier = -1e3 * 3600
-	}
-
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
-	db := q.Get("db")
-
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-	for ctx.Read(r, tsMultiplier) {
-		if err := ctx.InsertRows(db); err != nil {
-			return err
-		}
-	}
-	return ctx.Error()
+// InsertHandlerForHTTP processes remote write for influx line protocol.
+//
+// See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
+func InsertHandlerForHTTP(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		q := req.URL.Query()
+		precision := q.Get("precision")
+		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+		db := q.Get("db")
+		return parser.ParseStream(req.Body, isGzipped, precision, db, insertRows)
+	})
 }
 
-func (ctx *pushCtx) InsertRows(db string) error {
-	rows := ctx.Rows.Rows
+func insertRows(db string, rows []parser.Row) error {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
 	rowsLen := 0
 	for i := range rows {
 		rowsLen += len(rows[i].Fields)
@@ -125,80 +97,16 @@ func (ctx *pushCtx) InsertRows(db string) error {
 	return ic.FlushBufs()
 }
 
-func (ctx *pushCtx) Read(r io.Reader, tsMultiplier int64) bool {
-	if ctx.err != nil {
-		return false
-	}
-	ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlock(r, ctx.reqBuf, ctx.tailBuf)
-	if ctx.err != nil {
-		if ctx.err != io.EOF {
-			readErrors.Inc()
-			ctx.err = fmt.Errorf("cannot read influx line protocol data: %s", ctx.err)
-		}
-		return false
-	}
-	ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf))
-
-	// Adjust timestamps according to tsMultiplier
-	currentTs := time.Now().UnixNano() / 1e6
-	if tsMultiplier >= 1 {
-		for i := range ctx.Rows.Rows {
-			row := &ctx.Rows.Rows[i]
-			if row.Timestamp == 0 {
-				row.Timestamp = currentTs
-			} else {
-				row.Timestamp /= tsMultiplier
-			}
-		}
-	} else if tsMultiplier < 0 {
-		tsMultiplier = -tsMultiplier
-		currentTs -= currentTs % tsMultiplier
-		for i := range ctx.Rows.Rows {
-			row := &ctx.Rows.Rows[i]
-			if row.Timestamp == 0 {
-				row.Timestamp = currentTs
-			} else {
-				row.Timestamp *= tsMultiplier
-			}
-		}
-	}
-	return true
-}
-
-var (
-	readCalls  = metrics.NewCounter(`vm_read_calls_total{name="influx"}`)
-	readErrors = metrics.NewCounter(`vm_read_errors_total{name="influx"}`)
-)
-
 type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf         []byte
-	tailBuf        []byte
+	Common         common.InsertCtx
 	metricNameBuf  []byte
 	metricGroupBuf []byte
-
-	err error
-}
-
-func (ctx *pushCtx) Error() error {
-	if ctx.err == io.EOF {
-		return nil
-	}
-	return ctx.err
 }
 
 func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
 	ctx.Common.Reset(0)
-
-	ctx.reqBuf = ctx.reqBuf[:0]
-	ctx.tailBuf = ctx.tailBuf[:0]
 	ctx.metricNameBuf = ctx.metricNameBuf[:0]
 	ctx.metricGroupBuf = ctx.metricGroupBuf[:0]
-
-	ctx.err = nil
 }
 
 func getPushCtx() *pushCtx {

app/vminsert/main.go

@@ -6,20 +6,28 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prompush"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
+	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
+	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
+	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for Influx line protocol data. Usually :8189 must be set. Doesn't work if empty")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty")
@@ -28,29 +36,38 @@ var (
 )
 
 var (
-	graphiteServer     *graphite.Server
-	opentsdbServer     *opentsdb.Server
-	opentsdbhttpServer *opentsdbhttp.Server
+	influxServer       *influxserver.Server
+	graphiteServer     *graphiteserver.Server
+	opentsdbServer     *opentsdbserver.Server
+	opentsdbhttpServer *opentsdbhttpserver.Server
 )
 
 // Init initializes vminsert.
 func Init() {
 	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
 
-	concurrencylimiter.Init()
+	writeconcurrencylimiter.Init()
+	if len(*influxListenAddr) > 0 {
+		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
+	}
 	if len(*graphiteListenAddr) > 0 {
-		graphiteServer = graphite.MustStart(*graphiteListenAddr)
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		opentsdbServer = opentsdb.MustStart(*opentsdbListenAddr)
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
-		opentsdbhttpServer = opentsdbhttp.MustStart(*opentsdbHTTPListenAddr)
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
 	}
+	promscrape.Init(prompush.Push)
 }
 
 // Stop stops vminsert.
 func Stop() {
+	promscrape.Stop()
+	if len(*influxListenAddr) > 0 {
+		influxServer.MustStop()
+	}
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer.MustStop()
 	}
@@ -68,7 +85,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch path {
 	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
-		if err := prometheus.InsertHandler(r); err != nil {
+		if err := promremotewrite.InsertHandler(r); err != nil {
 			prometheusWriteErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
@@ -84,9 +101,18 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/import/csv":
+		csvimportRequests.Inc()
+		if err := csvimport.InsertHandler(r); err != nil {
+			csvimportErrors.Inc()
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
-		if err := influx.InsertHandler(r); err != nil {
+		if err := influx.InsertHandlerForHTTP(r); err != nil {
 			influxWriteErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
@@ -99,6 +125,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		fmt.Fprintf(w, `{"results":[{"series":[{"values":[]}]}]}`)
 		return true
+	case "/targets":
+		promscrapeTargetsRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain")
+		promscrape.WriteHumanReadableTargetsStatus(w)
+		return true
 	default:
 		// This is not our link
 		return false
@@ -106,14 +137,19 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 var (
-	prometheusWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/write", protocol="prometheus"}`)
-	prometheusWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/write", protocol="prometheus"}`)
+	prometheusWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/write", protocol="promremotewrite"}`)
+	prometheusWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/write", protocol="promremotewrite"}`)
 
-	vmimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import", protocol="vm"}`)
-	vmimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import", protocol="vm"}`)
+	vmimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import", protocol="vmimport"}`)
+	vmimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import", protocol="vmimport"}`)
+
+	csvimportRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/import/csv", protocol="csvimport"}`)
+	csvimportErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/import/csv", protocol="csvimport"}`)
 
 	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/write", protocol="influx"}`)
 
 	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/query", protocol="influx"}`)
+
+	promscrapeTargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
 )

app/vminsert/opentsdb/request_handler.go

@@ -1,160 +1,44 @@
 package opentsdb
 
 import (
-	"fmt"
 	"io"
-	"net"
-	"runtime"
-	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdb"}`)
-	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="opentsdb"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="opentsdb"}`)
 )
 
-// insertHandler processes remote write for OpenTSDB put protocol.
+// InsertHandler processes remote write for OpenTSDB put protocol.
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
-func insertHandler(r io.Reader) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(r)
+func InsertHandler(r io.Reader) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
 	})
 }
 
-func insertHandlerInternal(r io.Reader) error {
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-	for ctx.Read(r) {
-		if err := ctx.InsertRows(); err != nil {
-			return err
-		}
-	}
-	return ctx.Error()
-}
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
 
-func (ctx *pushCtx) InsertRows() error {
-	rows := ctx.Rows.Rows
-	ic := &ctx.Common
-	ic.Reset(len(rows))
+	ctx.Reset(len(rows))
 	for i := range rows {
 		r := &rows[i]
-		ic.Labels = ic.Labels[:0]
-		ic.AddLabel("", r.Metric)
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			ic.AddLabel(tag.Key, tag.Value)
+			ctx.AddLabel(tag.Key, tag.Value)
 		}
-		ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, r.Value)
+		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
 	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
-	return ic.FlushBufs()
+	return ctx.FlushBufs()
 }
-
-const flushTimeout = 3 * time.Second
-
-func (ctx *pushCtx) Read(r io.Reader) bool {
-	readCalls.Inc()
-	if ctx.err != nil {
-		return false
-	}
-	if c, ok := r.(net.Conn); ok {
-		if err := c.SetReadDeadline(time.Now().Add(flushTimeout)); err != nil {
-			readErrors.Inc()
-			ctx.err = fmt.Errorf("cannot set read deadline: %s", err)
-			return false
-		}
-	}
-	ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlock(r, ctx.reqBuf, ctx.tailBuf)
-	if ctx.err != nil {
-		if ne, ok := ctx.err.(net.Error); ok && ne.Timeout() {
-			// Flush the read data on timeout and try reading again.
-			ctx.err = nil
-		} else {
-			if ctx.err != io.EOF {
-				readErrors.Inc()
-				ctx.err = fmt.Errorf("cannot read OpenTSDB put protocol data: %s", ctx.err)
-			}
-			return false
-		}
-	}
-	ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf))
-
-	// Fill in missing timestamps
-	currentTimestamp := time.Now().Unix()
-	rows := ctx.Rows.Rows
-	for i := range rows {
-		r := &rows[i]
-		if r.Timestamp == 0 {
-			r.Timestamp = currentTimestamp
-		}
-	}
-
-	// Convert timestamps from seconds to milliseconds
-	for i := range rows {
-		rows[i].Timestamp *= 1e3
-	}
-	return true
-}
-
-type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf  []byte
-	tailBuf []byte
-
-	err error
-}
-
-func (ctx *pushCtx) Error() error {
-	if ctx.err == io.EOF {
-		return nil
-	}
-	return ctx.err
-}
-
-func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
-	ctx.Common.Reset(0)
-	ctx.reqBuf = ctx.reqBuf[:0]
-	ctx.tailBuf = ctx.tailBuf[:0]
-
-	ctx.err = nil
-}
-
-var (
-	readCalls  = metrics.NewCounter(`vm_read_calls_total{name="opentsdb"}`)
-	readErrors = metrics.NewCounter(`vm_read_errors_total{name="opentsdb"}`)
-)
-
-func getPushCtx() *pushCtx {
-	select {
-	case ctx := <-pushCtxPoolCh:
-		return ctx
-	default:
-		if v := pushCtxPool.Get(); v != nil {
-			return v.(*pushCtx)
-		}
-		return &pushCtx{}
-	}
-}
-
-func putPushCtx(ctx *pushCtx) {
-	ctx.reset()
-	select {
-	case pushCtxPoolCh <- ctx:
-	default:
-		pushCtxPool.Put(ctx)
-	}
-}
-
-var pushCtxPool sync.Pool
-var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/opentsdbhttp/request_handler.go

@@ -1,153 +1,50 @@
 package opentsdbhttp
 
 import (
-	"flag"
 	"fmt"
-	"io"
 	"net/http"
-	"runtime"
-	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
 )
 
-var maxInsertRequestSize = flag.Int("opentsdbhttp.maxInsertRequestSize", 32*1024*1024, "The maximum size of OpenTSDB HTTP put request")
-
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdb-http"}`)
-	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="opentsdb-http"}`)
-
-	readCalls       = metrics.NewCounter(`vm_read_calls_total{name="opentsdb-http"}`)
-	readErrors      = metrics.NewCounter(`vm_read_errors_total{name="opentsdb-http"}`)
-	unmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="opentsdb-http"}`)
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdbhttp"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="opentsdbhttp"}`)
 )
 
-// insertHandler processes HTTP OpenTSDB put requests.
+// InsertHandler processes HTTP OpenTSDB put requests.
 // See http://opentsdb.net/docs/build/html/api_http/put.html
-func insertHandler(req *http.Request) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(req)
-	})
+func InsertHandler(req *http.Request) error {
+	path := req.URL.Path
+	switch path {
+	case "/api/put":
+		return writeconcurrencylimiter.Do(func() error {
+			return parser.ParseStream(req, insertRows)
+		})
+	default:
+		return fmt.Errorf("unexpected path requested on HTTP OpenTSDB server: %q", path)
+	}
 }
 
-func insertHandlerInternal(req *http.Request) error {
-	readCalls.Inc()
+func insertRows(rows []parser.Row) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
 
-	r := req.Body
-	if req.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(r)
-		if err != nil {
-			readErrors.Inc()
-			return fmt.Errorf("cannot read gzipped http protocol data: %s", err)
-		}
-		defer common.PutGzipReader(zr)
-		r = zr
-	}
-
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-
-	// Read the request in ctx.reqBuf
-	lr := io.LimitReader(r, int64(*maxInsertRequestSize)+1)
-	reqLen, err := ctx.reqBuf.ReadFrom(lr)
-	if err != nil {
-		readErrors.Inc()
-		return fmt.Errorf("cannot read HTTP OpenTSDB request: %s", err)
-	}
-	if reqLen > int64(*maxInsertRequestSize) {
-		readErrors.Inc()
-		return fmt.Errorf("too big HTTP OpenTSDB request; mustn't exceed `-opentsdbhttp.maxInsertRequestSize=%d` bytes", *maxInsertRequestSize)
-	}
-
-	// Unmarshal the request to ctx.Rows
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-	v, err := p.ParseBytes(ctx.reqBuf.B)
-	if err != nil {
-		unmarshalErrors.Inc()
-		return fmt.Errorf("cannot parse HTTP OpenTSDB json: %s", err)
-	}
-	ctx.Rows.Unmarshal(v)
-
-	// Fill in missing timestamps
-	currentTimestamp := time.Now().Unix()
-	rows := ctx.Rows.Rows
+	ctx.Reset(len(rows))
 	for i := range rows {
 		r := &rows[i]
-		if r.Timestamp == 0 {
-			r.Timestamp = currentTimestamp
-		}
-	}
-
-	// Convert timestamps in seconds to milliseconds if needed.
-	// See http://opentsdb.net/docs/javadoc/net/opentsdb/core/Const.html#SECOND_MASK
-	for i := range rows {
-		r := &rows[i]
-		if r.Timestamp&secondMask == 0 {
-			r.Timestamp *= 1e3
-		}
-	}
-
-	// Insert ctx.Rows to db.
-	ic := &ctx.Common
-	ic.Reset(len(rows))
-	for i := range rows {
-		r := &rows[i]
-		ic.Labels = ic.Labels[:0]
-		ic.AddLabel("", r.Metric)
+		ctx.Labels = ctx.Labels[:0]
+		ctx.AddLabel("", r.Metric)
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			ic.AddLabel(tag.Key, tag.Value)
+			ctx.AddLabel(tag.Key, tag.Value)
 		}
-		ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, r.Value)
+		ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value)
 	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
-	return ic.FlushBufs()
+	return ctx.FlushBufs()
 }
-
-const secondMask int64 = 0x7FFFFFFF00000000
-
-var parserPool fastjson.ParserPool
-
-type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf bytesutil.ByteBuffer
-}
-
-func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
-	ctx.Common.Reset(0)
-	ctx.reqBuf.Reset()
-}
-
-func getPushCtx() *pushCtx {
-	select {
-	case ctx := <-pushCtxPoolCh:
-		return ctx
-	default:
-		if v := pushCtxPool.Get(); v != nil {
-			return v.(*pushCtx)
-		}
-		return &pushCtx{}
-	}
-}
-
-func putPushCtx(ctx *pushCtx) {
-	ctx.reset()
-	select {
-	case pushCtxPoolCh <- ctx:
-	default:
-		pushCtxPool.Put(ctx)
-	}
-}
-
-var pushCtxPool sync.Pool
-var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/prometheus/request_handler.go

@@ -1,112 +0,0 @@
-package prometheus
-
-import (
-	"fmt"
-	"net/http"
-	"runtime"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="prometheus"}`)
-	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="prometheus"}`)
-)
-
-// InsertHandler processes remote write for prometheus.
-func InsertHandler(r *http.Request) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(r)
-	})
-}
-
-func insertHandlerInternal(r *http.Request) error {
-	ctx := getPushCtx()
-	defer putPushCtx(ctx)
-	if err := ctx.Read(r); err != nil {
-		return err
-	}
-	timeseries := ctx.req.Timeseries
-	rowsLen := 0
-	for i := range timeseries {
-		rowsLen += len(timeseries[i].Samples)
-	}
-	ic := &ctx.Common
-	ic.Reset(rowsLen)
-	rowsTotal := 0
-	for i := range timeseries {
-		ts := &timeseries[i]
-		var metricNameRaw []byte
-		for i := range ts.Samples {
-			r := &ts.Samples[i]
-			metricNameRaw = ic.WriteDataPointExt(metricNameRaw, ts.Labels, r.Timestamp, r.Value)
-		}
-		rowsTotal += len(ts.Samples)
-	}
-	rowsInserted.Add(rowsTotal)
-	rowsPerInsert.Update(float64(rowsTotal))
-	return ic.FlushBufs()
-}
-
-type pushCtx struct {
-	Common common.InsertCtx
-
-	req    prompb.WriteRequest
-	reqBuf []byte
-}
-
-func (ctx *pushCtx) reset() {
-	ctx.Common.Reset(0)
-	ctx.req.Reset()
-	ctx.reqBuf = ctx.reqBuf[:0]
-}
-
-func (ctx *pushCtx) Read(r *http.Request) error {
-	prometheusReadCalls.Inc()
-
-	var err error
-	ctx.reqBuf, err = prompb.ReadSnappy(ctx.reqBuf[:0], r.Body)
-	if err != nil {
-		prometheusReadErrors.Inc()
-		return fmt.Errorf("cannot read prompb.WriteRequest: %s", err)
-	}
-	if err = ctx.req.Unmarshal(ctx.reqBuf); err != nil {
-		prometheusUnmarshalErrors.Inc()
-		return fmt.Errorf("cannot unmarshal prompb.WriteRequest with size %d bytes: %s", len(ctx.reqBuf), err)
-	}
-	return nil
-}
-
-var (
-	prometheusReadCalls       = metrics.NewCounter(`vm_read_calls_total{name="prometheus"}`)
-	prometheusReadErrors      = metrics.NewCounter(`vm_read_errors_total{name="prometheus"}`)
-	prometheusUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="prometheus"}`)
-)
-
-func getPushCtx() *pushCtx {
-	select {
-	case ctx := <-pushCtxPoolCh:
-		return ctx
-	default:
-		if v := pushCtxPool.Get(); v != nil {
-			return v.(*pushCtx)
-		}
-		return &pushCtx{}
-	}
-}
-
-func putPushCtx(ctx *pushCtx) {
-	ctx.reset()
-	select {
-	case pushCtxPoolCh <- ctx:
-	default:
-		pushCtxPool.Put(ctx)
-	}
-}
-
-var pushCtxPool sync.Pool
-var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/prompush/push.go

@@ -0,0 +1,113 @@
+package prompush
+
+import (
+	"runtime"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="promscrape"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promscrape"}`)
+)
+
+const maxRowsPerBlock = 10000
+
+// Push pushes wr to storage.
+func Push(wr *prompbmarshal.WriteRequest) {
+	ctx := getPushCtx()
+	defer putPushCtx(ctx)
+
+	tss := wr.Timeseries
+	for len(tss) > 0 {
+		// Process big tss in smaller blocks in order to reduce maxmimum memory usage
+		tssBlock := tss
+		if len(tssBlock) > maxRowsPerBlock {
+			tssBlock = tss[:maxRowsPerBlock]
+			tss = tss[maxRowsPerBlock:]
+		} else {
+			tss = nil
+		}
+		ctx.push(tssBlock)
+	}
+}
+
+func (ctx *pushCtx) push(tss []prompbmarshal.TimeSeries) {
+	rowsLen := 0
+	for i := range tss {
+		rowsLen += len(tss[i].Samples)
+	}
+	ic := &ctx.Common
+	ic.Reset(rowsLen)
+	rowsTotal := 0
+	labels := ctx.labels[:0]
+	for i := range tss {
+		ts := &tss[i]
+		labels = labels[:0]
+		for j := range ts.Labels {
+			label := &ts.Labels[j]
+			labels = append(labels, prompb.Label{
+				Name:  bytesutil.ToUnsafeBytes(label.Name),
+				Value: bytesutil.ToUnsafeBytes(label.Value),
+			})
+		}
+		var metricNameRaw []byte
+		for i := range ts.Samples {
+			r := &ts.Samples[i]
+			metricNameRaw = ic.WriteDataPointExt(metricNameRaw, labels, r.Timestamp, r.Value)
+		}
+		rowsTotal += len(ts.Samples)
+	}
+	ctx.labels = labels
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	if err := ic.FlushBufs(); err != nil {
+		logger.Errorf("cannot flush promscrape data to storage: %s", err)
+	}
+}
+
+type pushCtx struct {
+	Common common.InsertCtx
+	labels []prompb.Label
+}
+
+func (ctx *pushCtx) reset() {
+	ctx.Common.Reset(0)
+
+	for i := range ctx.labels {
+		label := &ctx.labels[i]
+		label.Name = nil
+		label.Value = nil
+	}
+	ctx.labels = ctx.labels[:0]
+}
+
+func getPushCtx() *pushCtx {
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
+	}
+}
+
+func putPushCtx(ctx *pushCtx) {
+	ctx.reset()
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
+}
+
+var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1))

app/vminsert/promremotewrite/request_handler.go

@@ -0,0 +1,47 @@
+package promremotewrite
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promremotewrite"}`)
+)
+
+// InsertHandler processes remote write for prometheus.
+func InsertHandler(req *http.Request) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
+	})
+}
+
+func insertRows(timeseries []prompb.TimeSeries) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	rowsLen := 0
+	for i := range timeseries {
+		rowsLen += len(timeseries[i].Samples)
+	}
+	ctx.Reset(rowsLen)
+	rowsTotal := 0
+	for i := range timeseries {
+		ts := &timeseries[i]
+		var metricNameRaw []byte
+		for i := range ts.Samples {
+			r := &ts.Samples[i]
+			metricNameRaw = ctx.WriteDataPointExt(metricNameRaw, ts.Labels, r.Timestamp, r.Value)
+		}
+		rowsTotal += len(ts.Samples)
+	}
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vminsert/vmimport/request_handler.go

@@ -1,61 +1,35 @@
 package vmimport
 
 import (
-	"flag"
-	"fmt"
-	"io"
 	"net/http"
 	"runtime"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
-var maxLineLen = flag.Int("import.maxLineLen", 100*1024*1024, "The maximum length in bytes of a single line accepted by `/api/v1/import`")
-
 var (
 	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="vmimport"}`)
-	rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="vmimport"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="vmimport"}`)
 )
 
 // InsertHandler processes `/api/v1/import` request.
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
 func InsertHandler(req *http.Request) error {
-	return concurrencylimiter.Do(func() error {
-		return insertHandlerInternal(req)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, insertRows)
 	})
 }
 
-func insertHandlerInternal(req *http.Request) error {
-	readCalls.Inc()
-
-	r := req.Body
-	if req.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(r)
-		if err != nil {
-			return fmt.Errorf("cannot read gzipped vmimport data: %s", err)
-		}
-		defer common.PutGzipReader(zr)
-		r = zr
-	}
-
+func insertRows(rows []parser.Row) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
-	for ctx.Read(r) {
-		if err := ctx.InsertRows(); err != nil {
-			return err
-		}
-	}
-	return ctx.Error()
-}
 
-func (ctx *pushCtx) InsertRows() error {
-	rows := ctx.Rows.Rows
 	rowsLen := 0
 	for i := range rows {
 		rowsLen += len(rows[i].Values)
@@ -85,54 +59,14 @@ func (ctx *pushCtx) InsertRows() error {
 	return ic.FlushBufs()
 }
 
-func (ctx *pushCtx) Read(r io.Reader) bool {
-	if ctx.err != nil {
-		return false
-	}
-	ctx.reqBuf, ctx.tailBuf, ctx.err = common.ReadLinesBlockExt(r, ctx.reqBuf, ctx.tailBuf, *maxLineLen)
-	if ctx.err != nil {
-		if ctx.err != io.EOF {
-			readErrors.Inc()
-			ctx.err = fmt.Errorf("cannot read vmimport data: %s", ctx.err)
-		}
-		return false
-	}
-	ctx.Rows.Unmarshal(bytesutil.ToUnsafeString(ctx.reqBuf))
-	return true
-}
-
-var (
-	readCalls  = metrics.NewCounter(`vm_read_calls_total{name="vmimport"}`)
-	readErrors = metrics.NewCounter(`vm_read_errors_total{name="vmimport"}`)
-)
-
 type pushCtx struct {
-	Rows   Rows
-	Common common.InsertCtx
-
-	reqBuf        []byte
-	tailBuf       []byte
+	Common        common.InsertCtx
 	metricNameBuf []byte
-
-	err error
-}
-
-func (ctx *pushCtx) Error() error {
-	if ctx.err == io.EOF {
-		return nil
-	}
-	return ctx.err
 }
 
 func (ctx *pushCtx) reset() {
-	ctx.Rows.Reset()
 	ctx.Common.Reset(0)
-
-	ctx.reqBuf = ctx.reqBuf[:0]
-	ctx.tailBuf = ctx.tailBuf[:0]
 	ctx.metricNameBuf = ctx.metricNameBuf[:0]
-
-	ctx.err = nil
 }
 
 func getPushCtx() *pushCtx {

app/vmrestore/Makefile

@@ -3,6 +3,9 @@
 vmrestore:
 	APP_NAME=vmrestore $(MAKE) app-local
 
+vmrestore-race:
+	APP_NAME=vmrestore RACE=-race $(MAKE) app-local
+
 vmrestore-prod:
 	APP_NAME=vmrestore $(MAKE) app-via-docker
 

app/vmrestore/README.md

@@ -47,12 +47,24 @@ Run `vmrestore -help` in order to see all the available options:
     	See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html
   -customS3Endpoint string
     	Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set
+  -envflag.enable
+    	Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set
+  -envflag.prefix string
+    	Prefix for environment variables if -envflag.enable is set
+  -fs.disableMmap
+    	Whether to use pread() instead of mmap() for reading data files
+  -loggerFormat string
+    	Format for logs. Possible values: default, json (default "default")
   -loggerLevel string
-    	Minimum level of errors to log. Possible values: INFO, ERROR, FATAL, PANIC (default "INFO")
+    	Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
+  -loggerOutput string
+    	Output for the logs. Supported values: stderr, stdout (default "stderr")
   -maxBytesPerSecond int
     	The maximum download speed. There is no limit if it is set to 0
   -memory.allowedPercent float
-    	Allowed percent of system memory VictoriaMetrics caches may occupy (default 60)
+    	Allowed percent of system memory VictoriaMetrics caches may occupy. Too low value may increase cache miss rate, which usually results in higher CPU and disk IO usage. Too high value may evict too much data from OS page cache, which will result in higher disk IO usage (default 60)
+  -skipBackupCompleteCheck
+    	Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file
   -src string
     	Source path with backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
   -storageDataPath string
@@ -69,7 +81,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 
 #### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.12.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.13.
 2. Run `make vmrestore` from the root folder of the repository.
    It builds `vmrestore` binary and puts it into the `bin` folder.
 
@@ -84,3 +96,10 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 Run `make package-vmrestore`. It builds `victoriametrics/vmrestore:<PKG_TAG>` docker image locally.
 `<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
 The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmrestore`.
+
+By default the image is built on top of `scratch` image. It is possible to build the package on top of any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of `alpine:3.11` image:
+
+```bash
+ROOT_IMAGE=alpine:3.11 make package-vmrestore
+```

app/vmrestore/deployment/Dockerfile

@@ -1,7 +1,6 @@
-ARG certs_image
-FROM $certs_image AS certs
-FROM scratch
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+ARG base_image
+FROM $base_image
+
+ENTRYPOINT ["/vmrestore-prod"]
 ARG src_binary
 COPY $src_binary ./vmrestore-prod
-ENTRYPOINT ["/vmrestore-prod"]

app/vmrestore/main.go

@@ -8,6 +8,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fslocal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -18,12 +19,12 @@ var (
 		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case only missing data is downloaded from backup")
 	concurrency             = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce restore duration")
 	maxBytesPerSecond       = flag.Int("maxBytesPerSecond", 0, "The maximum download speed. There is no limit if it is set to 0")
-	skipBackupCompleteCheck = flag.Bool("skipBackupCompleteCheck", false, "Whether to skip checking for `backup complete` file in `-src`. This may be useful for restoring from old backups, which were created without `backup complete` file")
+	skipBackupCompleteCheck = flag.Bool("skipBackupCompleteCheck", false, "Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file")
 )
 
 func main() {
 	flag.Usage = usage
-	flag.Parse()
+	envflag.Parse()
 	buildinfo.Init()
 
 	srcFS, err := newSrcFS()

app/vmselect/main.go

@@ -8,11 +8,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
@@ -21,15 +19,28 @@ import (
 
 var (
 	deleteAuthKey         = flag.String("deleteAuthKey", "", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series")
-	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", runtime.GOMAXPROCS(-1)*2, "The maximum number of concurrent search requests. It shouldn't exceed 2*vCPUs for better performance. See also -search.maxQueueDuration")
-	maxQueueDuration      = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached")
+	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
+		"It shouldn't be high, since a single request can saturate all the CPU cores. See also -search.maxQueueDuration")
+	maxQueueDuration  = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests limit is reached")
+	resetCacheAuthKey = flag.String("search.resetCacheAuthKey", "", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call")
 )
 
+func getDefaultMaxConcurrentRequests() int {
+	n := runtime.GOMAXPROCS(-1)
+	if n <= 4 {
+		n *= 2
+	}
+	if n > 16 {
+		// A single request can saturate all the CPU cores, so there is no sense
+		// in allowing higher number of concurrent requests - they will just contend
+		// for unavailable CPU time.
+		n = 16
+	}
+	return n
+}
+
 // Init initializes vmselect
 func Init() {
-	tmpDirPath := *vmstorage.DataPath + "/tmp"
-	fs.RemoveDirContents(tmpDirPath)
-	netstorage.InitTmpBlocksDir(tmpDirPath)
 	promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult")
 
 	concurrencyCh = make(chan struct{}, *maxConcurrentRequests)
@@ -56,6 +67,7 @@ var (
 
 // RequestHandler handles remote read API requests for Prometheus
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+	startTime := time.Now()
 	// Limit the number of concurrent queries.
 	select {
 	case concurrencyCh <- struct{}{}:
@@ -72,7 +84,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			timerpool.Put(t)
 			concurrencyLimitTimeout.Inc()
 			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("cannot handle more than %d concurrent requests", cap(concurrencyCh)),
+				Err: fmt.Errorf("cannot handle more than %d concurrent search requests during %s; possible solutions: "+
+					"increase `-search.maxQueueDuration`, increase `-search.maxConcurrentRequests`, increase server capacity",
+					*maxConcurrentRequests, *maxQueueDuration),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, "%s", err)
@@ -81,13 +95,22 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
+	if path == "/internal/resetRollupResultCache" {
+		if len(*resetCacheAuthKey) > 0 && r.FormValue("authKey") != *resetCacheAuthKey {
+			sendPrometheusError(w, r, fmt.Errorf("invalid authKey=%q for %q", r.FormValue("authKey"), path))
+			return true
+		}
+		promql.ResetRollupResultCache()
+		return true
+	}
+
 	if strings.HasPrefix(path, "/api/v1/label/") {
 		s := r.URL.Path[len("/api/v1/label/"):]
 		if strings.HasSuffix(s, "/values") {
 			labelValuesRequests.Inc()
 			labelName := s[:len(s)-len("/values")]
 			httpserver.EnableCORS(w, r)
-			if err := prometheus.LabelValuesHandler(labelName, w, r); err != nil {
+			if err := prometheus.LabelValuesHandler(startTime, labelName, w, r); err != nil {
 				labelValuesErrors.Inc()
 				sendPrometheusError(w, r, err)
 				return true
@@ -100,7 +123,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/query":
 		queryRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.QueryHandler(w, r); err != nil {
+		if err := prometheus.QueryHandler(startTime, w, r); err != nil {
 			queryErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -109,7 +132,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/query_range":
 		queryRangeRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.QueryRangeHandler(w, r); err != nil {
+		if err := prometheus.QueryRangeHandler(startTime, w, r); err != nil {
 			queryRangeErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -118,7 +141,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/series":
 		seriesRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.SeriesHandler(w, r); err != nil {
+		if err := prometheus.SeriesHandler(startTime, w, r); err != nil {
 			seriesErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -127,7 +150,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/series/count":
 		seriesCountRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.SeriesCountHandler(w, r); err != nil {
+		if err := prometheus.SeriesCountHandler(startTime, w, r); err != nil {
 			seriesCountErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -136,7 +159,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/labels":
 		labelsRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.LabelsHandler(w, r); err != nil {
+		if err := prometheus.LabelsHandler(startTime, w, r); err != nil {
 			labelsErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
@@ -145,15 +168,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/api/v1/labels/count":
 		labelsCountRequests.Inc()
 		httpserver.EnableCORS(w, r)
-		if err := prometheus.LabelsCountHandler(w, r); err != nil {
+		if err := prometheus.LabelsCountHandler(startTime, w, r); err != nil {
 			labelsCountErrors.Inc()
 			sendPrometheusError(w, r, err)
 			return true
 		}
 		return true
+	case "/api/v1/status/tsdb":
+		tsdbStatusRequests.Inc()
+		if err := prometheus.TSDBStatusHandler(startTime, w, r); err != nil {
+			tsdbStatusErrors.Inc()
+			sendPrometheusError(w, r, err)
+			return true
+		}
+		return true
 	case "/api/v1/export":
 		exportRequests.Inc()
-		if err := prometheus.ExportHandler(w, r); err != nil {
+		if err := prometheus.ExportHandler(startTime, w, r); err != nil {
 			exportErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
@@ -161,9 +192,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/federate":
 		federateRequests.Inc()
-		if err := prometheus.FederateHandler(w, r); err != nil {
+		if err := prometheus.FederateHandler(startTime, w, r); err != nil {
 			federateErrors.Inc()
-			httpserver.Errorf(w, "error int %q: %s", r.URL.Path, err)
+			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
 		}
 		return true
@@ -179,6 +210,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, "%s", `{"status":"success","data":{"alerts":[]}}`)
 		return true
+	case "/api/v1/metadata":
+		// Return dumb placeholder
+		metadataRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "%s", `{"status":"success","data":{}}`)
+		return true
 	case "/api/v1/admin/tsdb/delete_series":
 		deleteRequests.Inc()
 		authKey := r.FormValue("authKey")
@@ -186,7 +223,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, "invalid authKey %q. It must match the value from -deleteAuthKey command line flag", authKey)
 			return true
 		}
-		if err := prometheus.DeleteHandler(r); err != nil {
+		if err := prometheus.DeleteHandler(startTime, r); err != nil {
 			deleteErrors.Inc()
 			httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err)
 			return true
@@ -199,7 +236,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
-	logger.Errorf("error in %q: %s", r.URL.Path, err)
+	logger.Warnf("error in %q: %s", r.RequestURI, err)
 
 	w.Header().Set("Content-Type", "application/json")
 	statusCode := http.StatusUnprocessableEntity
@@ -232,6 +269,9 @@ var (
 	labelsCountRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/labels/count"}`)
 	labelsCountErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/labels/count"}`)
 
+	tsdbStatusRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/status/tsdb"}`)
+	tsdbStatusErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/status/tsdb"}`)
+
 	deleteRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/admin/tsdb/delete_series"}`)
 	deleteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/admin/tsdb/delete_series"}`)
 
@@ -241,6 +281,7 @@ var (
 	federateRequests = metrics.NewCounter(`vm_http_requests_total{path="/federate"}`)
 	federateErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/federate"}`)
 
-	rulesRequests  = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
-	alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
+	rulesRequests    = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)
+	alertsRequests   = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
+	metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
 )

app/vmselect/netstorage/netstorage.go

@@ -53,9 +53,8 @@ type Results struct {
 	fetchData bool
 	deadline  Deadline
 
-	tbf *tmpBlocksFile
-
 	packedTimeseries []packedTimeseries
+	sr               *storage.Search
 }
 
 // Len returns the number of results in rss.
@@ -65,8 +64,12 @@ func (rss *Results) Len() int {
 
 // Cancel cancels rss work.
 func (rss *Results) Cancel() {
-	putTmpBlocksFile(rss.tbf)
-	rss.tbf = nil
+	rss.mustClose()
+}
+
+func (rss *Results) mustClose() {
+	putStorageSearch(rss.sr)
+	rss.sr = nil
 }
 
 // RunParallel runs in parallel f for all the results from rss.
@@ -76,10 +79,7 @@ func (rss *Results) Cancel() {
 //
 // rss becomes unusable after the call to RunParallel.
 func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
-	defer func() {
-		putTmpBlocksFile(rss.tbf)
-		rss.tbf = nil
-	}()
+	defer rss.mustClose()
 
 	workersCount := 1 + len(rss.packedTimeseries)/32
 	if workersCount > gomaxprocs {
@@ -103,10 +103,10 @@ func (rss *Results) RunParallel(f func(rs *Result, workerID uint)) error {
 			rowsProcessed := 0
 			for pts := range workCh {
 				if time.Until(rss.deadline.Deadline) < 0 {
-					err = fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.Timeout)
+					err = fmt.Errorf("timeout exceeded during query execution: %s", rss.deadline.String())
 					break
 				}
-				if err = pts.Unpack(rss.tbf, rs, rss.tr, rss.fetchData, maxWorkersCount); err != nil {
+				if err = pts.Unpack(rs, rss.tr, rss.fetchData, maxWorkersCount); err != nil {
 					break
 				}
 				if len(rs.Timestamps) == 0 && rss.fetchData {
@@ -156,18 +156,18 @@ var gomaxprocs = runtime.GOMAXPROCS(-1)
 
 type packedTimeseries struct {
 	metricName string
-	addrs      []tmpBlockAddr
+	brs        []storage.BlockRef
 }
 
 // Unpack unpacks pts to dst.
-func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.TimeRange, fetchData bool, maxWorkersCount int) error {
+func (pts *packedTimeseries) Unpack(dst *Result, tr storage.TimeRange, fetchData bool, maxWorkersCount int) error {
 	dst.reset()
 
 	if err := dst.MetricName.Unmarshal(bytesutil.ToUnsafeBytes(pts.metricName)); err != nil {
 		return fmt.Errorf("cannot unmarshal metricName %q: %s", pts.metricName, err)
 	}
 
-	workersCount := 1 + len(pts.addrs)/32
+	workersCount := 1 + len(pts.brs)/32
 	if workersCount > maxWorkersCount {
 		workersCount = maxWorkersCount
 	}
@@ -175,19 +175,19 @@ func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.
 		logger.Panicf("BUG: workersCount cannot be zero")
 	}
 
-	sbs := make([]*sortBlock, 0, len(pts.addrs))
+	sbs := make([]*sortBlock, 0, len(pts.brs))
 	var sbsLock sync.Mutex
 
-	workCh := make(chan tmpBlockAddr, workersCount)
+	workCh := make(chan storage.BlockRef, workersCount)
 	doneCh := make(chan error)
 
 	// Start workers
 	for i := 0; i < workersCount; i++ {
 		go func() {
 			var err error
-			for addr := range workCh {
+			for br := range workCh {
 				sb := getSortBlock()
-				if err = sb.unpackFrom(tbf, addr, tr, fetchData); err != nil {
+				if err = sb.unpackFrom(br, tr, fetchData); err != nil {
 					break
 				}
 
@@ -204,10 +204,10 @@ func (pts *packedTimeseries) Unpack(tbf *tmpBlocksFile, dst *Result, tr storage.
 	}
 
 	// Feed workers with work
-	for _, addr := range pts.addrs {
-		workCh <- addr
+	for _, br := range pts.brs {
+		workCh <- br
 	}
-	pts.addrs = pts.addrs[:0]
+	pts.brs = pts.brs[:0]
 	close(workCh)
 
 	// Wait until workers finish
@@ -266,7 +266,7 @@ func mergeSortBlocks(dst *Result, sbh sortBlocksHeap) {
 			dst.Timestamps = append(dst.Timestamps, top.Timestamps[top.NextIdx:]...)
 			dst.Values = append(dst.Values, top.Values[top.NextIdx:]...)
 			putSortBlock(top)
-			return
+			break
 		}
 		sbNext := sbh[0]
 		tsNext := sbNext.Timestamps[sbNext.NextIdx]
@@ -287,8 +287,16 @@ func mergeSortBlocks(dst *Result, sbh sortBlocksHeap) {
 			putSortBlock(top)
 		}
 	}
+
+	timestamps, values := storage.DeduplicateSamples(dst.Timestamps, dst.Values)
+	dedups := len(dst.Timestamps) - len(timestamps)
+	dedupsDuringSelect.Add(dedups)
+	dst.Timestamps = timestamps
+	dst.Values = values
 }
 
+var dedupsDuringSelect = metrics.NewCounter(`vm_deduplicated_samples_total{type="select"}`)
+
 type sortBlock struct {
 	// b is used as a temporary storage for unpacked rows before they
 	// go to Timestamps and Values.
@@ -306,8 +314,8 @@ func (sb *sortBlock) reset() {
 	sb.NextIdx = 0
 }
 
-func (sb *sortBlock) unpackFrom(tbf *tmpBlocksFile, addr tmpBlockAddr, tr storage.TimeRange, fetchData bool) error {
-	tbf.MustReadBlockAt(&sb.b, addr)
+func (sb *sortBlock) unpackFrom(br storage.BlockRef, tr storage.TimeRange, fetchData bool) error {
+	br.MustReadBlock(&sb.b, fetchData)
 	if fetchData {
 		if err := sb.b.UnmarshalData(); err != nil {
 			return fmt.Errorf("cannot unmarshal block: %s", err)
@@ -441,6 +449,15 @@ func GetLabelEntries(deadline Deadline) ([]storage.TagEntry, error) {
 	return labelEntries, nil
 }
 
+// GetTSDBStatusForDate returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
+func GetTSDBStatusForDate(deadline Deadline, date uint64, topN int) (*storage.TSDBStatus, error) {
+	status, err := vmstorage.GetTSDBStatusForDate(date, topN)
+	if err != nil {
+		return nil, fmt.Errorf("error during tsdb status request: %s", err)
+	}
+	return status, nil
+}
+
 // GetSeriesCount returns the number of unique series.
 func GetSeriesCount(deadline Deadline) (uint64, error) {
 	n, err := vmstorage.GetSeriesCount()
@@ -466,6 +483,8 @@ func putStorageSearch(sr *storage.Search) {
 var ssPool sync.Pool
 
 // ProcessSearchQuery performs sq on storage nodes until the given deadline.
+//
+// Results.RunParallel or Results.Cancel must be called on the returned Results.
 func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadline) (*Results, error) {
 	// Setup search.
 	tfss, err := setupTfss(sq.TagFilterss)
@@ -481,60 +500,40 @@ func ProcessSearchQuery(sq *storage.SearchQuery, fetchData bool, deadline Deadli
 	defer vmstorage.WG.Done()
 
 	sr := getStorageSearch()
-	defer putStorageSearch(sr)
-	sr.Init(vmstorage.Storage, tfss, tr, fetchData, *maxMetricsPerSearch)
+	sr.Init(vmstorage.Storage, tfss, tr, *maxMetricsPerSearch)
 
-	tbf := getTmpBlocksFile()
-	m := make(map[string][]tmpBlockAddr)
+	m := make(map[string][]storage.BlockRef)
+	var orderedMetricNames []string
 	blocksRead := 0
-	bb := tmpBufPool.Get()
-	defer tmpBufPool.Put(bb)
 	for sr.NextMetricBlock() {
 		blocksRead++
-		bb.B = storage.MarshalBlock(bb.B[:0], sr.MetricBlock.Block)
-		addr, err := tbf.WriteBlockData(bb.B)
-		if err != nil {
-			putTmpBlocksFile(tbf)
-			return nil, fmt.Errorf("cannot write data block #%d to temporary blocks file: %s", blocksRead, err)
-		}
 		if time.Until(deadline.Deadline) < 0 {
-			putTmpBlocksFile(tbf)
-			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.Timeout)
+			return nil, fmt.Errorf("timeout exceeded while fetching data block #%d from storage: %s", blocksRead, deadline.String())
 		}
-		metricName := sr.MetricBlock.MetricName
-		m[string(metricName)] = append(m[string(metricName)], addr)
+		metricName := sr.MetricBlockRef.MetricName
+		brs := m[string(metricName)]
+		if len(brs) == 0 {
+			orderedMetricNames = append(orderedMetricNames, string(metricName))
+		}
+		m[string(metricName)] = append(brs, *sr.MetricBlockRef.BlockRef)
 	}
 	if err := sr.Error(); err != nil {
-		putTmpBlocksFile(tbf)
 		return nil, fmt.Errorf("search error after reading %d data blocks: %s", blocksRead, err)
 	}
-	if err := tbf.Finalize(); err != nil {
-		putTmpBlocksFile(tbf)
-		return nil, fmt.Errorf("cannot finalize temporary blocks file with %d blocks: %s", blocksRead, err)
-	}
 
 	var rss Results
-	rss.packedTimeseries = make([]packedTimeseries, len(m))
 	rss.tr = tr
 	rss.fetchData = fetchData
 	rss.deadline = deadline
-	rss.tbf = tbf
-	i := 0
-	for metricName, addrs := range m {
-		pts := &rss.packedTimeseries[i]
-		i++
-		pts.metricName = metricName
-		pts.addrs = addrs
+	pts := make([]packedTimeseries, len(orderedMetricNames))
+	for i, metricName := range orderedMetricNames {
+		pts[i] = packedTimeseries{
+			metricName: metricName,
+			brs:        m[metricName],
+		}
 	}
-
-	// Sort rss.packedTimeseries by the first addr offset in order
-	// to reduce the number of disk seeks during unpacking in RunParallel.
-	// In this case tmpBlocksFile must be read almost sequentially.
-	sort.Slice(rss.packedTimeseries, func(i, j int) bool {
-		pts := rss.packedTimeseries
-		return pts[i].addrs[0].offset < pts[j].addrs[0].offset
-	})
-
+	rss.packedTimeseries = pts
+	rss.sr = sr
 	return &rss, nil
 }
 
@@ -568,6 +567,7 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error)
 			}
 		}
 		tfss = append(tfss, tfs)
+		tfss = append(tfss, tfs.Finalize()...)
 	}
 	return tfss, nil
 }
@@ -575,13 +575,24 @@ func setupTfss(tagFilterss [][]storage.TagFilter) ([]*storage.TagFilters, error)
 // Deadline contains deadline with the corresponding timeout for pretty error messages.
 type Deadline struct {
 	Deadline time.Time
-	Timeout  time.Duration
+
+	timeout  time.Duration
+	flagHint string
 }
 
 // NewDeadline returns deadline for the given timeout.
-func NewDeadline(timeout time.Duration) Deadline {
+//
+// flagHint must contain a hit for command-line flag, which could be used
+// in order to increase timeout.
+func NewDeadline(timeout time.Duration, flagHint string) Deadline {
 	return Deadline{
 		Deadline: time.Now().Add(timeout),
-		Timeout:  timeout,
+		timeout:  timeout,
+		flagHint: flagHint,
 	}
 }
+
+// String returns human-readable string representation for d.
+func (d *Deadline) String() string {
+	return fmt.Sprintf("%.3f seconds; the timeout can be adjusted with `%s` command-line flag", d.timeout.Seconds(), d.flagHint)
+}

app/vmselect/netstorage/tmp_blocks_file.go

@@ -1,179 +0,0 @@
-package netstorage
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// InitTmpBlocksDir initializes directory to store temporary search results.
-//
-// It stores data in system-defined temporary directory if tmpDirPath is empty.
-func InitTmpBlocksDir(tmpDirPath string) {
-	if len(tmpDirPath) == 0 {
-		tmpDirPath = os.TempDir()
-	}
-	tmpBlocksDir = tmpDirPath + "/searchResults"
-	fs.MustRemoveAll(tmpBlocksDir)
-	if err := fs.MkdirAllIfNotExist(tmpBlocksDir); err != nil {
-		logger.Panicf("FATAL: cannot create %q: %s", tmpBlocksDir, err)
-	}
-}
-
-var tmpBlocksDir string
-
-func maxInmemoryTmpBlocksFile() int {
-	mem := memory.Allowed()
-	maxLen := mem / 1024
-	if maxLen < 64*1024 {
-		return 64 * 1024
-	}
-	return maxLen
-}
-
-var _ = metrics.NewGauge(`vm_tmp_blocks_max_inmemory_file_size_bytes`, func() float64 {
-	return float64(maxInmemoryTmpBlocksFile())
-})
-
-type tmpBlocksFile struct {
-	buf []byte
-
-	f *os.File
-
-	offset uint64
-}
-
-func getTmpBlocksFile() *tmpBlocksFile {
-	v := tmpBlocksFilePool.Get()
-	if v == nil {
-		return &tmpBlocksFile{
-			buf: make([]byte, 0, maxInmemoryTmpBlocksFile()),
-		}
-	}
-	return v.(*tmpBlocksFile)
-}
-
-func putTmpBlocksFile(tbf *tmpBlocksFile) {
-	tbf.MustClose()
-	tbf.buf = tbf.buf[:0]
-	tbf.f = nil
-	tbf.offset = 0
-	tmpBlocksFilePool.Put(tbf)
-}
-
-var tmpBlocksFilePool sync.Pool
-
-type tmpBlockAddr struct {
-	offset uint64
-	size   int
-}
-
-func (addr tmpBlockAddr) String() string {
-	return fmt.Sprintf("offset %d, size %d", addr.offset, addr.size)
-}
-
-var tmpBlocksFilesCreated = metrics.NewCounter(`vm_tmp_blocks_files_created_total`)
-
-// WriteBlockData writes b to tbf.
-//
-// It returns errors since the operation may fail on space shortage
-// and this must be handled.
-func (tbf *tmpBlocksFile) WriteBlockData(b []byte) (tmpBlockAddr, error) {
-	var addr tmpBlockAddr
-	addr.offset = tbf.offset
-	addr.size = len(b)
-	tbf.offset += uint64(addr.size)
-	if len(tbf.buf)+len(b) <= cap(tbf.buf) {
-		// Fast path - the data fits tbf.buf
-		tbf.buf = append(tbf.buf, b...)
-		return addr, nil
-	}
-
-	// Slow path: flush the data from tbf.buf to file.
-	if tbf.f == nil {
-		f, err := ioutil.TempFile(tmpBlocksDir, "")
-		if err != nil {
-			return addr, err
-		}
-		tbf.f = f
-		tmpBlocksFilesCreated.Inc()
-	}
-	_, err := tbf.f.Write(tbf.buf)
-	tbf.buf = append(tbf.buf[:0], b...)
-	if err != nil {
-		return addr, fmt.Errorf("cannot write block to %q: %s", tbf.f.Name(), err)
-	}
-	return addr, nil
-}
-
-func (tbf *tmpBlocksFile) Finalize() error {
-	if tbf.f == nil {
-		return nil
-	}
-	if _, err := tbf.f.Write(tbf.buf); err != nil {
-		return fmt.Errorf("cannot flush the remaining %d bytes to tmpBlocksFile: %s", len(tbf.buf), err)
-	}
-	tbf.buf = tbf.buf[:0]
-	if _, err := tbf.f.Seek(0, 0); err != nil {
-		logger.Panicf("FATAL: cannot seek to the start of file: %s", err)
-	}
-	// Hint the OS that the file is read almost sequentiallly.
-	// This should reduce the number of disk seeks, which is important
-	// for HDDs.
-	fs.MustFadviseSequentialRead(tbf.f, true)
-	return nil
-}
-
-func (tbf *tmpBlocksFile) MustReadBlockAt(dst *storage.Block, addr tmpBlockAddr) {
-	var buf []byte
-	if tbf.f == nil {
-		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]
-	} else {
-		bb := tmpBufPool.Get()
-		defer tmpBufPool.Put(bb)
-		bb.B = bytesutil.Resize(bb.B, addr.size)
-		n, err := tbf.f.ReadAt(bb.B, int64(addr.offset))
-		if err != nil {
-			logger.Panicf("FATAL: cannot read from %q at %s: %s", tbf.f.Name(), addr, err)
-		}
-		if n != len(bb.B) {
-			logger.Panicf("FATAL: too short number of bytes read at %s; got %d; want %d", addr, n, len(bb.B))
-		}
-		buf = bb.B
-	}
-	tail, err := storage.UnmarshalBlock(dst, buf)
-	if err != nil {
-		logger.Panicf("FATAL: cannot unmarshal data at %s: %s", addr, err)
-	}
-	if len(tail) > 0 {
-		logger.Panicf("FATAL: unexpected non-empty tail left after unmarshaling data at %s; len(tail)=%d", addr, len(tail))
-	}
-}
-
-var tmpBufPool bytesutil.ByteBufferPool
-
-func (tbf *tmpBlocksFile) MustClose() {
-	if tbf.f == nil {
-		return
-	}
-	fname := tbf.f.Name()
-
-	// Remove the file at first, then close it.
-	// This way the OS shouldn't try to flush file contents to storage
-	// on close.
-	if err := os.Remove(fname); err != nil {
-		logger.Panicf("FATAL: cannot remove %q: %s", fname, err)
-	}
-	if err := tbf.f.Close(); err != nil {
-		logger.Panicf("FATAL: cannot close %q: %s", fname, err)
-	}
-	tbf.f = nil
-}

app/vmselect/netstorage/tmp_blocks_file_test.go

@@ -1,153 +0,0 @@
-package netstorage
-
-import (
-	"fmt"
-	"math/rand"
-	"os"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-)
-
-func TestMain(m *testing.M) {
-	rand.Seed(time.Now().UnixNano())
-	tmpDir := "TestTmpBlocks"
-	InitTmpBlocksDir(tmpDir)
-	statusCode := m.Run()
-	if err := os.RemoveAll(tmpDir); err != nil {
-		logger.Panicf("cannot remove %q: %s", tmpDir, err)
-	}
-	os.Exit(statusCode)
-}
-
-func TestTmpBlocksFileSerial(t *testing.T) {
-	if err := testTmpBlocksFile(); err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-}
-
-func TestTmpBlocksFileConcurrent(t *testing.T) {
-	concurrency := 3
-	ch := make(chan error, concurrency)
-	for i := 0; i < concurrency; i++ {
-		go func() {
-			ch <- testTmpBlocksFile()
-		}()
-	}
-	for i := 0; i < concurrency; i++ {
-		select {
-		case err := <-ch:
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
-		case <-time.After(30 * time.Second):
-			t.Fatalf("timeout")
-		}
-	}
-}
-
-func testTmpBlocksFile() error {
-	createBlock := func() *storage.Block {
-		rowsCount := rand.Intn(8000) + 1
-		var timestamps, values []int64
-		ts := int64(rand.Intn(1023434))
-		for i := 0; i < rowsCount; i++ {
-			ts += int64(rand.Intn(1000) + 1)
-			timestamps = append(timestamps, ts)
-			values = append(values, int64(i*i+rand.Intn(20)))
-		}
-		tsid := &storage.TSID{
-			MetricID: 234211,
-		}
-		scale := int16(rand.Intn(123))
-		precisionBits := uint8(rand.Intn(63) + 1)
-		var b storage.Block
-		b.Init(tsid, timestamps, values, scale, precisionBits)
-		_, _, _ = b.MarshalData(0, 0)
-		return &b
-	}
-	for _, size := range []int{1024, 16 * 1024, maxInmemoryTmpBlocksFile() / 2, 2 * maxInmemoryTmpBlocksFile()} {
-		err := func() error {
-			tbf := getTmpBlocksFile()
-			defer putTmpBlocksFile(tbf)
-
-			// Write blocks until their summary size exceeds `size`.
-			var addrs []tmpBlockAddr
-			var blocks []*storage.Block
-			bb := tmpBufPool.Get()
-			defer tmpBufPool.Put(bb)
-			for tbf.offset < uint64(size) {
-				b := createBlock()
-				bb.B = storage.MarshalBlock(bb.B[:0], b)
-				addr, err := tbf.WriteBlockData(bb.B)
-				if err != nil {
-					return fmt.Errorf("cannot write block at offset %d: %s", tbf.offset, err)
-				}
-				if addr.offset+uint64(addr.size) != tbf.offset {
-					return fmt.Errorf("unexpected addr=%+v for offset %v", &addr, tbf.offset)
-				}
-				addrs = append(addrs, addr)
-				blocks = append(blocks, b)
-			}
-			if err := tbf.Finalize(); err != nil {
-				return fmt.Errorf("cannot finalize tbf: %s", err)
-			}
-
-			// Read blocks in parallel and verify them
-			concurrency := 2
-			workCh := make(chan int)
-			doneCh := make(chan error)
-			for i := 0; i < concurrency; i++ {
-				go func() {
-					doneCh <- func() error {
-						var b1 storage.Block
-						for idx := range workCh {
-							addr := addrs[idx]
-							b := blocks[idx]
-							if err := b.UnmarshalData(); err != nil {
-								return fmt.Errorf("cannot unmarshal data from the original block: %s", err)
-							}
-							b1.Reset()
-							tbf.MustReadBlockAt(&b1, addr)
-							if err := b1.UnmarshalData(); err != nil {
-								return fmt.Errorf("cannot unmarshal data from tbf: %s", err)
-							}
-							if b1.RowsCount() != b.RowsCount() {
-								return fmt.Errorf("unexpected number of rows in tbf block; got %d; want %d", b1.RowsCount(), b.RowsCount())
-							}
-							if !reflect.DeepEqual(b1.Timestamps(), b.Timestamps()) {
-								return fmt.Errorf("unexpected timestamps; got\n%v\nwant\n%v", b1.Timestamps(), b.Timestamps())
-							}
-							if !reflect.DeepEqual(b1.Values(), b.Values()) {
-								return fmt.Errorf("unexpected values; got\n%v\nwant\n%v", b1.Values(), b.Values())
-							}
-						}
-						return nil
-					}()
-				}()
-			}
-			for i := range addrs {
-				workCh <- i
-			}
-			close(workCh)
-			for i := 0; i < concurrency; i++ {
-				select {
-				case err := <-doneCh:
-					if err != nil {
-						return err
-					}
-				case <-time.After(time.Second):
-					return fmt.Errorf("timeout")
-				}
-			}
-			return nil
-		}()
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

app/vmselect/prometheus/prometheus.go

@@ -3,6 +3,7 @@ package prometheus
 import (
 	"flag"
 	"fmt"
+	"io"
 	"math"
 	"net/http"
 	"runtime"
@@ -18,25 +19,30 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson/fastfloat"
 	"github.com/valyala/quicktemplate"
 )
 
 var (
 	latencyOffset = flag.Duration("search.latencyOffset", time.Second*30, "The time when data points become visible in query results after the colection. "+
 		"Too small value can result in incomplete last points for query results")
-	maxExportDuration = flag.Duration("search.maxExportDuration", 10*time.Minute, "The maximum duration for `/api/v1/export` call")
+	maxExportDuration = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call")
 	maxQueryDuration  = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for search query execution")
 	maxQueryLen       = flag.Int("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
-	maxLookback       = flag.Duration("search.maxLookback", 0, "Synonim to `-search.lookback-delta` from Prometheus. "+
-		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via `max_lookback` arg")
+	maxLookback       = flag.Duration("search.maxLookback", 0, "Synonim to -search.lookback-delta from Prometheus. "+
+		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. "+
+		"See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons")
+	maxStalenessInterval = flag.Duration("search.maxStalenessInterval", 0, "The maximum interval for staleness calculations. "+
+		"By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning "+
+		"Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. "+
+		"See also '-search.maxLookback' flag, which has the same meanining due to historical reasons")
 )
 
 // Default step used if not set.
 const defaultStep = 5 * 60 * 1000
 
 // FederateHandler implements /federate . See https://prometheus.io/docs/prometheus/latest/federation/
-func FederateHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %s", err)
@@ -107,8 +113,7 @@ func FederateHandler(w http.ResponseWriter, r *http.Request) error {
 var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/federate"}`)
 
 // ExportHandler exports data in raw format from /api/v1/export.
-func ExportHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %s", err)
@@ -131,12 +136,13 @@ func ExportHandler(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 	format := r.FormValue("format")
+	maxRowsPerLine := int(fastfloat.ParseInt64BestEffort(r.FormValue("max_rows_per_line")))
 	deadline := getDeadlineForExport(r)
 	if start >= end {
 		end = start + defaultStep
 	}
-	if err := exportHandler(w, matches, start, end, format, deadline); err != nil {
-		return err
+	if err := exportHandler(w, matches, start, end, format, maxRowsPerLine, deadline); err != nil {
+		return fmt.Errorf("error when exporting data for queries=%q on the time range (start=%d, end=%d): %s", matches, start, end, err)
 	}
 	exportDuration.UpdateDuration(startTime)
 	return nil
@@ -144,9 +150,37 @@ func ExportHandler(w http.ResponseWriter, r *http.Request) error {
 
 var exportDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export"}`)
 
-func exportHandler(w http.ResponseWriter, matches []string, start, end int64, format string, deadline netstorage.Deadline) error {
+func exportHandler(w http.ResponseWriter, matches []string, start, end int64, format string, maxRowsPerLine int, deadline netstorage.Deadline) error {
 	writeResponseFunc := WriteExportStdResponse
 	writeLineFunc := WriteExportJSONLine
+	if maxRowsPerLine > 0 {
+		writeLineFunc = func(w io.Writer, rs *netstorage.Result) {
+			valuesOrig := rs.Values
+			timestampsOrig := rs.Timestamps
+			values := valuesOrig
+			timestamps := timestampsOrig
+			for len(values) > 0 {
+				var valuesChunk []float64
+				var timestampsChunk []int64
+				if len(values) > maxRowsPerLine {
+					valuesChunk = values[:maxRowsPerLine]
+					timestampsChunk = timestamps[:maxRowsPerLine]
+					values = values[maxRowsPerLine:]
+					timestamps = timestamps[maxRowsPerLine:]
+				} else {
+					valuesChunk = values
+					timestampsChunk = timestamps
+					values = nil
+					timestamps = nil
+				}
+				rs.Values = valuesChunk
+				rs.Timestamps = timestampsChunk
+				WriteExportJSONLine(w, rs)
+			}
+			rs.Values = valuesOrig
+			rs.Timestamps = timestampsOrig
+		}
+	}
 	contentType := "application/stream+json"
 	if format == "prometheus" {
 		contentType = "text/plain"
@@ -200,8 +234,7 @@ func exportHandler(w http.ResponseWriter, matches []string, start, end int64, fo
 // DeleteHandler processes /api/v1/admin/tsdb/delete_series prometheus API request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#delete-series
-func DeleteHandler(r *http.Request) error {
-	startTime := time.Now()
+func DeleteHandler(startTime time.Time, r *http.Request) error {
 	if err := r.ParseForm(); err != nil {
 		return fmt.Errorf("cannot parse request form values: %s", err)
 	}
@@ -235,8 +268,7 @@ var deleteDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 // LabelValuesHandler processes /api/v1/label/<labelName>/values request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
-func LabelValuesHandler(labelName string, w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func LabelValuesHandler(startTime time.Time, labelName string, w http.ResponseWriter, r *http.Request) error {
 	deadline := getDeadlineForQuery(r)
 
 	if err := r.ParseForm(); err != nil {
@@ -287,12 +319,18 @@ func labelValuesWithMatches(labelName string, matches []string, start, end int64
 	if err != nil {
 		return nil, err
 	}
-	for i, tfs := range tagFilterss {
-		// Add `labelName!=''` tag filter in order to filter out series without the labelName.
-		tagFilterss[i] = append(tfs, storage.TagFilter{
-			Key:        []byte(labelName),
-			IsNegative: true,
-		})
+
+	// Add `labelName!=''` tag filter in order to filter out series without the labelName.
+	// There is no need in adding `__name__!=''` filter, since all the time series should
+	// already have non-empty name.
+	if labelName != "__name__" {
+		key := []byte(labelName)
+		for i, tfs := range tagFilterss {
+			tagFilterss[i] = append(tfs, storage.TagFilter{
+				Key:        key,
+				IsNegative: true,
+			})
+		}
 	}
 	if start >= end {
 		end = start + defaultStep
@@ -333,14 +371,12 @@ func labelValuesWithMatches(labelName string, matches []string, start, end int64
 var labelValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/label/{}/values"}`)
 
 // LabelsCountHandler processes /api/v1/labels/count request.
-func LabelsCountHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func LabelsCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := getDeadlineForQuery(r)
 	labelEntries, err := netstorage.GetLabelEntries(deadline)
 	if err != nil {
 		return fmt.Errorf(`cannot obtain label entries: %s`, err)
 	}
-
 	w.Header().Set("Content-Type", "application/json")
 	WriteLabelsCountResponse(w, labelEntries)
 	labelsCountDuration.UpdateDuration(startTime)
@@ -349,11 +385,56 @@ func LabelsCountHandler(w http.ResponseWriter, r *http.Request) error {
 
 var labelsCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/labels/count"}`)
 
+const secsPerDay = 3600 * 24
+
+// TSDBStatusHandler processes /api/v1/status/tsdb request.
+//
+// See https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
+func TSDBStatusHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+	deadline := getDeadlineForQuery(r)
+	if err := r.ParseForm(); err != nil {
+		return fmt.Errorf("cannot parse form values: %s", err)
+	}
+	date := time.Now().Unix() / secsPerDay
+	dateStr := r.FormValue("date")
+	if len(dateStr) > 0 {
+		t, err := time.Parse("2006-01-02", dateStr)
+		if err != nil {
+			return fmt.Errorf("cannot parse `date` arg %q: %s", dateStr, err)
+		}
+		date = t.Unix() / secsPerDay
+	}
+	topN := 10
+	topNStr := r.FormValue("topN")
+	if len(topNStr) > 0 {
+		n, err := strconv.Atoi(topNStr)
+		if err != nil {
+			return fmt.Errorf("cannot parse `topN` arg %q: %s", topNStr, err)
+		}
+		if n <= 0 {
+			n = 1
+		}
+		if n > 1000 {
+			n = 1000
+		}
+		topN = n
+	}
+	status, err := netstorage.GetTSDBStatusForDate(deadline, uint64(date), topN)
+	if err != nil {
+		return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %s`, date, topN, err)
+	}
+	w.Header().Set("Content-Type", "application/json")
+	WriteTSDBStatusResponse(w, status)
+	tsdbStatusDuration.UpdateDuration(startTime)
+	return nil
+}
+
+var tsdbStatusDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/status/tsdb"}`)
+
 // LabelsHandler processes /api/v1/labels request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names
-func LabelsHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func LabelsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := getDeadlineForQuery(r)
 
 	if err := r.ParseForm(); err != nil {
@@ -442,8 +523,7 @@ func labelsWithMatches(matches []string, start, end int64, deadline netstorage.D
 var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/labels"}`)
 
 // SeriesCountHandler processes /api/v1/series/count request.
-func SeriesCountHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	deadline := getDeadlineForQuery(r)
 	n, err := netstorage.GetSeriesCount(deadline)
 	if err != nil {
@@ -460,8 +540,7 @@ var seriesCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
 // SeriesHandler processes /api/v1/series request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers
-func SeriesHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func SeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 
 	if err := r.ParseForm(); err != nil {
@@ -536,8 +615,7 @@ var seriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 // QueryHandler processes /api/v1/query request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
-func QueryHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func QueryHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 
 	query := r.FormValue("query")
@@ -548,20 +626,23 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-	queryOffset := getLatencyOffsetMilliseconds()
-	step, err := getDuration(r, "step", queryOffset)
-	if err != nil {
-		return err
-	}
-	deadline := getDeadlineForQuery(r)
 	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
 		return err
 	}
+	step, err := getDuration(r, "step", lookbackDelta)
+	if err != nil {
+		return err
+	}
+	if step <= 0 {
+		step = defaultStep
+	}
+	deadline := getDeadlineForQuery(r)
 
 	if len(query) > *maxQueryLen {
 		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), *maxQueryLen)
 	}
+	queryOffset := getLatencyOffsetMilliseconds()
 	if !getBool(r, "nocache") && ct-start < queryOffset {
 		// Adjust start time only if `nocache` arg isn't set.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/241
@@ -579,8 +660,8 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 		start -= offset
 		end := start
 		start = end - window
-		if err := exportHandler(w, []string{childQuery}, start, end, "promapi", deadline); err != nil {
-			return err
+		if err := exportHandler(w, []string{childQuery}, start, end, "promapi", 0, deadline); err != nil {
+			return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %s", childQuery, start, end, err)
 		}
 		queryDuration.UpdateDuration(startTime)
 		return nil
@@ -605,7 +686,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 		end := start
 		start = end - window
 		if err := queryRangeHandler(w, childQuery, start, end, step, r, ct); err != nil {
-			return err
+			return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %s", childQuery, start, end, step, err)
 		}
 		queryDuration.UpdateDuration(startTime)
 		return nil
@@ -620,7 +701,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) error {
 	}
 	result, err := promql.Exec(&ec, query, true)
 	if err != nil {
-		return fmt.Errorf("cannot execute %q: %s", query, err)
+		return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %s", query, start, step, err)
 	}
 
 	w.Header().Set("Content-Type", "application/json")
@@ -648,8 +729,7 @@ func parsePositiveDuration(s string, step int64) (int64, error) {
 // QueryRangeHandler processes /api/v1/query_range request.
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
-func QueryRangeHandler(w http.ResponseWriter, r *http.Request) error {
-	startTime := time.Now()
+func QueryRangeHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	ct := currentTime()
 
 	query := r.FormValue("query")
@@ -669,7 +749,7 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 	if err := queryRangeHandler(w, query, start, end, step, r, ct); err != nil {
-		return err
+		return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %s", query, start, end, step, err)
 	}
 	queryRangeDuration.UpdateDuration(startTime)
 	return nil
@@ -707,7 +787,7 @@ func queryRangeHandler(w http.ResponseWriter, query string, start, end, step int
 	}
 	result, err := promql.Exec(&ec, query, false)
 	if err != nil {
-		return fmt.Errorf("cannot execute %q: %s", query, err)
+		return fmt.Errorf("cannot execute query: %s", err)
 	}
 	queryOffset := getLatencyOffsetMilliseconds()
 	if ct-end < queryOffset {
@@ -813,7 +893,15 @@ func getTime(r *http.Request, argKey string, defaultValue int64) (int64, error)
 			case prometheusMaxTimeFormatted:
 				return maxTimeMsecs, nil
 			}
-			return 0, fmt.Errorf("cannot parse %q=%q: %s", argKey, argValue, err)
+			// Try parsing duration relative to the current time
+			d, err1 := time.ParseDuration(argValue)
+			if err1 != nil {
+				return 0, fmt.Errorf("cannot parse %q=%q: %s", argKey, argValue, err)
+			}
+			if d > 0 {
+				d = -d
+			}
+			t = time.Now().Add(d)
 		}
 		secs = float64(t.UnixNano()) / 1e9
 	}
@@ -864,21 +952,24 @@ func getDuration(r *http.Request, argKey string, defaultValue int64) (int64, err
 const maxDurationMsecs = 100 * 365 * 24 * 3600 * 1000
 
 func getMaxLookback(r *http.Request) (int64, error) {
-	d := int64(*maxLookback / time.Millisecond)
+	d := maxLookback.Milliseconds()
+	if d == 0 {
+		d = maxStalenessInterval.Milliseconds()
+	}
 	return getDuration(r, "max_lookback", d)
 }
 
 func getDeadlineForQuery(r *http.Request) netstorage.Deadline {
-	dMax := int64(maxQueryDuration.Seconds() * 1e3)
-	return getDeadlineWithMaxDuration(r, dMax)
+	dMax := maxQueryDuration.Milliseconds()
+	return getDeadlineWithMaxDuration(r, dMax, "-search.maxQueryDuration")
 }
 
 func getDeadlineForExport(r *http.Request) netstorage.Deadline {
-	dMax := int64(maxExportDuration.Seconds() * 1e3)
-	return getDeadlineWithMaxDuration(r, dMax)
+	dMax := maxExportDuration.Milliseconds()
+	return getDeadlineWithMaxDuration(r, dMax, "-search.maxExportDuration")
 }
 
-func getDeadlineWithMaxDuration(r *http.Request, dMax int64) netstorage.Deadline {
+func getDeadlineWithMaxDuration(r *http.Request, dMax int64, flagHint string) netstorage.Deadline {
 	d, err := getDuration(r, "timeout", 0)
 	if err != nil {
 		d = 0
@@ -887,7 +978,7 @@ func getDeadlineWithMaxDuration(r *http.Request, dMax int64) netstorage.Deadline
 		d = dMax
 	}
 	timeout := time.Duration(d) * time.Millisecond
-	return netstorage.NewDeadline(timeout)
+	return netstorage.NewDeadline(timeout, flagHint)
 }
 
 func getBool(r *http.Request, argKey string) bool {
@@ -917,7 +1008,7 @@ func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error)
 }
 
 func getLatencyOffsetMilliseconds() int64 {
-	d := int64(*latencyOffset / time.Millisecond)
+	d := latencyOffset.Milliseconds()
 	if d <= 1000 {
 		d = 1000
 	}

app/vmselect/prometheus/tsdb_status_response.qtpl

@@ -0,0 +1,28 @@
+{% import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" %}
+
+{% stripspace %}
+TSDBStatusResponse generates response for /api/v1/status/tsdb .
+{% func TSDBStatusResponse(status *storage.TSDBStatus) %}
+{
+	"status":"success",
+	"data":{
+		"seriesCountByMetricName":{%= tsdbStatusEntries(status.SeriesCountByMetricName) %},
+		"labelValueCountByLabelName":{%= tsdbStatusEntries(status.LabelValueCountByLabelName) %},
+		"seriesCountByLabelValuePair":{%= tsdbStatusEntries(status.SeriesCountByLabelValuePair) %}
+	}
+}
+{% endfunc %}
+
+{% func tsdbStatusEntries(a []storage.TopHeapEntry) %}
+[
+	{% for i, e := range a %}
+		{
+			"name":{%q= e.Name %},
+			"value":{%d= int(e.Count) %}
+		}
+		{% if i+1 < len(a) %},{% endif %}
+	{% endfor %}
+]
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/prometheus/tsdb_status_response.qtpl.go

@@ -0,0 +1,123 @@
+// Code generated by qtc from "tsdb_status_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:1
+package prometheus
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:1
+import "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+
+// TSDBStatusResponse generates response for /api/v1/status/tsdb .
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
+func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBStatus) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:5
+	qw422016.N().S(`{"status":"success","data":{"seriesCountByMetricName":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
+	streamtsdbStatusEntries(qw422016, status.SeriesCountByMetricName)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
+	qw422016.N().S(`,"labelValueCountByLabelName":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:10
+	streamtsdbStatusEntries(qw422016, status.LabelValueCountByLabelName)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:10
+	qw422016.N().S(`,"seriesCountByLabelValuePair":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:11
+	streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelValuePair)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:11
+	qw422016.N().S(`}}`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+}
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+func WriteTSDBStatusResponse(qq422016 qtio422016.Writer, status *storage.TSDBStatus) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	StreamTSDBStatusResponse(qw422016, status)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+}
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+func TSDBStatusResponse(status *storage.TSDBStatus) string {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	WriteTSDBStatusResponse(qb422016, status)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	return qs422016
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+}
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
+func streamtsdbStatusEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
+	qw422016.N().S(`[`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
+	for i, e := range a {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
+		qw422016.N().S(`{"name":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
+		qw422016.N().Q(e.Name)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
+		qw422016.N().S(`,"value":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+		qw422016.N().D(int(e.Count))
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+		qw422016.N().S(`}`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+		if i+1 < len(a) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+		}
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
+	}
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
+	qw422016.N().S(`]`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+}
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+func writetsdbStatusEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+	streamtsdbStatusEntries(qw422016, a)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+}
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+func tsdbStatusEntries(a []storage.TopHeapEntry) string {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+	writetsdbStatusEntries(qb422016, a)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+	return qs422016
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
+}

app/vmselect/promql/arch.go

@@ -1,5 +0,0 @@
-package promql
-
-import "unsafe"
-
-const maxByteSliceLen = 1<<(31+9*(unsafe.Sizeof(int(0))/8)) - 1

app/vmselect/promql/binary_op.go

@@ -32,7 +32,7 @@ var binaryOpFuncs = map[string]binaryOpFunc{
 	"or":     binaryOpOr,
 	"unless": binaryOpUnless,
 
-	// New op
+	// New ops
 	"if":      newBinaryOpArithFunc(binaryop.If),
 	"ifnot":   newBinaryOpArithFunc(binaryop.Ifnot),
 	"default": newBinaryOpArithFunc(binaryop.Default),
@@ -285,10 +285,21 @@ func resetMetricGroupIfRequired(be *metricsql.BinaryOpExpr, ts *timeseries) {
 func binaryOpAnd(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
 	var rvs []*timeseries
-	for k := range mRight {
-		if tss := mLeft[k]; tss != nil {
-			rvs = append(rvs, tss...)
+	for k, tssRight := range mRight {
+		tssLeft := mLeft[k]
+		if tssLeft == nil {
+			continue
 		}
+		for i := range tssLeft[0].Values {
+			if !isAllNaNs(tssRight, i) {
+				continue
+			}
+			for _, tsLeft := range tssLeft {
+				tsLeft.Values[i] = nan
+			}
+		}
+		tssLeft = removeNaNs(tssLeft)
+		rvs = append(rvs, tssLeft...)
 	}
 	return rvs, nil
 }
@@ -310,14 +321,35 @@ func binaryOpOr(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 func binaryOpUnless(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
 	var rvs []*timeseries
-	for k, tss := range mLeft {
-		if mRight[k] == nil {
-			rvs = append(rvs, tss...)
+	for k, tssLeft := range mLeft {
+		tssRight := mRight[k]
+		if tssRight == nil {
+			rvs = append(rvs, tssLeft...)
+			continue
 		}
+		for i := range tssLeft[0].Values {
+			if isAllNaNs(tssRight, i) {
+				continue
+			}
+			for _, tsLeft := range tssLeft {
+				tsLeft.Values[i] = nan
+			}
+		}
+		tssLeft = removeNaNs(tssLeft)
+		rvs = append(rvs, tssLeft...)
 	}
 	return rvs, nil
 }
 
+func isAllNaNs(tss []*timeseries, idx int) bool {
+	for _, ts := range tss {
+		if !math.IsNaN(ts.Values[idx]) {
+			return false
+		}
+	}
+	return true
+}
+
 func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*timeseries) (map[string][]*timeseries, map[string][]*timeseries) {
 	groupTags := be.GroupModifier.Args
 	groupOp := strings.ToLower(be.GroupModifier.Op)

app/vmselect/promql/eval.go

@@ -419,7 +419,7 @@ func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.E
 		if err != nil {
 			return nil, err
 		}
-		ecNew = newEvalConfig(ec)
+		ecNew = newEvalConfig(ecNew)
 		ecNew.Start -= offset
 		ecNew.End -= offset
 		if ecNew.MayCache {
@@ -429,6 +429,16 @@ func evalRollupFunc(ec *EvalConfig, name string, rf rollupFunc, expr metricsql.E
 			ecNew.End = end
 		}
 	}
+	if name == "rollup_candlestick" {
+		// Automatically apply `offset -step` to `rollup_candlestick` function
+		// in order to obtain expected OHLC results.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/309#issuecomment-582113462
+		step := ecNew.Step
+		ecNew = newEvalConfig(ecNew)
+		ecNew.Start += step
+		ecNew.End += step
+		offset -= step
+	}
 	var rvs []*timeseries
 	var err error
 	if me, ok := re.Expr.(*metricsql.MetricExpr); ok {
@@ -619,9 +629,15 @@ func evalRollupFuncWithMetricExpr(ec *EvalConfig, name string, rf rollupFunc,
 
 	// Fetch the remaining part of the result.
 	tfs := toTagFilters(me.LabelFilters)
+	minTimestamp := start - maxSilenceInterval
+	if window > ec.Step {
+		minTimestamp -= window
+	} else {
+		minTimestamp -= ec.Step
+	}
 	sq := &storage.SearchQuery{
-		MinTimestamp: start - window - maxSilenceInterval,
-		MaxTimestamp: ec.End + ec.Step,
+		MinTimestamp: minTimestamp,
+		MaxTimestamp: ec.End,
 		TagFilterss:  [][]storage.TagFilter{tfs},
 	}
 	rss, err := netstorage.ProcessSearchQuery(sq, true, ec.Deadline)

app/vmselect/promql/exec.go

@@ -26,8 +26,8 @@ func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result,
 		defer func() {
 			d := time.Since(startTime)
 			if d >= *logSlowQueryDuration {
-				logger.Infof("slow query according to -search.logSlowQueryDuration=%s: duration=%s, start=%d, end=%d, step=%d, query=%q",
-					*logSlowQueryDuration, d, ec.Start/1000, ec.End/1000, ec.Step/1000, q)
+				logger.Infof("slow query according to -search.logSlowQueryDuration=%s: duration=%.3f seconds, start=%d, end=%d, step=%d, query=%q",
+					*logSlowQueryDuration, d.Seconds(), ec.Start/1000, ec.End/1000, ec.Step/1000, q)
 				slowQueries.Inc()
 			}
 		}()
@@ -40,25 +40,11 @@ func Exec(ec *EvalConfig, q string, isFirstPointOnly bool) ([]netstorage.Result,
 		return nil, err
 	}
 
-	// Add an additional point to the end. This point is used
-	// in calculating the last value for rate, deriv, increase
-	// and delta funcs.
-	ec.End += ec.Step
-
 	rv, err := evalExpr(ec, e)
 	if err != nil {
 		return nil, err
 	}
 
-	// Remove the additional point at the end.
-	for _, ts := range rv {
-		ts.Values = ts.Values[:len(ts.Values)-1]
-
-		// ts.Timestamps may be shared between timeseries, so truncate it with len(ts.Values) instead of len(ts.Timestamps)-1
-		ts.Timestamps = ts.Timestamps[:len(ts.Values)]
-	}
-	ec.End -= ec.Step
-
 	if isFirstPointOnly {
 		// Remove all the points except the first one from every time series.
 		for _, ts := range rv {
@@ -85,7 +71,8 @@ func maySortResults(e metricsql.Expr, tss []*timeseries) bool {
 		return true
 	}
 	switch fe.Name {
-	case "sort", "sort_desc":
+	case "sort", "sort_desc",
+		"sort_by_label", "sort_by_label_desc":
 		return false
 	default:
 		return true

app/vmselect/promql/exec_test.go

@@ -21,7 +21,7 @@ func TestExecSuccess(t *testing.T) {
 			Start:    start,
 			End:      end,
 			Step:     step,
-			Deadline: netstorage.NewDeadline(time.Minute),
+			Deadline: netstorage.NewDeadline(time.Minute, ""),
 		}
 		for i := 0; i < 5; i++ {
 			result, err := Exec(ec, q, false)
@@ -338,7 +338,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `timestamp(123)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Values:     []float64{900, 1100, 1300, 1500, 1700, 1900},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -349,7 +349,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `timestamp(time())`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Values:     []float64{900, 1100, 1300, 1500, 1700, 1900},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -360,7 +360,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `timestamp(456/time()+123)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Values:     []float64{900, 1100, 1300, 1500, 1700, 1900},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -371,7 +371,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `timestamp(time()>=1600)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Values:     []float64{nan, nan, nan, nan, 1700, 1900},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -974,6 +974,65 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`label_map(match)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(label_map((
+			label_set(time(), "label", "v1"),
+			label_set(time()+100, "label", "v2"),
+			label_set(time()+200, "label", "v3"),
+			label_set(time()+300, "x", "y"),
+			label_set(time()+400, "label", "v4"),
+		), "label", "v1", "foo", "v2", "bar", "", "qwe", "v4", ""))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("label"),
+			Value: []byte("foo"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1100, 1300, 1500, 1700, 1900, 2100},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("label"),
+			Value: []byte("bar"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1200, 1400, 1600, 1800, 2000, 2200},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("label"),
+			Value: []byte("v3"),
+		}}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1300, 1500, 1700, 1900, 2100, 2300},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("label"),
+				Value: []byte("qwe"),
+			},
+			{
+				Key:   []byte("x"),
+				Value: []byte("y"),
+			},
+		}
+		r5 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1400, 1600, 1800, 2000, 2200, 2400},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5}
+		f(q, resultExpected)
+	})
 	t.Run(`label_copy(new_tag)`, func(t *testing.T) {
 		t.Parallel()
 		q := `label_copy(
@@ -1466,6 +1525,38 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`label_match()`, func(t *testing.T) {
+		t.Parallel()
+		q := `
+		label_match((
+			alias(time(), "foo"),
+			alias(2*time(), "bar"),
+		), "__name__", "f.+")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("foo")
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`label_mismatch()`, func(t *testing.T) {
+		t.Parallel()
+		q := `
+		label_mismatch((
+			alias(time(), "foo"),
+			alias(2*time(), "bar"),
+		), "__name__", "f.+")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2000, 2400, 2800, 3200, 3600, 4000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.MetricGroup = []byte("bar")
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`two_timeseries`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_desc(time() or label_set(2, "xx", "foo"))`
@@ -1565,6 +1656,48 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`sort_by_label()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			alias(1, "foo"),
+			alias(2, "bar"),
+		), "__name__")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("bar")
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("foo")
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`sort_by_label_desc()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label_desc((
+			alias(1, "foo"),
+			alias(2, "bar"),
+		), "__name__")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("foo")
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.MetricGroup = []byte("bar")
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`a cmp scalar (leave MetricGroup)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_desc((
@@ -1688,12 +1821,34 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`time() and time() > 1300`, func(t *testing.T) {
+		t.Parallel()
+		q := `time() and time() > 1300`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`time() unless 2`, func(t *testing.T) {
 		t.Parallel()
 		q := `time() unless 2`
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`time() unless time() > 1500`, func(t *testing.T) {
+		t.Parallel()
+		q := `time() unless time() > 1500`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`timseries-with-tags unless 2`, func(t *testing.T) {
 		t.Parallel()
 		q := `label_set(time(), "foo", "bar") unless 2`
@@ -2922,6 +3077,23 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6}
 		f(q, resultExpected)
 	})
+	t.Run(`prometheus_buckets(zero-vmrange-value)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(prometheus_buckets(label_set(0, "vmrange", "0...0")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("+Inf"),
+			},
+		}
+		resultsExpected := []netstorage.Result{r1}
+		f(q, resultsExpected)
+	})
 	t.Run(`prometheus_buckets(valid)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(prometheus_buckets((
@@ -3216,6 +3388,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`range_over_time(time)`, func(t *testing.T) {
+		t.Parallel()
+		q := `range_over_time(alias(time()/100, "foobar")[3i])`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`sum(multi-vector)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sum(label_set(10, "foo", "bar") or label_set(time()/100, "baz", "sss"))`
@@ -3379,7 +3562,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{13, 14, 12, 8, 12, 13},
+			Values:     []float64{14, 15, 12, 13, 15, 11},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{
@@ -3389,12 +3572,12 @@ func TestExecSuccess(t *testing.T) {
 			},
 			{
 				Key:   []byte("vmrange"),
-				Value: []byte("1.0e0...1.5e0"),
+				Value: []byte("2.0e0...2.5e0"),
 			},
 		}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{14, 15, 12, 13, 15, 11},
+			Values:     []float64{13, 14, 12, 8, 12, 13},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{
@@ -3404,7 +3587,7 @@ func TestExecSuccess(t *testing.T) {
 			},
 			{
 				Key:   []byte("vmrange"),
-				Value: []byte("2.0e0...2.5e0"),
+				Value: []byte("1.0e0...1.5e0"),
 			},
 		}
 		r3 := netstorage.Result{
@@ -3430,24 +3613,24 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{13, 14, 12, 8, 12, 13},
+			Values:     []float64{14, 15, 12, 13, 15, 11},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{
 			{
 				Key:   []byte("vmrange"),
-				Value: []byte("1.0e0...1.5e0"),
+				Value: []byte("2.0e0...2.5e0"),
 			},
 		}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{14, 15, 12, 13, 15, 11},
+			Values:     []float64{13, 14, 12, 8, 12, 13},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{
 			{
 				Key:   []byte("vmrange"),
-				Value: []byte("2.0e0...2.5e0"),
+				Value: []byte("1.0e0...1.5e0"),
 			},
 		}
 		r3 := netstorage.Result{
@@ -3695,12 +3878,12 @@ func TestExecSuccess(t *testing.T) {
 		q := `sort(bottomk_avg(1, label_set(10, "foo", "bar") or label_set(time()/150, "baz", "sss")))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{10, 10, 10, nan, nan, nan},
+			Values:     []float64{6.666666666666667, 8, 9.333333333333334, 10.666666666666666, 12, 13.333333333333334},
 			Timestamps: timestampsExpected,
 		}
 		r1.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
+			Key:   []byte("baz"),
+			Value: []byte("sss"),
 		}}
 		resultExpected := []netstorage.Result{r1}
 		f(q, resultExpected)
@@ -3844,6 +4027,22 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1}
 		f(q, resultExpected)
 	})
+	t.Run(`keep_next_value()`, func(t *testing.T) {
+		t.Parallel()
+		q := `keep_next_value(label_set(time() < 1300 default time() > 1700, "__name__", "foobar", "x", "y"))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1800, 1800, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.MetricGroup = []byte("foobar")
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("x"),
+			Value: []byte("y"),
+		}}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
 	t.Run(`distinct_over_time([500s])`, func(t *testing.T) {
 		t.Parallel()
 		q := `distinct_over_time((time() < 1700)[500s])`
@@ -4600,7 +4799,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0.85, 0.15, 0.43, 0.76, 0.47, 0.21},
+			Values:     []float64{0.9, 0.32, 0.82, 0.13, 0.28, 0.86},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
@@ -4609,7 +4808,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r3 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0.32, 0.82, 0.13, 0.28, 0.86, 0.57},
+			Values:     []float64{0.1, 0.04, 0.49, 0.46, 0.57, 0.92},
 			Timestamps: timestampsExpected,
 		}
 		r3.MetricName.Tags = []storage.Tag{{
@@ -4618,7 +4817,7 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r4 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0.85, 0.94, 0.97, 0.93, 0.98, 0.92},
+			Values:     []float64{0.9, 0.94, 0.97, 0.93, 0.98, 0.92},
 			Timestamps: timestampsExpected,
 		}
 		r4.MetricName.Tags = []storage.Tag{{
@@ -5186,7 +5385,7 @@ func TestExecError(t *testing.T) {
 			Start:    1000,
 			End:      2000,
 			Step:     100,
-			Deadline: netstorage.NewDeadline(time.Minute),
+			Deadline: netstorage.NewDeadline(time.Minute, ""),
 		}
 		for i := 0; i < 4; i++ {
 			rv, err := Exec(ec, q, false)
@@ -5231,13 +5430,19 @@ func TestExecError(t *testing.T) {
 	f(`label_transform(1)`)
 	f(`label_set()`)
 	f(`label_set(1, "foo")`)
+	f(`label_map()`)
+	f(`label_map(1)`)
 	f(`label_del()`)
 	f(`label_keep()`)
+	f(`label_match()`)
+	f(`label_mismatch()`)
 	f(`round()`)
 	f(`round(1,2,3)`)
 	f(`scalar()`)
 	f(`sort(1,2)`)
 	f(`sort_desc()`)
+	f(`sort_by_label()`)
+	f(`sort_by_label_desc()`)
 	f(`timestamp()`)
 	f(`vector()`)
 	f(`histogram_quantile()`)
@@ -5280,6 +5485,7 @@ func TestExecError(t *testing.T) {
 	f(`median()`)
 	f(`median("foo", "bar")`)
 	f(`keep_last_value()`)
+	f(`keep_next_value()`)
 	f(`distinct_over_time()`)
 	f(`distinct()`)
 	f(`alias()`)
@@ -5333,6 +5539,8 @@ func TestExecError(t *testing.T) {
 	f(`label_transform(1, "foo", 3, 4)`)
 	f(`label_transform(1, "foo", "bar", 4)`)
 	f(`label_transform(1, "foo", "invalid(regexp", "baz`)
+	f(`label_match(1, 2, 3)`)
+	f(`label_mismatch(1, 2, 3)`)
 	f(`alias(1, 2)`)
 	f(`aggr_over_time(1, 2)`)
 	f(`aggr_over_time(("foo", "bar"), 3)`)

app/vmselect/promql/rollup.go

@@ -1,6 +1,7 @@
 package promql
 
 import (
+	"flag"
 	"fmt"
 	"math"
 	"strings"
@@ -14,9 +15,11 @@ import (
 	"github.com/valyala/histogram"
 )
 
-var rollupFuncs = map[string]newRollupFunc{
-	"default_rollup": newRollupFuncOneArg(rollupDefault), // default rollup func
+var minStalenessInterval = flag.Duration("search.minStalenessInterval", 0, "The mimimum interval for staleness calculations. "+
+	"This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. "+
+	"See also '-search.maxStalenessInterval'")
 
+var rollupFuncs = map[string]newRollupFunc{
 	// Standard rollup funcs from PromQL.
 	// See funcs accepting range-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ .
 	"changes":            newRollupFuncOneArg(rollupChanges),
@@ -41,6 +44,8 @@ var rollupFuncs = map[string]newRollupFunc{
 	"absent_over_time":   newRollupFuncOneArg(rollupAbsent),
 
 	// Additional rollup funcs.
+	"default_rollup":        newRollupFuncOneArg(rollupDefault), // default rollup func
+	"range_over_time":       newRollupFuncOneArg(rollupRange),
 	"sum2_over_time":        newRollupFuncOneArg(rollupSum2),
 	"geomean_over_time":     newRollupFuncOneArg(rollupGeomean),
 	"first_over_time":       newRollupFuncOneArg(rollupFirst),
@@ -67,6 +72,11 @@ var rollupFuncs = map[string]newRollupFunc{
 	"aggr_over_time":        newRollupFuncTwoArgs(rollupFake),
 	"hoeffding_bound_upper": newRollupHoeffdingBoundUpper,
 	"hoeffding_bound_lower": newRollupHoeffdingBoundLower,
+
+	// `timestamp` function must return timestamp for the last datapoint on the current window
+	// in order to properly handle offset and timestamps unaligned to the current step.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415 for details.
+	"timestamp": newRollupFuncOneArg(rollupTimestamp),
 }
 
 // rollupAggrFuncs are functions that can be passed to `aggr_over_time()`
@@ -91,6 +101,7 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"absent_over_time": rollupAbsent,
 
 	// Additional rollup funcs.
+	"range_over_time":     rollupRange,
 	"sum2_over_time":      rollupSum2,
 	"geomean_over_time":   rollupGeomean,
 	"first_over_time":     rollupFirst,
@@ -147,6 +158,8 @@ var rollupFuncsKeepMetricGroup = map[string]bool{
 	"geomean_over_time":     true,
 	"hoeffding_bound_lower": true,
 	"hoeffding_bound_upper": true,
+	"first_over_time":       true,
+	"last_over_time":        true,
 }
 
 func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) {
@@ -258,10 +271,10 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en
 		}
 		rcs = appendRollupConfigs(rcs)
 	case "rollup_candlestick":
-		rcs = append(rcs, newRollupConfig(rollupFirst, "open"))
-		rcs = append(rcs, newRollupConfig(rollupLast, "close"))
-		rcs = append(rcs, newRollupConfig(rollupMin, "low"))
-		rcs = append(rcs, newRollupConfig(rollupMax, "high"))
+		rcs = append(rcs, newRollupConfig(rollupOpen, "open"))
+		rcs = append(rcs, newRollupConfig(rollupClose, "close"))
+		rcs = append(rcs, newRollupConfig(rollupLow, "low"))
+		rcs = append(rcs, newRollupConfig(rollupHigh, "high"))
 	case "aggr_over_time":
 		aggrFuncNames, err := getRollupAggrFuncNames(expr)
 		if err != nil {
@@ -406,7 +419,7 @@ func (tsm *timeseriesMap) GetOrCreateTimeseries(labelValue string) *timeseries {
 //
 // rc.Timestamps are used as timestamps for dstValues.
 //
-// timestamps must cover time range [rc.Start - rc.Window - maxSilenceInterval ... rc.End + rc.Step].
+// timestamps must cover time range [rc.Start - rc.Window - maxSilenceInterval ... rc.End].
 //
 // Do cannot be called from concurrent goroutines.
 func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []int64) []float64 {
@@ -443,6 +456,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	if rc.LookbackDelta > 0 && maxPrevInterval > rc.LookbackDelta {
 		maxPrevInterval = rc.LookbackDelta
 	}
+	if *minStalenessInterval > 0 {
+		if msi := minStalenessInterval.Milliseconds(); msi > 0 && maxPrevInterval < msi {
+			maxPrevInterval = msi
+		}
+	}
 	window := rc.Window
 	if window <= 0 {
 		window = rc.Step
@@ -1052,6 +1070,12 @@ func rollupSum(rfa *rollupFuncArg) float64 {
 	return sum
 }
 
+func rollupRange(rfa *rollupFuncArg) float64 {
+	max := rollupMax(rfa)
+	min := rollupMin(rfa)
+	return max - min
+}
+
 func rollupSum2(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
@@ -1150,10 +1174,18 @@ func rollupDeltaInternal(rfa *rollupFuncArg, canUseRealPrevValue bool) float64 {
 		if len(values) == 0 {
 			return nan
 		}
-		// Assume that the previous non-existing value was 0.
-		prevValue = 0
-		if canUseRealPrevValue && !math.IsNaN(rfa.prevValue) {
-			prevValue = rfa.prevValue
+		// Assume that the previous non-existing value was 0
+		// only if the first value is quite small.
+		// This should prevent from improper increase() results for os-level counters
+		// such as cpu time or bytes sent over the network interface.
+		// These counters may start long ago before the first value appears in the db.
+		if values[0] < 1e6 {
+			prevValue = 0
+			if canUseRealPrevValue && !math.IsNaN(rfa.realPrevValue) {
+				prevValue = rfa.realPrevValue
+			}
+		} else {
+			prevValue = values[0]
 		}
 	}
 	if len(values) == 0 {
@@ -1404,6 +1436,97 @@ func rollupResets(rfa *rollupFuncArg) float64 {
 	return float64(n)
 }
 
+// getCandlestickValues returns a subset of rfa.values suitable for rollup_candlestick
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/309 for details.
+func getCandlestickValues(rfa *rollupFuncArg) []float64 {
+	currTimestamp := rfa.currTimestamp
+	timestamps := rfa.timestamps
+	for len(timestamps) > 0 && timestamps[len(timestamps)-1] >= currTimestamp {
+		timestamps = timestamps[:len(timestamps)-1]
+	}
+	if len(timestamps) == 0 {
+		return nil
+	}
+	return rfa.values[:len(timestamps)]
+}
+
+func getFirstValueForCandlestick(rfa *rollupFuncArg) float64 {
+	if rfa.prevTimestamp+rfa.step >= rfa.currTimestamp {
+		return rfa.prevValue
+	}
+	return nan
+}
+
+func rollupOpen(rfa *rollupFuncArg) float64 {
+	v := getFirstValueForCandlestick(rfa)
+	if !math.IsNaN(v) {
+		return v
+	}
+	values := getCandlestickValues(rfa)
+	if len(values) == 0 {
+		return nan
+	}
+	return values[0]
+}
+
+func rollupClose(rfa *rollupFuncArg) float64 {
+	values := getCandlestickValues(rfa)
+	if len(values) == 0 {
+		return getFirstValueForCandlestick(rfa)
+	}
+	return values[len(values)-1]
+}
+
+func rollupHigh(rfa *rollupFuncArg) float64 {
+	values := getCandlestickValues(rfa)
+	max := getFirstValueForCandlestick(rfa)
+	if math.IsNaN(max) {
+		if len(values) == 0 {
+			return nan
+		}
+		max = values[0]
+		values = values[1:]
+	}
+	for _, v := range values {
+		if v > max {
+			max = v
+		}
+	}
+	return max
+}
+
+func rollupLow(rfa *rollupFuncArg) float64 {
+	values := getCandlestickValues(rfa)
+	min := getFirstValueForCandlestick(rfa)
+	if math.IsNaN(min) {
+		if len(values) == 0 {
+			return nan
+		}
+		min = values[0]
+		values = values[1:]
+	}
+	for _, v := range values {
+		if v < min {
+			min = v
+		}
+	}
+	return min
+}
+
+func rollupTimestamp(rfa *rollupFuncArg) float64 {
+	// There is no need in handling NaNs here, since they must be cleaned up
+	// before calling rollup funcs.
+	timestamps := rfa.timestamps
+	if len(timestamps) == 0 {
+		// Do not take into account rfa.prevTimestamp, since it may lead
+		// to inconsistent results comparing to Prometheus on broken time series
+		// with irregular data points.
+		return nan
+	}
+	return float64(timestamps[len(timestamps)-1]) / 1e3
+}
+
 func rollupFirst(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.

app/vmselect/promql/rollup_result_cache.go

@@ -18,7 +18,12 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-var disableCache = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful during data backfilling")
+var (
+	disableCache         = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful during data backfilling")
+	cacheTimestampOffset = flag.Duration("search.cacheTimestampOffset", 5*time.Minute, "The maximum duration since the current time for response data, "+
+		"which is always queried from the original raw data, without using the response cache. Increase this value if you see gaps in responses "+
+		"due to time synchronization issues between VictoriaMetrics and data sources")
+)
 
 var rollupResultCacheV = &rollupResultCache{
 	c: workingsetcache.New(1024*1024, time.Hour), // This is a cache for testing.
@@ -74,8 +79,8 @@ func InitRollupResultCache(cachePath string) {
 		return stats
 	}
 	if len(rollupResultCachePath) > 0 {
-		logger.Infof("loaded rollupResult cache from %q in %s; entriesCount: %d, sizeBytes: %d",
-			rollupResultCachePath, time.Since(startTime), fcs().EntriesCount, fcs().BytesSize)
+		logger.Infof("loaded rollupResult cache from %q in %.3f seconds; entriesCount: %d, sizeBytes: %d",
+			rollupResultCachePath, time.Since(startTime).Seconds(), fcs().EntriesCount, fcs().BytesSize)
 	}
 
 	metrics.NewGauge(`vm_cache_entries{type="promql/rollupResult"}`, func() float64 {
@@ -113,8 +118,8 @@ func StopRollupResultCache() {
 	rollupResultCacheV.c.UpdateStats(&fcs)
 	rollupResultCacheV.c.Stop()
 	rollupResultCacheV.c = nil
-	logger.Infof("saved rollupResult cache to %q in %s; entriesCount: %d, sizeBytes: %d",
-		rollupResultCachePath, time.Since(startTime), fcs.EntriesCount, fcs.BytesSize)
+	logger.Infof("saved rollupResult cache to %q in %.3f seconds; entriesCount: %d, sizeBytes: %d",
+		rollupResultCachePath, time.Since(startTime).Seconds(), fcs.EntriesCount, fcs.BytesSize)
 }
 
 type rollupResultCache struct {
@@ -127,6 +132,7 @@ var rollupResultCacheResets = metrics.NewCounter(`vm_cache_resets_total{type="pr
 func ResetRollupResultCache() {
 	rollupResultCacheResets.Inc()
 	rollupResultCacheV.c.Reset()
+	logger.Infof("rollupResult cache has been cleared")
 }
 
 func (rrc *rollupResultCache) Get(ec *EvalConfig, expr metricsql.Expr, window int64) (tss []*timeseries, newStart int64) {
@@ -215,10 +221,10 @@ func (rrc *rollupResultCache) Put(ec *EvalConfig, expr metricsql.Expr, window in
 		return
 	}
 
-	// Remove values up to currentTime - step - maxSilenceInterval,
+	// Remove values up to currentTime - step - cacheTimestampOffset,
 	// since these values may be added later.
 	timestamps := tss[0].Timestamps
-	deadline := (time.Now().UnixNano() / 1e6) - ec.Step - maxSilenceInterval
+	deadline := (time.Now().UnixNano() / 1e6) - ec.Step - cacheTimestampOffset.Milliseconds()
 	i := len(timestamps) - 1
 	for i >= 0 && timestamps[i] > deadline {
 		i--

app/vmselect/promql/rollup_test.go

@@ -370,6 +370,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("irate", 0)
 	f("rate", 2200)
 	f("resets", 5)
+	f("range_over_time", 111)
 	f("avg_over_time", 47.083333333333336)
 	f("min_over_time", 12)
 	f("max_over_time", 123)

app/vmselect/promql/timeseries.go

@@ -2,6 +2,7 @@ package promql
 
 import (
 	"fmt"
+	"reflect"
 	"sort"
 	"strconv"
 	"sync"
@@ -168,7 +169,7 @@ func (ts *timeseries) marshalFastNoTimestamps(dst []byte) []byte {
 	// during marshalFastTimestamps.
 	var valuesBuf []byte
 	if len(ts.Values) > 0 {
-		valuesBuf = (*[maxByteSliceLen]byte)(unsafe.Pointer(&ts.Values[0]))[:len(ts.Values)*8]
+		valuesBuf = float64ToByteSlice(ts.Values)
 	}
 	dst = append(dst, valuesBuf...)
 	return dst
@@ -178,7 +179,7 @@ func marshalFastTimestamps(dst []byte, timestamps []int64) []byte {
 	dst = encoding.MarshalUint32(dst, uint32(len(timestamps)))
 	var timestampsBuf []byte
 	if len(timestamps) > 0 {
-		timestampsBuf = (*[maxByteSliceLen]byte)(unsafe.Pointer(&timestamps[0]))[:len(timestamps)*8]
+		timestampsBuf = int64ToByteSlice(timestamps)
 	}
 	dst = append(dst, timestampsBuf...)
 	return dst
@@ -199,8 +200,7 @@ func unmarshalFastTimestamps(src []byte) ([]byte, []int64, error) {
 	if len(src) < bufSize {
 		return src, nil, fmt.Errorf("cannot unmarshal timestamps; got %d bytes; want at least %d bytes", len(src), bufSize)
 	}
-	timestamps := (*[maxByteSliceLen / 8]int64)(unsafe.Pointer(&src[0]))[:timestampsCount]
-	timestamps = timestamps[:len(timestamps):len(timestamps)]
+	timestamps := byteSliceToInt64(src[:bufSize])
 	src = src[bufSize:]
 
 	return src, timestamps, nil
@@ -229,12 +229,43 @@ func (ts *timeseries) unmarshalFastNoTimestamps(src []byte) ([]byte, error) {
 	if len(src) < bufSize {
 		return src, fmt.Errorf("cannot unmarshal values; got %d bytes; want at least %d bytes", len(src), bufSize)
 	}
-	values := (*[maxByteSliceLen / 8]float64)(unsafe.Pointer(&src[0]))[:valuesCount]
-	ts.Values = values[:len(values):len(values)]
+	ts.Values = byteSliceToFloat64(src[:bufSize])
 
 	return src[bufSize:], nil
 }
 
+func float64ToByteSlice(a []float64) (b []byte) {
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	sh.Data = uintptr(unsafe.Pointer(&a[0]))
+	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	return
+}
+
+func int64ToByteSlice(a []int64) (b []byte) {
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	sh.Data = uintptr(unsafe.Pointer(&a[0]))
+	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	return
+}
+
+func byteSliceToInt64(b []byte) (a []int64) {
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
+	sh.Data = uintptr(unsafe.Pointer(&b[0]))
+	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	return
+}
+
+func byteSliceToFloat64(b []byte) (a []float64) {
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
+	sh.Data = uintptr(unsafe.Pointer(&b[0]))
+	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	return
+}
+
 // unmarshalMetricNameFast unmarshals mn from src, so mn members
 // hold references to src.
 //