app/vmselect/promql: return expected results from increase() over the beginning of time series, which start from big value

Examples for such counters: OS-level counters for network or cpu stats.

Makefile

@@ -79,16 +79,16 @@ install-errcheck:
 check-all: fmt vet lint errcheck golangci-lint
 
 test:
-	GO111MODULE=on go test -tags=integration -mod=vendor ./lib/... ./app/...
+	GO111MODULE=on go test -mod=vendor ./lib/... ./app/...
 
 test-pure:
-	GO111MODULE=on CGO_ENABLED=0 go test -tags=integration -mod=vendor ./lib/... ./app/...
+	GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/...
 
 test-full:
-	GO111MODULE=on go test -tags=integration -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GO111MODULE=on GOARCH=386 go test -tags=integration -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GO111MODULE=on GOARCH=386 go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
 	GO111MODULE=on go test -mod=vendor -bench=. ./lib/...

README.md

@@ -88,7 +88,6 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
     - [Building docker images](#building-docker-images)
   - [Start with docker-compose](#start-with-docker-compose)
   - [Setting up service](#setting-up-service)
-  - [Third-party contributions](#third-party-contributions)
   - [How to work with snapshots?](#how-to-work-with-snapshots)
   - [How to delete time series?](#how-to-delete-time-series)
   - [How to export time series?](#how-to-export-time-series)
@@ -96,6 +95,7 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
   - [Federation](#federation)
   - [Capacity planning](#capacity-planning)
   - [High availability](#high-availability)
+  - [Retention](#retention)
   - [Multiple retentions](#multiple-retentions)
   - [Downsampling](#downsampling)
   - [Multi-tenancy](#multi-tenancy)
@@ -111,6 +111,7 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
 - [Roadmap](#roadmap)
 - [Contacts](#contacts)
 - [Community and contributions](#community-and-contributions)
+- [Third-party contributions](#third-party-contributions)
 - [Reporting bugs](#reporting-bugs)
 - [Victoria Metrics Logo](#victoria-metrics-logo)
   - [Logo Usage Guidelines](#logo-usage-guidelines)
@@ -502,12 +503,6 @@ More details may be found [here](https://github.com/VictoriaMetrics/VictoriaMetr
 Read [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) on how to set up VictoriaMetrics as a service in your OS.
 
 
-### Third-party contributions
-
-* [Unofficial yum repository](https://copr.fedorainfracloud.org/coprs/antonpatsev/VictoriaMetrics/) ([source code](https://github.com/patsevanton/victoriametrics-rpm))
-* [Prometheus -> VictoriaMetrics exporter](https://github.com/ryotarai/prometheus-tsdb-dump)
-
-
 ### How to work with snapshots?
 
 VictoriaMetrics can create [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
@@ -707,6 +702,16 @@ If you have Prometheus HA pairs with replicas `r1` and `r2` in each pair, then c
 to write data to `victoriametrics-addr-1`, while each `r2` should write data to `victoriametrics-addr-2`.
 
 
+### Retention
+
+Retention is configured with `-retentionPeriod` command-line flag. For instance, `-retentionPeriod=3` means
+that the data will be stored for 3 months and then deleted.
+Data is split in per-month subdirectories inside `<-storageDataPath>/data/small` and `<-storageDataPath>/data/big` folders.
+Directories for months outside the configured retention are deleted on the first day of new month.
+In order to keep data according to `-retentionPeriod` max disk space usage is going to be `-retentionPeriod` + 1 month.
+For example if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
+
+
 ### Multiple retentions
 
 Just start multiple VictoriaMetrics instances with distinct values for the following flags:
@@ -786,8 +791,11 @@ mkfs.ext4 ... -O 64bit,huge_file,extent -T huge
 
 ### Monitoring
 
-VictoriaMetrics exports internal metrics in Prometheus format on the `/metrics` page.
-Add this page to Prometheus' scrape config in order to collect VictoriaMetrics metrics.
+VictoriaMetrics exports internal metrics in Prometheus format at `/metrics` page.
+These metrics may be collected either via Prometheus by adding the corresponding scrape config to it.
+Alternatively they can be self-scraped by setting `-selfScrapeInterval` command-line flag to duration greater than 0.
+For example, `-scrapeInterval=10s` would enable self-scraping of `/metrics` page with 10 seconds interval.
+
 There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176).
 
 The most interesting metrics are:
@@ -910,6 +918,13 @@ We are open to third-party pull requests provided they follow [KISS design princ
 Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.
 
 
+### Third-party contributions
+
+* [Unofficial yum repository](https://copr.fedorainfracloud.org/coprs/antonpatsev/VictoriaMetrics/) ([source code](https://github.com/patsevanton/victoriametrics-rpm))
+* [Prometheus -> VictoriaMetrics exporter #1](https://github.com/ryotarai/prometheus-tsdb-dump)
+* [Prometheus -> VictoriaMetrics exporter #2](https://github.com/AnchorFree/tsdb-remote-write)
+
+
 ## Reporting bugs
 
 Report bugs and propose new features [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).

app/victoria-metrics/main.go

@@ -26,6 +26,7 @@ func main() {
 	vmstorage.Init()
 	vmselect.Init()
 	vminsert.Init()
+	startSelfScraper()
 
 	go httpserver.Serve(*httpListenAddr, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
@@ -33,6 +34,8 @@ func main() {
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
 
+	stopSelfScraper()
+
 	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
 	if err := httpserver.Stop(*httpListenAddr); err != nil {

app/victoria-metrics/main_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 package main
 
 import (
@@ -302,6 +300,9 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 	s := newSuite(t)
 	var tt []test
 	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
 		if filepath.Ext(path) != ".json" {
 			return nil
 		}

app/victoria-metrics/self_scraper.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"flag"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+var selfScrapeInterval = flag.Duration("selfScrapeInterval", 0, "Interval for self-scraping own metrics at `/metrics` page")
+
+var selfScraperStopCh chan struct{}
+var selfScraperWG sync.WaitGroup
+
+func startSelfScraper() {
+	selfScraperStopCh = make(chan struct{})
+	selfScraperWG.Add(1)
+	go func() {
+		defer selfScraperWG.Done()
+		selfScraper(*selfScrapeInterval)
+	}()
+}
+
+func stopSelfScraper() {
+	close(selfScraperStopCh)
+	selfScraperWG.Wait()
+}
+
+func selfScraper(scrapeInterval time.Duration) {
+	if scrapeInterval <= 0 {
+		// Self-scrape is disabled.
+		return
+	}
+	logger.Infof("started self-scraping `/metrics` page with interval %.3f seconds", scrapeInterval.Seconds())
+
+	var bb bytesutil.ByteBuffer
+	var rows prometheus.Rows
+	var mrs []storage.MetricRow
+	var labels []prompb.Label
+	t := time.NewTicker(scrapeInterval)
+	var currentTimestamp int64
+	for {
+		select {
+		case <-selfScraperStopCh:
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			currentTimestamp = currentTime.UnixNano() / 1e6
+		}
+		bb.Reset()
+		httpserver.WritePrometheusMetrics(&bb)
+		s := bytesutil.ToUnsafeString(bb.B)
+		rows.Reset()
+		rows.Unmarshal(s)
+		mrs = mrs[:0]
+		for i := range rows.Rows {
+			r := &rows.Rows[i]
+			labels = labels[:0]
+			labels = addLabel(labels, "", r.Metric)
+			labels = addLabel(labels, "job", "victoria-metrics")
+			labels = addLabel(labels, "instance", "self")
+			for j := range r.Tags {
+				t := &r.Tags[j]
+				labels = addLabel(labels, t.Key, t.Value)
+			}
+			if len(mrs) < cap(mrs) {
+				mrs = mrs[:len(mrs)+1]
+			} else {
+				mrs = append(mrs, storage.MetricRow{})
+			}
+			mr := &mrs[len(mrs)-1]
+			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
+			mr.Timestamp = currentTimestamp
+			mr.Value = r.Value
+		}
+		logger.Infof("writing %d rows at timestamp %d", len(mrs), currentTimestamp)
+		vmstorage.AddRows(mrs)
+	}
+}
+
+func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
+	if len(dst) < cap(dst) {
+		dst = dst[:len(dst)+1]
+	} else {
+		dst = append(dst, prompb.Label{})
+	}
+	lb := &dst[len(dst)-1]
+	lb.Name = bytesutil.ToUnsafeBytes(key)
+	lb.Value = bytesutil.ToUnsafeBytes(value)
+	return dst
+}

app/victoria-metrics/test/prom_types.go

@@ -1,18 +1,18 @@
-// +build integration
-
-// Source https://github.com/prometheus/prometheus/blob/master/prompb/remote.pb.go . Code is copy pasted and cleaned up
 package test
 
+// Source https://github.com/prometheus/prometheus/blob/master/prompb/remote.pb.go . Code is copy pasted and cleaned up
 import (
 	"encoding/binary"
 	"math"
 	"math/bits"
 )
 
+// WriteRequest is write request
 type WriteRequest struct {
 	Timeseries []TimeSeries `protobuf:"bytes,1,rep,name=timeseries,proto3" json:"timeseries"`
 }
 
+// Size returns m size in bytes after marshaling.
 func (m *WriteRequest) Size() (n int) {
 	if m == nil {
 		return 0
@@ -31,6 +31,7 @@ func sovRemote(x uint64) (n int) {
 	return (bits.Len64(x|1) + 6) / 7
 }
 
+// Marshal marshals m.
 func (m *WriteRequest) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -41,11 +42,13 @@ func (m *WriteRequest) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA
 func (m *WriteRequest) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *WriteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	if len(m.Timeseries) > 0 {
@@ -77,11 +80,13 @@ func encodeVarintRemote(dAtA []byte, offset int, v uint64) int {
 	return base
 }
 
+// Sample is time series sample.
 type Sample struct {
 	Value     float64 `protobuf:"fixed64,1,opt,name=value,proto3" json:"value,omitempty"`
 	Timestamp int64   `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
 }
 
+// Reset resets m.
 func (m *Sample) Reset() { *m = Sample{} }
 
 // TimeSeries represents samples and labels for a single time series.
@@ -90,21 +95,27 @@ type TimeSeries struct {
 	Samples []Sample `protobuf:"bytes,2,rep,name=samples,proto3" json:"samples"`
 }
 
+// Reset resets m.
 func (m *TimeSeries) Reset() { *m = TimeSeries{} }
 
+// Label is time series label.
 type Label struct {
 	Name  string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
 }
 
+// Reset resets m.
 func (m *Label) Reset() { *m = Label{} }
 
+// Labels is a set of labels.
 type Labels struct {
 	Labels []Label `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels"`
 }
 
+// Reset resets m.
 func (m *Labels) Reset() { *m = Labels{} }
 
+// Marshal marshals m.
 func (m *Sample) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -115,11 +126,13 @@ func (m *Sample) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA.
 func (m *Sample) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *Sample) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	if m.Timestamp != 0 {
@@ -136,6 +149,7 @@ func (m *Sample) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+// Marshal marshals m.
 func (m *TimeSeries) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -146,11 +160,13 @@ func (m *TimeSeries) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA.
 func (m *TimeSeries) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *TimeSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	if len(m.Samples) > 0 {
@@ -184,6 +200,7 @@ func (m *TimeSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+// Marshal marshals m.
 func (m *Label) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -194,11 +211,13 @@ func (m *Label) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA.
 func (m *Label) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *Label) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	_ = i
@@ -221,6 +240,7 @@ func (m *Label) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+// Marshal marshals m.
 func (m *Labels) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -231,11 +251,13 @@ func (m *Labels) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
+// MarshalTo marshals m to dAtA.
 func (m *Labels) MarshalTo(dAtA []byte) (int, error) {
 	size := m.Size()
 	return m.MarshalToSizedBuffer(dAtA[:size])
 }
 
+// MarshalToSizedBuffer marshals m to dAtA.
 func (m *Labels) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	i := len(dAtA)
 	if len(m.Labels) > 0 {
@@ -267,6 +289,7 @@ func encodeVarintTypes(dAtA []byte, offset int, v uint64) int {
 	return base
 }
 
+// Size returns the size of marshaled m.
 func (m *Sample) Size() (n int) {
 	if m == nil {
 		return 0
@@ -280,6 +303,7 @@ func (m *Sample) Size() (n int) {
 	return n
 }
 
+// Size returns the size of marshaled m.
 func (m *TimeSeries) Size() (n int) {
 	if m == nil {
 		return 0
@@ -301,6 +325,7 @@ func (m *TimeSeries) Size() (n int) {
 	return n
 }
 
+// Size returns the size of marshaled m.
 func (m *Label) Size() (n int) {
 	if m == nil {
 		return 0
@@ -318,6 +343,7 @@ func (m *Label) Size() (n int) {
 	return n
 }
 
+// Size returns the size of marshaled m.
 func (m *Labels) Size() (n int) {
 	if m == nil {
 		return 0

app/victoria-metrics/test/prom_writter.go

@@ -1,9 +1,8 @@
-// +build integration
-
 package test
 
 import "github.com/golang/snappy"
 
+// Compress marshals and compresses wr.
 func Compress(wr WriteRequest) ([]byte, error) {
 	data, err := wr.Marshal()
 	if err != nil {

app/vmselect/promql/exec_test.go

@@ -2954,6 +2954,23 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6}
 		f(q, resultExpected)
 	})
+	t.Run(`prometheus_buckets(zero-vmrange-value)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(prometheus_buckets(label_set(0, "vmrange", "0...0")))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("+Inf"),
+			},
+		}
+		resultsExpected := []netstorage.Result{r1}
+		f(q, resultsExpected)
+	})
 	t.Run(`prometheus_buckets(valid)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(prometheus_buckets((

app/vmselect/promql/rollup.go

@@ -1157,10 +1157,18 @@ func rollupDeltaInternal(rfa *rollupFuncArg, canUseRealPrevValue bool) float64 {
 		if len(values) == 0 {
 			return nan
 		}
-		// Assume that the previous non-existing value was 0.
-		prevValue = 0
-		if canUseRealPrevValue && !math.IsNaN(rfa.prevValue) {
-			prevValue = rfa.prevValue
+		// Assume that the previous non-existing value was 0
+		// only if the first value is quite small.
+		// This should prevent from improper increase() results for os-level counters
+		// such as cpu time or bytes sent over the network interface.
+		// These counters may start long ago before the first value appears in the db.
+		if values[0] < 1e6 {
+			prevValue = 0
+			if canUseRealPrevValue && !math.IsNaN(rfa.realPrevValue) {
+				prevValue = rfa.realPrevValue
+			}
+		} else {
+			prevValue = values[0]
 		}
 	}
 	if len(values) == 0 {

app/vmselect/promql/transform.go

@@ -363,6 +363,7 @@ func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
 			ts := xs.ts
 			if isZeroTS(ts) {
 				// Skip time series with zeros. They are substituted by xssNew below.
+				xsPrev = xs
 				continue
 			}
 			if xs.start != xsPrev.end {

docs/Single-server-VictoriaMetrics.md

@@ -78,7 +78,6 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
     - [Building docker images](#building-docker-images)
   - [Start with docker-compose](#start-with-docker-compose)
   - [Setting up service](#setting-up-service)
-  - [Third-party contributions](#third-party-contributions)
   - [How to work with snapshots?](#how-to-work-with-snapshots)
   - [How to delete time series?](#how-to-delete-time-series)
   - [How to export time series?](#how-to-export-time-series)
@@ -86,6 +85,7 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
   - [Federation](#federation)
   - [Capacity planning](#capacity-planning)
   - [High availability](#high-availability)
+  - [Retention](#retention)
   - [Multiple retentions](#multiple-retentions)
   - [Downsampling](#downsampling)
   - [Multi-tenancy](#multi-tenancy)
@@ -101,6 +101,7 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM
 - [Roadmap](#roadmap)
 - [Contacts](#contacts)
 - [Community and contributions](#community-and-contributions)
+- [Third-party contributions](#third-party-contributions)
 - [Reporting bugs](#reporting-bugs)
 - [Victoria Metrics Logo](#victoria-metrics-logo)
   - [Logo Usage Guidelines](#logo-usage-guidelines)
@@ -492,12 +493,6 @@ More details may be found [here](https://github.com/VictoriaMetrics/VictoriaMetr
 Read [these instructions](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/43) on how to set up VictoriaMetrics as a service in your OS.
 
 
-### Third-party contributions
-
-* [Unofficial yum repository](https://copr.fedorainfracloud.org/coprs/antonpatsev/VictoriaMetrics/) ([source code](https://github.com/patsevanton/victoriametrics-rpm))
-* [Prometheus -> VictoriaMetrics exporter](https://github.com/ryotarai/prometheus-tsdb-dump)
-
-
 ### How to work with snapshots?
 
 VictoriaMetrics can create [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
@@ -697,6 +692,16 @@ If you have Prometheus HA pairs with replicas `r1` and `r2` in each pair, then c
 to write data to `victoriametrics-addr-1`, while each `r2` should write data to `victoriametrics-addr-2`.
 
 
+### Retention
+
+Retention is configured with `-retentionPeriod` command-line flag. For instance, `-retentionPeriod=3` means
+that the data will be stored for 3 months and then deleted.
+Data is split in per-month subdirectories inside `<-storageDataPath>/data/small` and `<-storageDataPath>/data/big` folders.
+Directories for months outside the configured retention are deleted on the first day of new month.
+In order to keep data according to `-retentionPeriod` max disk space usage is going to be `-retentionPeriod` + 1 month.
+For example if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
+
+
 ### Multiple retentions
 
 Just start multiple VictoriaMetrics instances with distinct values for the following flags:
@@ -776,8 +781,11 @@ mkfs.ext4 ... -O 64bit,huge_file,extent -T huge
 
 ### Monitoring
 
-VictoriaMetrics exports internal metrics in Prometheus format on the `/metrics` page.
-Add this page to Prometheus' scrape config in order to collect VictoriaMetrics metrics.
+VictoriaMetrics exports internal metrics in Prometheus format at `/metrics` page.
+These metrics may be collected either via Prometheus by adding the corresponding scrape config to it.
+Alternatively they can be self-scraped by setting `-selfScrapeInterval` command-line flag to duration greater than 0.
+For example, `-scrapeInterval=10s` would enable self-scraping of `/metrics` page with 10 seconds interval.
+
 There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176).
 
 The most interesting metrics are:
@@ -900,6 +908,13 @@ We are open to third-party pull requests provided they follow [KISS design princ
 Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.
 
 
+### Third-party contributions
+
+* [Unofficial yum repository](https://copr.fedorainfracloud.org/coprs/antonpatsev/VictoriaMetrics/) ([source code](https://github.com/patsevanton/victoriametrics-rpm))
+* [Prometheus -> VictoriaMetrics exporter #1](https://github.com/ryotarai/prometheus-tsdb-dump)
+* [Prometheus -> VictoriaMetrics exporter #2](https://github.com/AnchorFree/tsdb-remote-write)
+
+
 ## Reporting bugs
 
 Report bugs and propose new features [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).

lib/httpserver/httpserver.go

@@ -181,7 +181,7 @@ func handlerWrapper(w http.ResponseWriter, r *http.Request, rh RequestHandler) {
 		}
 		startTime := time.Now()
 		w.Header().Set("Content-Type", "text/plain")
-		writePrometheusMetrics(w)
+		WritePrometheusMetrics(w)
 		metricsHandlerDuration.UpdateDuration(startTime)
 		return
 	default:
@@ -418,7 +418,7 @@ var (
 // Errorf writes formatted error message to w and to logger.
 func Errorf(w http.ResponseWriter, format string, args ...interface{}) {
 	errStr := fmt.Sprintf(format, args...)
-	logger.Errorf("%s", errStr)
+	logger.ErrorfSkipframes(1, "%s", errStr)
 
 	// Extract statusCode from args
 	statusCode := http.StatusBadRequest

lib/httpserver/metrics.go

@@ -12,7 +12,8 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
-func writePrometheusMetrics(w io.Writer) {
+// WritePrometheusMetrics writes all the registered metrics to w in Prometheus exposition format.
+func WritePrometheusMetrics(w io.Writer) {
 	metrics.WritePrometheus(w, true)
 
 	fmt.Fprintf(w, "vm_app_version{version=%q} 1\n", buildinfo.Version)

lib/logger/logger.go

@@ -68,6 +68,11 @@ func Errorf(format string, args ...interface{}) {
 	logLevel("ERROR", format, args...)
 }
 
+// ErrorfSkipframes logs error message and skips the given number of frames for the caller.
+func ErrorfSkipframes(skipframes int, format string, args ...interface{}) {
+	logLevelSkipframes(skipframes, "ERROR", format, args...)
+}
+
 // Fatalf logs fatal message and terminates the app.
 func Fatalf(format string, args ...interface{}) {
 	logLevel("FATAL", format, args...)
@@ -79,19 +84,15 @@ func Panicf(format string, args ...interface{}) {
 }
 
 func logLevel(level, format string, args ...interface{}) {
+	logLevelSkipframes(1, level, format, args...)
+}
+
+func logLevelSkipframes(skipframes int, level, format string, args ...interface{}) {
 	if shouldSkipLog(level) {
 		return
 	}
-
-	// rate limit ERROR log messages
-	if level == "ERROR" {
-		if n := atomic.AddUint64(&errorsLogged, 1); n > 10 {
-			return
-		}
-	}
-
 	msg := fmt.Sprintf(format, args...)
-	logMessage(level, msg, 3)
+	logMessage(level, msg, 3+skipframes)
 }
 
 func errorsLoggedCleaner() {
@@ -107,13 +108,18 @@ type logWriter struct {
 }
 
 func (lw *logWriter) Write(p []byte) (int, error) {
-	if !shouldSkipLog("ERROR") {
-		logMessage("ERROR", string(p), 4)
-	}
+	logLevelSkipframes(2, "ERROR", "%s", p)
 	return len(p), nil
 }
 
 func logMessage(level, msg string, skipframes int) {
+	// rate limit ERROR log messages
+	if level == "ERROR" {
+		if n := atomic.AddUint64(&errorsLogged, 1); n > 10 {
+			return
+		}
+	}
+
 	timestamp := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
 	levelLowercase := strings.ToLower(level)
 	_, file, line, ok := runtime.Caller(skipframes)