docs/CHANGELOG.md: cut v1.86.2

These targets became obsolete after ec2c82e800

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,46 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-It would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
-to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
-and verify whether the bug is reproducible there.
-It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html).
-
-**To Reproduce**
-Steps to reproduce the behavior.
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Logs**
-Check if any warnings or errors were logged by VictoriaMetrics components
-or components in communication with VictoriaMetrics (e.g. Prometheus, Grafana).
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-For VictoriaMetrics health-state issues please provide full-length screenshots 
-of Grafana dashboards if possible:
-* [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
-* [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
-
-See how to setup monitoring here:
-* [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
-* [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
-
-**Version**
-The line returned when passing `--version` command line flag to the binary. For example:
-```
-$ ./victoria-metrics-prod --version
-victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
-```
-
-**Used command-line flags**
-Please provide the command-line flags used for running VictoriaMetrics and its components. 

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,86 @@
+name: Bug report
+description: Create a report to help us improve
+labels: [bug]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
+        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+        and verify whether the bug is reproducible there.
+        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html) first.
+  - type: textarea
+    id: describe-the-bug
+    attributes:
+      label: Describe the bug
+      description: |
+        A clear and concise description of what the bug is.
+      placeholder: |
+        When I do `A` VictoriaMetrics does `B`. I expect it to do `C`.
+    validations:
+      required: true
+  - type: textarea
+    id: to-reproduce
+    attributes:
+      label: To Reproduce
+      description: |
+        Steps to reproduce the behavior.
+        If reproducing an issue requires some specific configuration file, please paste it here.
+      placeholder: |
+        Steps to reproduce the behavior.
+    validations:
+      required: true
+  - type: textarea
+    id: version
+    attributes:
+      label: Version
+      description: |
+        The line returned when passing `--version` command line flag to the binary. For example:
+        ```
+        $ ./victoria-metrics-prod --version
+        victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
+        ```
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs
+      description: |
+        Check if any warnings or errors were logged by VictoriaMetrics components
+        or components in communication with VictoriaMetrics (e.g. Prometheus, Grafana).
+    validations:
+      required: false
+  - type: textarea
+    id: screenshots
+    attributes:
+      label: Screenshots
+      description: |
+        If applicable, add screenshots to help explain your problem.
+  
+        For VictoriaMetrics health-state issues please provide full-length screenshots
+        of Grafana dashboards if possible:
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
+        
+        See how to setup monitoring here:
+          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
+          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
+    validations:
+      required: false
+  - type: textarea
+    id: flags
+    attributes:
+      label: Used command-line flags
+      description: |
+        Please provide the command-line flags used for running VictoriaMetrics and its components.
+    validations:
+      required: false
+  - type: textarea
+    id: additional-info
+    attributes:
+      label: Additional information
+      placeholder: |
+        Additional information that doesn't fit elsewhere
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/configuration.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Ask on Slack
+    url: https://slack.victoriametrics.com/
+    about: You can ask for help here!

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,43 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: [enhancement]
+body:
+  - type: textarea
+    id: describe-the-problem
+    attributes:
+      label: Is your feature request related to a problem? Please describe
+      description: |
+        A clear and concise description of what the problem is.
+      placeholder: |
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: false
+  - type: textarea
+    id: describe-the-solution
+    attributes:
+      label: Describe the solution you'd like
+      description: |
+        A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative-solutions
+    attributes:
+      label: Describe alternatives you've considered
+      description: |
+        A clear and concise description of any alternative solutions or features you've considered.
+      placeholder: |
+        I have tried to do `A`, but that doesn't solve a problem completely.
+        I have tried to do `A` and `B`, but implementing this would be better.
+    validations:
+      required: false
+  - type: textarea
+    id: feature-additional-info
+    attributes:
+      label: Additional information
+      description: |
+        Additional information which you consider helpful for implementing this feature.
+      placeholder: |
+        Add any other context or screenshots about the feature request here.
+    validations:
+      required: false

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.19.4
+          go-version: 1.19.5
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/codeql-analysis-js.yml

@@ -0,0 +1,42 @@
+name: "CodeQL - JS"
+
+on:
+  push:
+    branches: [master, cluster]
+    paths:
+      - "**.js"
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master, cluster]
+    paths:
+      - "**.js"
+  schedule:
+    - cron: "30 18 * * 2"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["javascript"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -13,12 +13,20 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ master, cluster ]
+    branches: [master, cluster]
+    paths-ignore:
+      - "**.md"
+      - "**.txt"
+      - "**.js"
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ master, cluster ]
+    branches: [master, cluster]
+    paths-ignore:
+      - "**.md"
+      - "**.txt"
+      - "**.js"
   schedule:
-    - cron: '30 18 * * 2'
+    - cron: "30 18 * * 2"
 
 jobs:
   analyze:
@@ -32,45 +40,47 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'go', 'javascript' ]
+        language: ["go"]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
         # Learn more about CodeQL language support at https://git.io/codeql-language-support
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
 
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.19
-      if: ${{ matrix.language == 'go' }}
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.5
+          check-latest: true
+          cache: true
+        if: ${{ matrix.language == 'go' }}
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
 
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+      #- run: |
+      #   make bootstrap
+      #   make release
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/main-test.yml

@@ -0,0 +1,66 @@
+name: main - test
+on:
+  push:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+  pull_request:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.5
+          check-latest: true
+          cache: true
+
+      - name: Dependencies
+        run: |
+          make install-golangci-lint
+          make check-all
+          git diff --exit-code
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        scenario: ["test-full", "test-pure", "test-full-386"]
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.5
+          check-latest: true
+          cache: true
+
+      - name: run tests
+        run: |
+          make ${{ matrix.scenario}}
+
+      - name: Publish coverage
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.txt

.github/workflows/main.yml

@@ -1,13 +1,10 @@
 name: main
 on:
-  push:
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
-  pull_request:
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
+  workflow_run:
+    workflows: ["main - test"]
+    types:
+      - completed
+
 permissions:
   contents: read
 
@@ -16,30 +13,17 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      - name: Setup Go
-        uses: actions/setup-go@main
-        with:
-          go-version: 1.19.4
-        id: go
       - name: Code checkout
-        uses: actions/checkout@master
-      - name: Dependencies
-        run: |
-          make install-golint
-          make install-errcheck
-          make install-golangci-lint
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.5
+          check-latest: true
+          cache: true
+
       - name: Build
         run: |
-          export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
-          make check-all
-          git diff --exit-code
-          make test-full
-          make test-pure
-          make test-full-386
           make victoria-metrics-crossbuild
           make vmuitils-crossbuild
-      - name: Publish coverage
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.txt
-

.github/workflows/nightly-build.yml

@@ -12,13 +12,28 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      - name: Setup Go
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.19.4
+          go-version: 1.19.5
         id: go
-      - name: Code checkout
+      -
+        name: Setup docker scan
+        run: |
+          mkdir -p ~/.docker/cli-plugins && \
+          curl https://github.com/docker/scan-cli-plugin/releases/latest/download/docker-scan_linux_amd64 -L -s -S -o ~/.docker/cli-plugins/docker-scan &&\
+          chmod +x ~/.docker/cli-plugins/docker-scan
+      -
+        name: Code checkout
         uses: actions/checkout@master
-      - name: Publish
+      -
+        name: Publish
         run: |
           LATEST_TAG=nightly PKG_TAG=nightly make publish

.golangci.yml

@@ -0,0 +1,15 @@
+run:
+  timeout: 2m
+
+enable:
+  - revive
+
+issues:
+  exclude-rules:
+    - linters:
+      - staticcheck
+      text: "SA(4003|1019|5011):"
+
+linters-settings:
+  errcheck:
+    exclude: ./errcheck_excludes.txt

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2022 VictoriaMetrics, Inc.
+   Copyright 2019-2023 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -167,10 +167,10 @@ vmutils-crossbuild: \
 	vmutils-windows-amd64
 
 publish-release:
-	git checkout $(TAG) && $(MAKE) release publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish
+	git checkout $(TAG) && LATEST_TAG=stable $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && LATEST_TAG=cluster-stable $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && LATEST_TAG=enterprise-stable $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && LATEST_TAG=enterprise-cluster-stable $(MAKE) release publish
 
 release: \
 	release-victoria-metrics \
@@ -315,21 +315,7 @@ vet:
 	go vet ./lib/...
 	go vet ./app/...
 
-lint: install-golint
-	golint lib/...
-	golint app/...
-
-install-golint:
-	which golint || go install golang.org/x/lint/golint@latest
-
-errcheck: install-errcheck
-	errcheck -exclude=errcheck_excludes.txt ./lib/...
-	errcheck -exclude=errcheck_excludes.txt ./app/...
-
-install-errcheck:
-	which errcheck || go install github.com/kisielk/errcheck@latest
-
-check-all: fmt vet lint errcheck golangci-lint govulncheck
+check-all: fmt vet golangci-lint govulncheck
 
 test:
 	go test ./lib/... ./app/...
@@ -380,10 +366,10 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
+	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.50.1
 
 govulncheck: install-govulncheck
 	govulncheck ./...

README.md

@@ -40,19 +40,36 @@ VictoriaMetrics has the following prominent features:
 * It can be used as a drop-in replacement for Prometheus in Grafana, because it supports [Prometheus querying API](#prometheus-querying-api-usage).
 * It can be used as a drop-in replacement for Graphite in Grafana, because it supports [Graphite API](#graphite-api-usage).
 * It features easy setup and operation:
-  * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d) without external dependencies.
+  * VictoriaMetrics consists of a single [small executable](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d)
+    without external dependencies.
   * All the configuration is done via explicit command-line flags with reasonable defaults.
   * All the data is stored in a single directory pointed by `-storageDataPath` command-line flag.
-  * Easy and fast backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) to S3 or GCS can be done with [vmbackup](https://docs.victoriametrics.com/vmbackup.html) / [vmrestore](https://docs.victoriametrics.com/vmrestore.html) tools. See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
-* It implements PromQL-based query language - [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html), which provides improved functionality on top of PromQL.
+  * Easy and fast backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+    can be done with [vmbackup](https://docs.victoriametrics.com/vmbackup.html) / [vmrestore](https://docs.victoriametrics.com/vmrestore.html) tools.
+    See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
+* It implements PromQL-like query language - [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html), which provides improved functionality on top of PromQL.
 * It provides global query view. Multiple Prometheus instances or any other data sources may ingest data into VictoriaMetrics. Later this data may be queried via a single query.
-* It provides high performance and good vertical and horizontal scalability for both [data ingestion](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b) and [data querying](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4). It [outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
-* It [uses 10x less RAM than InfluxDB](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) and [up to 7x less RAM than Prometheus, Thanos or Cortex](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f) when dealing with millions of unique time series (aka [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality)).
+* It provides high performance and good vertical and horizontal scalability for both
+  [data ingestion](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b)
+  and [data querying](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4).
+  It [outperforms InfluxDB and TimescaleDB by up to 20x](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
+* It [uses 10x less RAM than InfluxDB](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893)
+  and [up to 7x less RAM than Prometheus, Thanos or Cortex](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f)
+  when dealing with millions of unique time series (aka [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality)).
 * It is optimized for time series with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate).
-* It provides high data compression, so [up to 70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) may be crammed into limited storage comparing to TimescaleDB and [up to 7x less storage space is required compared to Prometheus, Thanos or Cortex](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f).
-* It is optimized for storage with high-latency IO and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc). See [disk IO graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
-* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, M3DB, Cortex, InfluxDB or TimescaleDB. See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae), [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683) and [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk from [PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
-* It protects the storage from data corruption on unclean shutdown (i.e. OOM, hardware reset or `kill -9`) thanks to [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+* It provides high data compression, so up to 70x more data points may be stored into limited storage comparing to TimescaleDB
+  according to [these benchmarks](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4)
+  and up to 7x less storage space is required compared to Prometheus, Thanos or Cortex
+  according to [this benchmark](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f).
+* It is optimized for storage with high-latency IO and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc).
+  See [disk IO graphs from these benchmarks](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b).
+* A single-node VictoriaMetrics may substitute moderately sized clusters built with competing solutions such as Thanos, M3DB, Cortex, InfluxDB or TimescaleDB.
+  See [vertical scalability benchmarks](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae),
+  [comparing Thanos to VictoriaMetrics cluster](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683)
+  and [Remote Write Storage Wars](https://promcon.io/2019-munich/talks/remote-write-storage-wars/) talk
+  from [PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
+* It protects the storage from data corruption on unclean shutdown (i.e. OOM, hardware reset or `kill -9`) thanks to
+  [the storage architecture](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
 * It supports metrics' scraping, ingestion and [backfilling](#backfilling) via the following protocols:
   * [Metrics scraping from Prometheus exporters](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
   * [Prometheus remote write API](#prometheus-setup).
@@ -65,11 +82,15 @@ VictoriaMetrics has the following prominent features:
   * [Arbitrary CSV data](#how-to-import-csv-data).
   * [Native binary format](#how-to-import-data-in-native-format).
   * [DataDog agent or DogStatsD](#how-to-send-data-from-datadog-agent).
+* It supports powerful [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html), which can be used as a [statsd](https://github.com/statsd/statsd) alternative.
 * It supports metrics [relabeling](#relabeling).
-* It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
-* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://docs.victoriametrics.com/enterprise.html).
+* It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and
+  [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
+* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data
+  and various [Enterprise workloads](https://docs.victoriametrics.com/enterprise.html).
 * It has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
-* It can store data on [NFS-based storages](https://en.wikipedia.org/wiki/Network_File_System) such as [Amazon EFS](https://aws.amazon.com/efs/) and [Google Filestore](https://cloud.google.com/filestore).
+* It can store data on [NFS-based storages](https://en.wikipedia.org/wiki/Network_File_System) such as [Amazon EFS](https://aws.amazon.com/efs/)
+  and [Google Filestore](https://cloud.google.com/filestore).
 
 See also [various Articles about VictoriaMetrics](https://docs.victoriametrics.com/Articles.html).
 
@@ -84,7 +105,6 @@ Case studies:
 * [Brandwatch](https://docs.victoriametrics.com/CaseStudies.html#brandwatch)
 * [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
 * [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
-* [Dreamteam](https://docs.victoriametrics.com/CaseStudies.html#dreamteam)
 * [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
 * [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
 * [Grammarly](https://docs.victoriametrics.com/CaseStudies.html#grammarly)
@@ -271,6 +291,8 @@ Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](
 VictoriaMetrics provides UI for query troubleshooting and exploration. The UI is available at `http://victoriametrics:8428/vmui`.
 The UI allows exploring query results via graphs and tables.
 It also provides the following features:
+
+- [metrics explorer](#metrics-explorer)
 - [cardinality explorer](#cardinality-explorer)
 - [query tracer](#query-tracing)
 - [top queries explorer](#top-queries)
@@ -306,9 +328,21 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric
 * queries with the biggest average execution duration;
 * queries that took the most summary time for execution.
 
+## Metrics explorer
+
+[VMUI](#vmui) provides an ability to explore metrics exported by a particular `job` / `instance` in the following way:
+
+1. Open the `vmui` at `http://victoriametrics:8428/vmui/`.
+2. Click the `Explore metrics` tab.
+3. Select the `job` you want to explore.
+4. Optionally select the `instance` for the selected job to explore.
+5. Select metrics you want to explore and compare.
+
+It is possible to change the selected time range for the graphs in the top right corner.
+
 ## Cardinality explorer
 
-VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways:
+VictoriaMetrics provides an ability to explore time series cardinality at `Explore cardinality` tab in [vmui](#vmui) in the following ways:
 
 - To identify metric names with the highest number of series.
 - To identify labels with the highest number of series.
@@ -363,7 +397,7 @@ DataDog agent allows configuring destinations for metrics sending via ENV variab
 or via [configuration file](https://docs.datadoghq.com/agent/guide/agent-configuration-files/) in section `dd_url`.
 
 <p align="center">
-  <img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM.png" width="800">
+  <img src="docs/Single-server-VictoriaMetrics-sending_DD_metrics_to_VM.png" width="800">
 </p>
 
 To configure DataDog agent via ENV variable add the following prefix:
@@ -397,7 +431,7 @@ DataDog allows configuring [Dual Shipping](https://docs.datadoghq.com/agent/guid
 sending via ENV variable `DD_ADDITIONAL_ENDPOINTS` or via configuration file `additional_endpoints`.
  
 <p align="center">
-  <img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM_and_DD.png" width="800">
+  <img src="docs/Single-server-VictoriaMetrics-sending_DD_metrics_to_VM_and_DD.png" width="800">
 </p>
  
 Run DataDog using the following ENV variable with VictoriaMetrics as additional metrics receiver:
@@ -701,8 +735,7 @@ VictoriaMetrics accepts optional `extra_label=<label_name>=<label_value>` query
 VictoriaMetrics accepts optional `extra_filters[]=series_selector` query arg, which can be used for enforcing arbitrary label filters for queries. For example,
 `/api/v1/query_range?extra_filters[]={env=~"prod|staging",user="xyz"}&query=<query>` would automatically add `{env=~"prod|staging",user="xyz"}` label filters to the given `<query>`. This functionality can be used for limiting the scope of time series visible to the given tenant. It is expected that the `extra_filters[]` query args are automatically set by auth proxy sitting in front of VictoriaMetrics. See [vmauth](https://docs.victoriametrics.com/vmauth.html) and [vmgateway](https://docs.victoriametrics.com/vmgateway.html) as examples of such proxies.
 
-VictoriaMetrics accepts relative times in `time`, `start` and `end` query args additionally to unix timestamps and [RFC3339](https://www.ietf.org/rfc/rfc3339.txt).
-For example, the following query would return data for the last 30 minutes: `/api/v1/query_range?start=-30m&query=...`.
+VictoriaMetrics accepts multiple formats for `time`, `start` and `end` query args - see [these docs](#timestamp-formats).
 
 VictoriaMetrics accepts `round_digits` query arg for `/api/v1/query` and `/api/v1/query_range` handlers. It can be used for rounding response values to the given number of digits after the decimal point. For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
 
@@ -727,6 +760,18 @@ Additionally, VictoriaMetrics provides the following handlers:
   For example, request to `/api/v1/status/top_queries?topN=5&maxLifetime=30s` would return up to 5 queries per list, which were executed during the last 30 seconds.
   VictoriaMetrics tracks the last `-search.queryStats.lastQueriesCount` queries with durations at least `-search.queryStats.minQueryDuration`.
 
+### Timestamp formats
+
+VictoriaMetrics accepts the following formats for `time`, `start` and `end` query args
+in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and
+in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series).
+
+- Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`.
+- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`.
+- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`.
+- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`.
+
+
 ## Graphite API usage
 
 VictoriaMetrics supports data ingestion in Graphite protocol - see [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details.
@@ -946,8 +991,9 @@ Each JSON line contains samples for a single time series. An example output:
 {"metric":{"__name__":"up","job":"prometheus","instance":"localhost:9090"},"values":[1,1,1],"timestamps":[1549891461511,1549891476511,1549891491511]}
 ```
 
-Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
-unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data.
+See [allowed formats](#timestamp-formats) for these args.
+
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/api/v1/export -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@@ -994,8 +1040,9 @@ where:
 * `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
 for metrics to export.
 
-Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
-unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data.
+See [allowed formats](#timestamp-formats) for these args.
+
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/api/v1/export/csv -d 'format=<format>' -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@@ -1021,8 +1068,9 @@ wget -O- -q 'http://your_victoriametrics_instance:8428/api/v1/series/count' | jq
 # relaunch victoriametrics with search.maxExportSeries more than value from previous command
 ```
 
-Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data. These args may contain either
-unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+Optional `start` and `end` args may be added to the request in order to limit the time frame for the exported data.
+See [allowed formats](#timestamp-formats) for these args.
+
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/api/v1/export/native -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@@ -1258,7 +1306,8 @@ VictoriaMetrics exports [Prometheus-compatible federation data](https://promethe
 at `http://<victoriametrics-addr>:8428/federate?match[]=<timeseries_selector_for_federation>`.
 
 Optional `start` and `end` args may be added to the request in order to scrape the last point for each selected time series on the `[start ... end]` interval.
-`start` and `end` may contain either unix timestamp in seconds or [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) values.
+See [allowed formats](#timestamp-formats) for these args.
+
 For example:
 ```console
 curl http://<victoriametrics-addr>:8428/federate -d 'match[]=<timeseries_selector_for_export>' -d 'start=1654543486' -d 'end=1654543486'
@@ -1426,8 +1475,8 @@ This increases overhead during data querying, since VictoriaMetrics needs to rea
 bigger number of parts per each request. That's why it is recommended to have at least 20%
 of free disk space under directory pointed by `-storageDataPath` command-line flag.
 
-Information about merging process is available in [the dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
-and [the dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176).
+Information about merging process is available in [the dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics/)
+and [the dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/).
 See more details in [monitoring docs](#monitoring).
 
 See [this article](https://valyala.medium.com/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282) for more details.
@@ -1594,8 +1643,8 @@ Alternatively, single-node VictoriaMetrics can self-scrape the metrics when `-se
 set to duration greater than 0. For example, `-selfScrapeInterval=10s` would enable self-scraping of `/metrics` page 
 with 10 seconds interval.
 
-Official Grafana dashboards available for [single-node](https://grafana.com/dashboards/10229) 
-and [clustered](https://grafana.com/grafana/dashboards/11176) VictoriaMetrics. 
+Official Grafana dashboards available for [single-node](https://grafana.com/grafana/dashboards/10229-victoriametrics/) 
+and [clustered](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/) VictoriaMetrics. 
 See an [alternative dashboard for clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11831) 
 created by community.
 
@@ -1844,8 +1893,8 @@ The following metrics for each type of cache are exported at [`/metrics` page](#
 * `vm_cache_misses_total` - the number of cache misses
 * `vm_cache_entries` - the number of entries in the cache
 
-Both Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
-and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176)
+Both Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics/)
+and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
 contain `Caches` section with cache metrics visualized. The panels show the current
 memory usage by each type of cache, and also a cache hit rate. If hit rate is close to 100%
 then cache efficiency is already very high and does not need any tuning.
@@ -2175,6 +2224,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
      Format for logs. Possible values: default, json (default "default")
+  -loggerJSONFields string
+     Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg
   -loggerLevel string
      Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
@@ -2184,7 +2235,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -maxConcurrentInserts int
-     The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16)
+     The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
   -maxInsertRequestSize size
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
@@ -2281,6 +2332,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.minResponseSizeForStreamParse size
      The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1000000)
+  -promscrape.nomad.waitTime duration
+     Wait time used by Nomad service discovery. Default value is used if not set
+  -promscrape.nomadSDCheckInterval duration
+     Interval for checking for changes in Nomad. This works only if nomad_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#nomad_sd_configs for details (default 30s)
   -promscrape.noStaleMarkers
      Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.openstackSDCheckInterval duration
@@ -2427,6 +2482,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 10000000)
   -storageDataPath string
      Path to storage data (default "victoria-metrics-data")
+  -streamAggr.config string
+     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html .See also -remoteWrite.streamAggr.keepInput
+  -streamAggr.keepInput
+     Whether to keep input samples after the aggregation with -streamAggr.config .By default the input is dropped after the aggregation, so only the aggregate data is stored. See https://docs.victoriametrics.com/stream-aggregation.html
   -tls
      Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string
@@ -2444,4 +2503,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Show VictoriaMetrics version
   -vmalert.proxyURL string
      Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules
+  -vmui.customDashboardsPath string
+     Optional path to vmui dashboards. See https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/vmui/packages/vmui/public/dashboards
 ```

app/victoria-metrics/main_test.go

@@ -261,7 +261,7 @@ func testRead(t *testing.T) {
 					for _, q := range test.Query {
 						q = testutil.PopulateTimeTplString(q, insertionTime)
 						if test.Issue != "" {
-							test.Issue = "Regression in " + test.Issue
+							test.Issue = "\nRegression in " + test.Issue
 						}
 						switch true {
 						case strings.HasPrefix(q, "/api/v1/export"):
@@ -284,7 +284,7 @@ func testRead(t *testing.T) {
 							queryResult := Query{}
 							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
 							if err := checkQueryResult(queryResult, test.ResultQuery); err != nil {
-								t.Fatalf("Query. %s fails with error %s.%s", q, err, test.Issue)
+								t.Fatalf("Query. %s fails with error: %s.%s", q, err, test.Issue)
 							}
 						default:
 							t.Fatalf("unsupported read query %s", q)

app/victoria-metrics/test/parser_test.go

@@ -12,7 +12,7 @@ func TestPopulateTimeTplString(t *testing.T) {
 	}
 	f := func(s, resultExpected string) {
 		t.Helper()
-		result := PopulateTimeTplString(s, now)
+		result := PopulateTimeTplString(s, now.UTC())
 		if result != resultExpected {
 			t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
 		}

app/victoria-metrics/testdata/graphite/subquery-aggregation.json

@@ -6,7 +6,7 @@
     "forms_daily_count;item=x 2 {TIME_S-2m}",
     "forms_daily_count;item=y 3 {TIME_S-1m}",
     "forms_daily_count;item=y 4 {TIME_S-2m}"],
-  "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}"],
+  "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}&latency_offset=1ms"],
   "result_query": {
     "status":"success",
     "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","2"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","4"]}]}

app/vmagent/README.md

@@ -24,8 +24,8 @@ additionally to [discovering Prometheus-compatible targets and scraping metrics
   see [these docs](https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter).
 * Can add, remove and modify labels (aka tags) via Prometheus relabeling. Can filter data before sending it to remote storage. See [these docs](#relabeling) for details.
 * Can accept data via all the ingestion protocols supported by VictoriaMetrics - see [these docs](#how-to-push-data-to-vmagent).
-* Can replicate collected metrics simultaneously to multiple remote storage systems -
-  see [these docs](#replication-and-high-availability).
+* Can aggregate incoming samples by time and by labels before sending them to remote storage - see [these docs](https://docs.victoriametrics.com/stream-aggregation.html).
+* Can replicate collected metrics simultaneously to multiple remote storage systems - see [these docs](#replication-and-high-availability).
 * Works smoothly in environments with unstable connections to remote storage. If the remote storage is unavailable, the collected metrics
   are buffered at `-remoteWrite.tmpDataPath`. The buffered metrics are sent to remote storage as soon as the connection
   to the remote storage is repaired. The maximum disk usage for the buffer can be limited with `-remoteWrite.maxDiskUsagePerURL`.
@@ -60,6 +60,8 @@ Example command line:
 /path/to/vmagent -promscrape.config=/path/to/prometheus.yml -remoteWrite.url=https://victoria-metrics-host:8428/api/v1/write
 ```
 
+Example of scrape configuration for `-promscrape.config` argument you can find [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus.yml)
+
 See [how to scrape Prometheus-compatible targets](#how-to-collect-metrics-in-prometheus-format) for more details.
 
 If you use single-node VictoriaMetrics, then you can discover and scrape Prometheus-compatible targets directly from VictoriaMetrics
@@ -126,6 +128,12 @@ If you use Prometheus only for scraping metrics from various targets and forward
 then `vmagent` can replace Prometheus. Typically, `vmagent` requires lower amounts of RAM, CPU and network bandwidth compared with Prometheus.
 See [these docs](#how-to-collect-metrics-in-prometheus-format) for details.
 
+### Statsd alternative
+
+`vmagent` can be used as an alternative to [statsd](https://github.com/statsd/statsd)
+when [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) is enabled.
+See [these docs](https://docs.victoriametrics.com/stream-aggregation.html#statsd-alternative) for details.
+
 ### Flexible metrics relay
 
 `vmagent` can accept metrics in [various popular data ingestion protocols](#how-to-push-data-to-vmagent), apply [relabeling](#relabeling)
@@ -598,8 +606,8 @@ provide the following tools for debugging target-level and metric-level relabeli
 
 - Target-level debugging (e.g. `relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs))
   can be performed by navigating to `http://vmagent:8429/targets` page (`http://victoriametrics:8428/targets` page for single-node VictoriaMetrics)
-  and clicking the `debug` link at the target, which must be debugged.
-  The opened page will show step-by-step results for the actual relabeling rules applied to the target labels.
+  and clicking the `debug target relabeling` link at the target, which must be debugged.
+  The opened page will show step-by-step results for the actual target relabeling rules applied to the discovered target labels.
 
   The `http://vmagent:8429/targets` page shows only active targets. If you need to understand why some target
   is dropped during the relabeling, then navigate to `http://vmagent:8428/service-discovery` page
@@ -608,11 +616,9 @@ provide the following tools for debugging target-level and metric-level relabeli
   which result to target drop.
 
 - Metric-level debugging (e.g. `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs)
-  and all the relabeling, which can be set up via `-relabelConfig`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig`
-  command-line flags) can be performed by navigating to `http://vmagent:8429/metric-relabel-debug` page
-  (`http://victoriametrics:8428/metric-relabel-debug` page for single-node VictoriaMetrics)
-  and submitting there relabeling rules together with the metric to be relabeled.
-  The page will show step-by-step results for the entered relabeling rules executed against the entered metric.
+  can be performed by navigating to `http://vmagent:8429/targets` page (`http://victoriametrics:8428/targets` page for single-node VictoriaMetrics)
+  and clicking the `debug metrics relabeling` link at the target, which must be debugged.
+  The opened page will show step-by-step results for the actual metric relabeling rules applied to the given target labels.
 
 ## Prometheus staleness markers
 
@@ -816,7 +822,7 @@ See also [cardinality explorer docs](https://docs.victoriametrics.com/#cardinali
 We recommend setting up regular scraping of this page either through `vmagent` itself or by Prometheus
 so that the exported metrics may be analyzed later.
 
-Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683) for `vmagent` state overview.
+Use official [Grafana dashboard](https://grafana.com/grafana/dashboards/12683-victoriametrics-vmagent/) for `vmagent` state overview.
 Graphs on this dashboard contain useful hints - hover the `i` icon at the top left corner of each graph in order to read it.
 If you have suggestions for improvements or have found a bug - please open an issue on github or add a review to the dashboard.
 
@@ -1232,6 +1238,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
      Format for logs. Possible values: default, json (default "default")
+  -loggerJSONFields string
+     Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg
   -loggerLevel string
      Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
@@ -1241,7 +1249,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -maxConcurrentInserts int
-     The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16)
+     The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
   -maxInsertRequestSize size
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
@@ -1332,6 +1340,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.minResponseSizeForStreamParse size
      The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1000000)
+  -promscrape.nomad.waitTime duration
+     Wait time used by Nomad service discovery. Default value is used if not set
+  -promscrape.nomadSDCheckInterval duration
+     Interval for checking for changes in Nomad. This works only if nomad_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#nomad_sd_configs for details (default 30s)
   -promscrape.noStaleMarkers
      Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.openstackSDCheckInterval duration
@@ -1443,7 +1455,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
-     Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details
+     Optional path to file with relabeling configs, which are applied to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling
   -remoteWrite.roundDigits array
      Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
      Supports array of values separated by comma or specified via multiple flags.
@@ -1455,6 +1467,12 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -remoteWrite.significantFigures array
      The number of significant figures to leave in metric values before writing them to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. This option may be used for improving data compression for the stored metrics. See also -remoteWrite.roundDigits
      Supports array of values separated by comma or specified via multiple flags.
+  -remoteWrite.streamAggr.config array
+     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html . See also -remoteWrite.streamAggr.keepInput
+     Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.streamAggr.keepInput array
+     Whether to keep input samples after the aggregation with -remoteWrite.streamAggr.config. By default the input is dropped after the aggregation, so only the aggregate data is sent to the -remoteWrite.url. See https://docs.victoriametrics.com/stream-aggregation.html
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsCAFile array
      Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. By default system CA is used
      Supports an array of values separated by comma or specified via multiple flags.
@@ -1476,7 +1494,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.multitenantURL
      Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelConfig array
-     Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url
+     Optional path to relabel configs for the corresponding -remoteWrite.url. See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling
      Supports an array of values separated by comma or specified via multiple flags.
   -sortLabels
      Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit

app/vmagent/csvimport/request_handler.go

@@ -10,7 +10,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -26,10 +25,8 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(at, rows, extraLabels)
-		})
+	return parser.ParseStream(req, func(rows []parser.Row) error {
+		return insertRows(at, rows, extraLabels)
 	})
 }
 

app/vmagent/datadog/request_handler.go

@@ -10,7 +10,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -28,11 +27,9 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		ce := req.Header.Get("Content-Encoding")
-		return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
-			return insertRows(at, series, extraLabels)
-		})
+	ce := req.Header.Get("Content-Encoding")
+	return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+		return insertRows(at, series, extraLabels)
 	})
 }
 

app/vmagent/graphite/request_handler.go

@@ -7,7 +7,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -20,9 +19,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, insertRows)
-	})
+	return parser.ParseStream(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vmagent/influx/request_handler.go

@@ -16,7 +16,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -37,10 +36,8 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
-			return insertRows(nil, db, rows, nil)
-		})
+	return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+		return insertRows(nil, db, rows, nil)
 	})
 }
 
@@ -52,15 +49,13 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-		q := req.URL.Query()
-		precision := q.Get("precision")
-		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
-		db := q.Get("db")
-		return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
-			return insertRows(at, db, rows, extraLabels)
-		})
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	q := req.URL.Query()
+	precision := q.Get("precision")
+	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	db := q.Get("db")
+	return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+		return insertRows(at, db, rows, extraLabels)
 	})
 }
 

app/vmagent/main.go

@@ -38,7 +38,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -104,7 +103,6 @@ func main() {
 	startTime := time.Now()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
-	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
 			return influx.InsertHandlerForReader(r, false)
@@ -349,12 +347,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
-			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
-				StatusCode: http.StatusUnauthorized,
-			}
-			httpserver.Errorf(w, r, "%s", err)
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -363,12 +356,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
-			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
-				StatusCode: http.StatusUnauthorized,
-			}
-			httpserver.Errorf(w, r, "%s", err)
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()

app/vmagent/native/request_handler.go

@@ -12,7 +12,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -31,10 +30,8 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
-			return insertRows(at, block, extraLabels)
-		})
+	return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+		return insertRows(at, block, extraLabels)
 	})
 }
 

app/vmagent/opentsdb/request_handler.go

@@ -7,7 +7,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -20,9 +19,7 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, insertRows)
-	})
+	return parser.ParseStream(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vmagent/opentsdbhttp/request_handler.go

@@ -9,7 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -25,10 +24,8 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(at, rows, extraLabels)
-		})
+	return parser.ParseStream(req, func(rows []parser.Row) error {
+		return insertRows(at, rows, extraLabels)
 	})
 }
 

app/vmagent/prometheusimport/request_handler.go

@@ -11,7 +11,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -31,21 +30,17 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-		return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
-			return insertRows(at, rows, extraLabels)
-		}, nil)
-	})
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+		return insertRows(at, rows, extraLabels)
+	}, nil)
 }
 
 // InsertHandlerForReader processes metrics from given reader with optional gzip format
 func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, 0, isGzipped, func(rows []parser.Row) error {
-			return insertRows(nil, rows, nil)
-		}, nil)
-	})
+	return parser.ParseStream(r, 0, isGzipped, func(rows []parser.Row) error {
+		return insertRows(nil, rows, nil)
+	}, nil)
 }
 
 func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {

app/vmagent/promremotewrite/request_handler.go

@@ -13,7 +13,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -29,19 +28,15 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
-			return insertRows(at, tss, extraLabels)
-		})
+	return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+		return insertRows(at, tss, extraLabels)
 	})
 }
 
 // InsertHandlerForReader processes metrics from given reader
 func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, func(tss []prompb.TimeSeries) error {
-			return insertRows(at, tss, nil)
-		})
+	return parser.ParseStream(r, func(tss []prompb.TimeSeries) error {
+		return insertRows(at, tss, nil)
 	})
 }
 

app/vmagent/remotewrite/relabel.go

@@ -15,11 +15,13 @@ import (
 var (
 	unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
 		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
-	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+
-		"The path can point either to local file or to http url. These entries are applied to all the metrics "+
-		"before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details")
-	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+
-		"The path can point either to local file or to http url")
+	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
+		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
+		"The path can point either to local file or to http url. "+
+		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
+	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
+		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
+		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
 
 	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
 		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
@@ -86,7 +88,7 @@ func initLabelsGlobal() {
 }
 
 func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
-	if len(extraLabels) == 0 && pcs.Len() == 0 {
+	if len(extraLabels) == 0 && pcs.Len() == 0 && !*usePromCompatibleNaming {
 		// Nothing to change.
 		return tss
 	}

app/vmagent/remotewrite/relabel_test.go

@@ -0,0 +1,49 @@
+package remotewrite
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+func TestApplyRelabeling(t *testing.T) {
+	f := func(extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
+		rctx := &relabelCtx{}
+		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
+		gotTss := rctx.applyRelabeling(tss, extraLabels, pcs)
+		if !reflect.DeepEqual(gotTss, expTss) {
+			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
+		}
+	}
+
+	f(nil, nil, "up", "up")
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, "up", `up{foo="bar"}`)
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, `up{foo="baz"}`, `up{foo="bar"}`)
+
+	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
+- target_label: "foo"
+  replacement: "aaa"
+- action: labeldrop
+  regex: "env.*"
+`))
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	f(nil, pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
+
+	oldVal := *usePromCompatibleNaming
+	*usePromCompatibleNaming = true
+	f(nil, nil, `foo.bar`, `foo_bar`)
+	*usePromCompatibleNaming = oldVal
+}
+
+func parseSeries(data string) []prompbmarshal.TimeSeries {
+	var tss []prompbmarshal.TimeSeries
+	tss = append(tss, prompbmarshal.TimeSeries{
+		Labels: promutils.MustNewLabelsFromString(data).GetLabels(),
+	})
+	return tss
+}

app/vmagent/remotewrite/remotewrite.go

@@ -21,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/cespare/xxhash/v2"
@@ -58,6 +59,13 @@ var (
 		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
 	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
 		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent.html#cardinality-limiter")
+
+	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
+		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
+		"See also -remoteWrite.streamAggr.keepInput")
+	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep input samples after the aggregation with -remoteWrite.streamAggr.config. "+
+		"By default the input is dropped after the aggregation, so only the aggregate data is sent to the -remoteWrite.url. "+
+		"See https://docs.victoriametrics.com/stream-aggregation.html")
 )
 
 var (
@@ -140,6 +148,7 @@ func Init() {
 		logger.Fatalf("cannot load relabel configs: %s", err)
 	}
 	allRelabelConfigs.Store(rcs)
+
 	configSuccess.Set(1)
 	configTimestamp.Set(fasttime.UnixTimestamp())
 
@@ -435,9 +444,13 @@ var (
 )
 
 type remoteWriteCtx struct {
-	idx        int
-	fq         *persistentqueue.FastQueue
-	c          *client
+	idx int
+	fq  *persistentqueue.FastQueue
+	c   *client
+
+	sas                 *streamaggr.Aggregators
+	streamAggrKeepInput bool
+
 	pss        []*pendingSeries
 	pssNextIdx uint64
 
@@ -469,6 +482,7 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	}
 	c.init(argIdx, *queues, sanitizedURL)
 
+	// Initialize pss
 	sf := significantFigures.GetOptionalArgOrDefault(argIdx, 0)
 	rd := roundDigits.GetOptionalArgOrDefault(argIdx, 100)
 	pssLen := *queues
@@ -481,7 +495,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	for i := range pss {
 		pss[i] = newPendingSeries(fq.MustWriteBlock, sf, rd)
 	}
-	return &remoteWriteCtx{
+
+	rwctx := &remoteWriteCtx{
 		idx: argIdx,
 		fq:  fq,
 		c:   c,
@@ -490,6 +505,19 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
 	}
+
+	// Initialize sas
+	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
+	if sasFile != "" {
+		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal)
+		if err != nil {
+			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggrFile=%q: %s", sasFile, err)
+		}
+		rwctx.sas = sas
+		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
+	}
+
+	return rwctx
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
@@ -501,6 +529,8 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.fq.UnblockAllReaders()
 	rwctx.c.MustStop()
 	rwctx.c = nil
+	rwctx.sas.MustStop()
+	rwctx.sas = nil
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
 
@@ -509,6 +539,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 }
 
 func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
+	// Apply relabeling
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
 	rcs := allRelabelConfigs.Load().(*relabelConfigs)
@@ -526,11 +557,17 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		rowsCountAfterRelabel := getRowsCount(tss)
 		rwctx.rowsDroppedByRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 	}
-	pss := rwctx.pss
-	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
 	rowsCount := getRowsCount(tss)
 	rwctx.rowsPushedAfterRelabel.Add(rowsCount)
-	pss[idx].Push(tss)
+
+	// Apply stream aggregation if any
+	rwctx.sas.Push(tss)
+	if rwctx.sas == nil || rwctx.streamAggrKeepInput {
+		// Push samples to the remote storage
+		rwctx.pushInternal(tss)
+	}
+
+	// Return back relabeling contexts to the pool
 	if rctx != nil {
 		*v = prompbmarshal.ResetTimeSeries(tss)
 		tssRelabelPool.Put(v)
@@ -538,6 +575,12 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 	}
 }
 
+func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
+	pss := rwctx.pss
+	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
+	pss[idx].Push(tss)
+}
+
 var tssRelabelPool = &sync.Pool{
 	New: func() interface{} {
 		a := []prompbmarshal.TimeSeries{}

app/vmagent/vmimport/request_handler.go

@@ -13,7 +13,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -31,20 +30,16 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-		return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
-			return insertRows(at, rows, extraLabels)
-		})
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+		return insertRows(at, rows, extraLabels)
 	})
 }
 
 // InsertHandlerForReader processes metrics from given reader
 func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, isGzipped, func(rows []parser.Row) error {
-			return insertRows(nil, rows, nil)
-		})
+	return parser.ParseStream(r, isGzipped, func(rows []parser.Row) error {
+		return insertRows(nil, rows, nil)
 	})
 }
 

app/vmalert/README.md

@@ -69,16 +69,17 @@ Then configure `vmalert` accordingly:
     -external.label=replica=a                # Multiple external labels may be set
 ```
 
-Note there's a separate `remoteWrite.url` to allow writing results of
+Note there's a separate `-remoteWrite.url` command-line flag to allow writing results of
 alerting/recording rules into a different storage than the initial data that's
 queried. This allows using `vmalert` to aggregate data from a short-term,
 high-frequency, high-cardinality storage into a long-term storage with
 decreased cardinality and a bigger interval between samples.
+See also [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html).
 
 See the full list of configuration flags in [configuration](#configuration) section.
 
 If you run multiple `vmalert` services for the same datastore or AlertManager - do not forget
-to specify different `external.label` flags in order to define which `vmalert` generated rules or alerts.
+to specify different `-external.label` command-line flags in order to define which `vmalert` generated rules or alerts.
 
 Configuration for [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)
 and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) rules is very
@@ -191,6 +192,11 @@ expr: <string>
 # Is applicable to alerting rules only.
 [ debug: <bool> | default = false ]
 
+# Defines the number of rule's updates entries stored in memory
+# and available for view on rule's Details page.
+# Overrides `rule.updateEntriesLimit` value for this specific rule.
+[ update_entries_limit: <integer> | default 0 ]
+
 # Labels to add or overwrite for each alert.
 labels:
   [ <labelname>: <tmpl_string> ]
@@ -319,6 +325,12 @@ expr: <string>
 # Labels to add or overwrite before storing the result.
 labels:
   [ <labelname>: <labelvalue> ]
+
+
+# Defines the number of rule's updates entries stored in memory
+# and available for view on rule's Details page.
+# Overrides `rule.updateEntriesLimit` value for this specific rule.
+[ update_entries_limit: <integer> | default 0 ]
 ```
 
 For recording rules to work `-remoteWrite.url` must be specified.
@@ -469,7 +481,8 @@ Alertmanager will automatically deduplicate alerts with identical labels, so ens
 all `vmalert`s are having the same config.
 
 Don't forget to configure [cluster mode](https://prometheus.io/docs/alerting/latest/alertmanager/)
-for Alertmanagers for better reliability.
+for Alertmanagers for better reliability. List all Alertmanager URLs in vmalert's `-notifier.url` 
+to ensure [high availability](https://github.com/prometheus/alertmanager#high-availability).
 
 This example uses single-node VM server for the sake of simplicity.
 Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed.
@@ -502,8 +515,8 @@ groups:
       expr: avg_over_time(http_requests[5m])
 ```
 
-Ability of `vmalert` to be configured with different `datasource.url` and `remoteWrite.url` allows
-reading data from one data source and backfilling results to another. This helps to build a system
+Ability of `vmalert` to be configured with different `-datasource.url` and `-remoteWrite.url` command-line flags
+allows reading data from one data source and backfilling results to another. This helps to build a system
 for aggregating and downsampling the data.
 
 The following example shows how to build a topology where `vmalert` will process data from one cluster
@@ -527,7 +540,7 @@ Please note, [replay](#rules-backfilling) feature may be used for transforming h
 
 Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only recording rules are used.
 
-See also [downsampling docs](https://docs.victoriametrics.com/#downsampling).
+See also [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) and [downsampling](https://docs.victoriametrics.com/#downsampling).
 
 #### Multiple remote writes
 
@@ -678,7 +691,7 @@ The default list of alerting rules for these metric can be found [here](https://
 We recommend setting up regular scraping of this page either through `vmagent` or by Prometheus so that the exported
 metrics may be analyzed later.
 
-Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950) for `vmalert` overview. 
+Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/14950-victoriametrics-vmalert/) for `vmalert` overview. 
 Graphs on this dashboard contain useful hints - hover the `i` icon in the top left corner of each graph in order to read it.
 If you have suggestions for improvements or have found a bug - please open an issue on github or add
 a review to the dashboard.
@@ -695,7 +708,7 @@ may get empty response from datasource and produce empty recording rules or rese
 
 <img alt="vmalert evaluation when data is delayed" src="vmalert_ts_data_delay.gif">
 
-By default recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
+By default, recently written samples to VictoriaMetrics aren't visible for queries for up to 30s.
 This behavior is controlled by `-search.latencyOffset` command-line flag and the `latency_offset` query ag at `vmselect`.
 Usually, this results into a 30s shift for recording rules results.
 Note that too small value passed to `-search.latencyOffset` or to `latency_offest` query arg may lead to incomplete query results.
@@ -721,8 +734,9 @@ If `-remoteWrite.url` command-line flag is configured, vmalert will persist aler
 [vmui](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmui) or Grafana to track how alerts state
 changed in time.
 
-vmalert also stores last N state updates for each rule. To check updates, click on `Details` link next to rule's name
-on `/vmalert/groups` page and check the `Last updates` section:
+vmalert stores last `-rule.updateEntriesLimit` (or `update_entries_limit` [per-rule config](https://docs.victoriametrics.com/vmalert.html#alerting-rules)) 
+state updates for each rule. To check updates, click on `Details` link next to rule's name on `/vmalert/groups` page 
+and check the `Last updates` section:
 
 <img alt="vmalert state" src="vmalert_state.png">
 
@@ -731,7 +745,7 @@ HTTP request sent by vmalert to the `-datasource.url` during evaluation. If spec
 no samples returned and curl command returns data - then it is very likely there was no data in datasource on the
 moment when rule was evaluated.
 
-vmalert also alows configuring more detailed logging for specific rule. Just set `debug: true` in rule's configuration
+vmalert allows configuring more detailed logging for specific alerting rule. Just set `debug: true` in rule's configuration
 and vmalert will start printing additional log messages:
 ```terminal
 2022-09-15T13:35:41.155Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:41+02:00: query returned 0 samples (elapsed: 5.896041ms)
@@ -890,6 +904,8 @@ The shortlist of configuration flags is the following:
      Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
      Format for logs. Possible values: default, json (default "default")
+  -loggerJSONFields string
+     Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg
   -loggerLevel string
      Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string
@@ -955,7 +971,7 @@ The shortlist of configuration flags is the following:
      Optional TLS server name to use for connections to -notifier.url. By default the server name from -notifier.url is used
      Supports an array of values separated by comma or specified via multiple flags.
   -notifier.url array
-     Prometheus alertmanager URL, e.g. http://127.0.0.1:9093
+     Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.
      Supports an array of values separated by comma or specified via multiple flags.
   -pprofAuthKey string
      Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings
@@ -1102,6 +1118,8 @@ The shortlist of configuration flags is the following:
       -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder,
      absolute path to all .tpl files in root.
      Supports an array of values separated by comma or specified via multiple flags.
+  -rule.updateEntriesLimit int
+     Defines the max number of rule's state updates stored in-memory. Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overriden per rule via update_entries_limit param. (default 20)
   -rule.validateExpressions
      Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
@@ -1185,6 +1203,8 @@ dns_sd_configs:
 ```
 
 The list of configured or discovered Notifiers can be explored via [UI](#Web).
+If Alertmanager runs in cluster mode then all its URLs needs to be available during discovery
+to ensure [high availability](https://github.com/prometheus/alertmanager#high-availability).
 
 The configuration file [specification](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go)
 is the following:

app/vmalert/alerting.go

@@ -74,10 +74,15 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			Debug:              cfg.Debug,
 		}),
 		alerts:  make(map[uint64]*notifier.Alert),
-		state:   newRuleState(),
 		metrics: &alertingRuleMetrics{},
 	}
 
+	if cfg.UpdateEntriesLimit != nil {
+		ar.state = newRuleState(*cfg.UpdateEntriesLimit)
+	} else {
+		ar.state = newRuleState(*ruleUpdateEntriesLimit)
+	}
+
 	labels := fmt.Sprintf(`alertname=%q, group=%q, id="%d"`, ar.Name, group.Name, ar.ID())
 	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
 		func() float64 {
@@ -491,6 +496,7 @@ func (ar *AlertingRule) ToAPI() APIRule {
 		State:          "inactive",
 		Alerts:         ar.AlertsToAPI(),
 		LastSamples:    lastState.samples,
+		MaxUpdates:     ar.state.size(),
 		Updates:        ar.state.getAll(),
 
 		// encode as strings to avoid rounding in JSON

app/vmalert/alerting_test.go

@@ -709,7 +709,6 @@ func TestAlertingRule_Template(t *testing.T) {
 					"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
-				state:  newRuleState(),
 			},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 1, "instance", "foo"),
@@ -749,7 +748,6 @@ func TestAlertingRule_Template(t *testing.T) {
 					"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
-				state:  newRuleState(),
 			},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
@@ -789,7 +787,6 @@ func TestAlertingRule_Template(t *testing.T) {
 					"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
 				},
 				alerts: make(map[uint64]*notifier.Alert),
-				state:  newRuleState(),
 			},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 1,
@@ -820,6 +817,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			fq := &fakeQuerier{}
 			tc.rule.GroupID = fakeGroup.ID()
 			tc.rule.q = fq
+			tc.rule.state = newRuleState(10)
 			fq.add(tc.metrics...)
 			if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
 				t.Fatalf("unexpected err: %s", err)
@@ -936,6 +934,6 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
 		For:          waitFor,
 		EvalInterval: waitFor,
 		alerts:       make(map[uint64]*notifier.Alert),
-		state:        newRuleState(),
+		state:        newRuleState(10),
 	}
 }

app/vmalert/config/config.go

@@ -114,6 +114,9 @@ type Rule struct {
 	Labels      map[string]string   `yaml:"labels,omitempty"`
 	Annotations map[string]string   `yaml:"annotations,omitempty"`
 	Debug       bool                `yaml:"debug,omitempty"`
+	// UpdateEntriesLimit defines max number of rule's state updates stored in memory.
+	// Overrides `-rule.updateEntriesLimit`.
+	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`

app/vmalert/config/config_test.go

@@ -550,6 +550,20 @@ rules:
   - alert: foo
     expr: sum by(job) (up == 1)
     debug: true
+`)
+	})
+	t.Run("`update_entries_limit` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+    update_entries_limit: 33
 `)
 	})
 }

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -12,6 +12,7 @@ groups:
         expr: vm_tcplistener_conns > 0
         for: 3m
         debug: true
+        update_entries_limit: 40
         annotations:
           labels: "Available labels: {{ $labels }}"
           summary: Too high connection number for {{ $labels.instance }}
@@ -20,6 +21,7 @@ groups:
             {{ end }}
           description: "It is {{ $value }} connections for {{$labels.instance}}"
       - alert: ExampleAlertAlwaysFiring
+        update_entries_limit: -1
         expr: sum by(job)
           (up == 1)
         labels:

app/vmalert/config/testdata/rules/rules3-good.rules

@@ -7,6 +7,7 @@ groups:
       - alert: Conns
         expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500)
         for: 3m
+
         annotations:
           summary: Too high connection number for {{$labels.instance}}
           description: "It is {{ $value }} connections for {{$labels.instance}}"

app/vmalert/group_test.go

@@ -460,7 +460,7 @@ func TestFaultyRW(t *testing.T) {
 
 	r := &RecordingRule{
 		Name:  "test",
-		state: newRuleState(),
+		state: newRuleState(10),
 		q:     fq,
 	}
 

app/vmalert/main.go

@@ -56,7 +56,9 @@ absolute path to all .tpl files in root.`)
 	validateExpressions = flag.Bool("rule.validateExpressions", true, "Whether to validate rules expressions via MetricsQL engine")
 	maxResolveDuration  = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
 		"which is by default equal to 3 evaluation intervals of the parent group.")
-	resendDelay = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
+	resendDelay            = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier")
+	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
+		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overriden per rule via update_entries_limit param.")
 
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+

app/vmalert/notifier/init.go

@@ -17,7 +17,8 @@ var (
 	configPath                    = flag.String("notifier.config", "", "Path to configuration file for notifiers")
 	suppressDuplicateTargetErrors = flag.Bool("notifier.suppressDuplicateTargetErrors", false, "Whether to suppress 'duplicate target' errors during discovery")
 
-	addrs = flagutil.NewArrayString("notifier.url", "Prometheus alertmanager URL, e.g. http://127.0.0.1:9093")
+	addrs = flagutil.NewArrayString("notifier.url", "Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. "+
+		"List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.")
 
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")

app/vmalert/recording.go

@@ -58,7 +58,6 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		Labels:  cfg.Labels,
 		GroupID: group.ID(),
 		metrics: &recordingRuleMetrics{},
-		state:   newRuleState(),
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
@@ -67,6 +66,12 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		}),
 	}
 
+	if cfg.UpdateEntriesLimit != nil {
+		rr.state = newRuleState(*cfg.UpdateEntriesLimit)
+	} else {
+		rr.state = newRuleState(*ruleUpdateEntriesLimit)
+	}
+
 	labels := fmt.Sprintf(`recording=%q, group=%q, id="%d"`, rr.Name, group.Name, rr.ID())
 	rr.metrics.errors = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_error{%s}`, labels),
 		func() float64 {
@@ -212,6 +217,7 @@ func (rr *RecordingRule) ToAPI() APIRule {
 		EvaluationTime: lastState.duration.Seconds(),
 		Health:         "ok",
 		LastSamples:    lastState.samples,
+		MaxUpdates:     rr.state.size(),
 		Updates:        rr.state.getAll(),
 
 		// encode as strings to avoid rounding

app/vmalert/recording_test.go

@@ -19,7 +19,7 @@ func TestRecordingRule_Exec(t *testing.T) {
 		expTS   []prompbmarshal.TimeSeries
 	}{
 		{
-			&RecordingRule{Name: "foo", state: newRuleState()},
+			&RecordingRule{Name: "foo"},
 			[]datasource.Metric{metricWithValueAndLabels(t, 10,
 				"__name__", "bar",
 			)},
@@ -30,7 +30,7 @@ func TestRecordingRule_Exec(t *testing.T) {
 			},
 		},
 		{
-			&RecordingRule{Name: "foobarbaz", state: newRuleState()},
+			&RecordingRule{Name: "foobarbaz"},
 			[]datasource.Metric{
 				metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
 				metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
@@ -53,8 +53,7 @@ func TestRecordingRule_Exec(t *testing.T) {
 		},
 		{
 			&RecordingRule{
-				Name:  "job:foo",
-				state: newRuleState(),
+				Name: "job:foo",
 				Labels: map[string]string{
 					"source": "test",
 				}},
@@ -80,6 +79,7 @@ func TestRecordingRule_Exec(t *testing.T) {
 			fq := &fakeQuerier{}
 			fq.add(tc.metrics...)
 			tc.rule.q = fq
+			tc.rule.state = newRuleState(10)
 			tss, err := tc.rule.Exec(context.TODO(), time.Now(), 0)
 			if err != nil {
 				t.Fatalf("unexpected Exec err: %s", err)
@@ -198,7 +198,7 @@ func TestRecordingRuleLimit(t *testing.T) {
 		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
 		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
 	}
-	rule := &RecordingRule{Name: "job:foo", state: newRuleState(), Labels: map[string]string{
+	rule := &RecordingRule{Name: "job:foo", state: newRuleState(10), Labels: map[string]string{
 		"source": "test_limit",
 	}}
 	var err error
@@ -216,7 +216,7 @@ func TestRecordingRuleLimit(t *testing.T) {
 func TestRecordingRule_ExecNegative(t *testing.T) {
 	rr := &RecordingRule{
 		Name:  "job:foo",
-		state: newRuleState(),
+		state: newRuleState(10),
 		Labels: map[string]string{
 			"job": "test",
 		},

app/vmalert/rule.go

@@ -37,6 +37,8 @@ type ruleState struct {
 	sync.RWMutex
 	entries []ruleStateEntry
 	cur     int
+	// disabled defines whether ruleState tracks ruleStateEntry
+	disabled bool
 }
 
 type ruleStateEntry struct {
@@ -57,21 +59,36 @@ type ruleStateEntry struct {
 	curl string
 }
 
-const defaultStateEntriesLimit = 20
-
-func newRuleState() *ruleState {
+func newRuleState(size int) *ruleState {
+	if size < 1 {
+		return &ruleState{disabled: true}
+	}
 	return &ruleState{
-		entries: make([]ruleStateEntry, defaultStateEntriesLimit),
+		entries: make([]ruleStateEntry, size),
 	}
 }
 
 func (s *ruleState) getLast() ruleStateEntry {
+	if s.disabled {
+		return ruleStateEntry{}
+	}
+
 	s.RLock()
 	defer s.RUnlock()
 	return s.entries[s.cur]
 }
 
+func (s *ruleState) size() int {
+	s.RLock()
+	defer s.RUnlock()
+	return len(s.entries)
+}
+
 func (s *ruleState) getAll() []ruleStateEntry {
+	if s.disabled {
+		return nil
+	}
+
 	entries := make([]ruleStateEntry, 0)
 
 	s.RLock()
@@ -94,6 +111,10 @@ func (s *ruleState) getAll() []ruleStateEntry {
 }
 
 func (s *ruleState) add(e ruleStateEntry) {
+	if s.disabled {
+		return
+	}
+
 	s.Lock()
 	defer s.Unlock()
 

app/vmalert/rule_test.go

@@ -6,8 +6,27 @@ import (
 	"time"
 )
 
+func TestRule_stateDisabled(t *testing.T) {
+	state := newRuleState(-1)
+	e := state.getLast()
+	if !e.at.IsZero() {
+		t.Fatalf("expected entry to be zero")
+	}
+
+	state.add(ruleStateEntry{at: time.Now()})
+	if !e.at.IsZero() {
+		t.Fatalf("expected entry to be zero")
+	}
+
+	if len(state.getAll()) != 0 {
+		t.Fatalf("expected for state to have %d entries; got %d",
+			0, len(state.getAll()),
+		)
+	}
+}
 func TestRule_state(t *testing.T) {
-	state := newRuleState()
+	stateEntriesN := 20
+	state := newRuleState(stateEntriesN)
 	e := state.getLast()
 	if !e.at.IsZero() {
 		t.Fatalf("expected entry to be zero")
@@ -39,7 +58,7 @@ func TestRule_state(t *testing.T) {
 	}
 
 	var last time.Time
-	for i := 0; i < defaultStateEntriesLimit*2; i++ {
+	for i := 0; i < stateEntriesN*2; i++ {
 		last = time.Now()
 		state.add(ruleStateEntry{at: last})
 	}
@@ -50,9 +69,9 @@ func TestRule_state(t *testing.T) {
 			e.at, last)
 	}
 
-	if len(state.getAll()) != defaultStateEntriesLimit {
+	if len(state.getAll()) != stateEntriesN {
 		t.Fatalf("expected for state to have %d entries only; got %d",
-			defaultStateEntriesLimit, len(state.getAll()),
+			stateEntriesN, len(state.getAll()),
 		)
 	}
 }
@@ -61,7 +80,7 @@ func TestRule_state(t *testing.T) {
 // execution of state updates.
 // Should be executed with -race flag
 func TestRule_stateConcurrent(t *testing.T) {
-	state := newRuleState()
+	state := newRuleState(20)
 
 	const workers = 50
 	const iterations = 100

app/vmalert/templates/template.go

@@ -27,11 +27,11 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	textTpl "text/template"
 	"time"
 
-	textTpl "text/template"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -350,15 +350,7 @@ func templateFuncs() textTpl.FuncMap {
 			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
 				return fmt.Sprintf("%.4g", v), nil
 			}
-			prefix := ""
-			for _, p := range []string{"ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"} {
-				if math.Abs(v) < 1024 {
-					break
-				}
-				prefix = p
-				v /= 1024
-			}
-			return fmt.Sprintf("%.4g%s", v, prefix), nil
+			return formatutil.HumanizeBytes(v), nil
 		},
 
 		// humanizeDuration converts given seconds to a human-readable duration

app/vmalert/templates/template_test.go

@@ -1,6 +1,7 @@
 package templates
 
 import (
+	"math"
 	"strings"
 	"testing"
 	textTpl "text/template"
@@ -50,6 +51,31 @@ func TestTemplateFuncs(t *testing.T) {
 	if !ok {
 		t.Fatalf("unexpected mismatch")
 	}
+
+	formatting := func(funcName string, p interface{}, resultExpected string) {
+		t.Helper()
+		v := funcs[funcName]
+		fLocal := v.(func(s interface{}) (string, error))
+		result, err := fLocal(p)
+		if err != nil {
+			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
+		}
+	}
+	formatting("humanize1024", float64(0), "0")
+	formatting("humanize1024", math.Inf(0), "+Inf")
+	formatting("humanize1024", math.NaN(), "NaN")
+	formatting("humanize1024", float64(127087), "124.1ki")
+	formatting("humanize1024", float64(130137088), "124.1Mi")
+	formatting("humanize1024", float64(133260378112), "124.1Gi")
+	formatting("humanize1024", float64(136458627186688), "124.1Ti")
+	formatting("humanize1024", float64(139733634239168512), "124.1Pi")
+	formatting("humanize1024", float64(143087241460908556288), "124.1Ei")
+	formatting("humanize1024", float64(146521335255970361638912), "124.1Zi")
+	formatting("humanize1024", float64(150037847302113650318245888), "124.1Yi")
+	formatting("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
 }
 
 func mkTemplate(current, replacement interface{}) textTemplate {

app/vmalert/web.qtpl

@@ -440,7 +440,7 @@
     </div>
 
     <br>
-    <div class="display-6 pb-3">Last {%d len(rule.Updates) %} updates</span>:</div>
+    <div class="display-6 pb-3">Last {%d len(rule.Updates) %}/{%d rule.MaxUpdates %} updates</span>:</div>
         <table class="table table-striped table-hover table-sm">
             <thead>
                 <tr>

app/vmalert/web.qtpl.go

@@ -1345,6 +1345,10 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
     <div class="display-6 pb-3">Last `)
 //line app/vmalert/web.qtpl:443
 	qw422016.N().D(len(rule.Updates))
+//line app/vmalert/web.qtpl:443
+	qw422016.N().S(`/`)
+//line app/vmalert/web.qtpl:443
+	qw422016.N().D(rule.MaxUpdates)
 //line app/vmalert/web.qtpl:443
 	qw422016.N().S(` updates</span>:</div>
         <table class="table table-striped table-hover table-sm">

app/vmalert/web_test.go

@@ -17,7 +17,7 @@ func TestHandler(t *testing.T) {
 		alerts: map[uint64]*notifier.Alert{
 			0: {State: notifier.StateFiring},
 		},
-		state: newRuleState(),
+		state: newRuleState(10),
 	}
 	g := &Group{
 		Name:  "group",

app/vmalert/web_types.go

@@ -121,6 +121,8 @@ type APIRule struct {
 	// GroupID is an unique Group's ID
 	GroupID string `json:"group_id"`
 
+	// MaxUpdates is the max number of recorded ruleStateEntry objects
+	MaxUpdates int `json:"max_updates_entries"`
 	// Updates contains the ordered list of recorded ruleStateEntry objects
 	Updates []ruleStateEntry `json:"updates"`
 }

app/vmauth/README.md

@@ -270,6 +270,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
      Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
      Format for logs. Possible values: default, json (default "default")
+  -loggerJSONFields string
+     Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg
   -loggerLevel string
      Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string

app/vmauth/main.go

@@ -60,9 +60,7 @@ func main() {
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
-		authKey := r.FormValue("authKey")
-		if authKey != *reloadAuthKey {
-			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -reloadAuthKey command line flag", authKey)
+		if !httpserver.CheckAuthFlag(w, r, *reloadAuthKey, "reloadAuthKey") {
 			return true
 		}
 		configReloadRequests.Inc()

app/vmbackup/README.md

@@ -225,6 +225,8 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
      Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
      Format for logs. Possible values: default, json (default "default")
+  -loggerJSONFields string
+     Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg
   -loggerLevel string
      Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string

app/vmbackup/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -41,25 +42,36 @@ func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
+	flagutil.RegisterSecretFlag("snapshot.createURL")
+	flagutil.RegisterSecretFlag("snapshot.deleteURL")
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
 	pushmetrics.Init()
 
 	if len(*snapshotCreateURL) > 0 {
+		// create net/url object
+		createUrl, err := url.Parse(*snapshotCreateURL)
+		if err != nil {
+			logger.Fatalf("cannot parse snapshotCreateURL: %s", err)
+		}
 		if len(*snapshotName) > 0 {
 			logger.Fatalf("-snapshotName shouldn't be set if -snapshot.createURL is set, since snapshots are created automatically in this case")
 		}
-		logger.Infof("Snapshot create url %s", *snapshotCreateURL)
+		logger.Infof("Snapshot create url %s", createUrl.Redacted())
 		if len(*snapshotDeleteURL) <= 0 {
 			err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
 			if err != nil {
 				logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err)
 			}
 		}
-		logger.Infof("Snapshot delete url %s", *snapshotDeleteURL)
+		deleteUrl, err := url.Parse(*snapshotCreateURL)
+		if err != nil {
+			logger.Fatalf("cannot parse snapshotDeleteURL: %s", err)
+		}
+		logger.Infof("Snapshot delete url %s", deleteUrl.Redacted())
 
-		name, err := snapshot.Create(*snapshotCreateURL)
+		name, err := snapshot.Create(createUrl.String())
 		if err != nil {
 			logger.Fatalf("cannot create snapshot: %s", err)
 		}
@@ -69,7 +81,7 @@ func main() {
 		}
 
 		defer func() {
-			err := snapshot.Delete(*snapshotDeleteURL, name)
+			err := snapshot.Delete(deleteUrl.String(), name)
 			if err != nil {
 				logger.Fatalf("cannot delete snapshot: %s", err)
 			}
@@ -118,7 +130,7 @@ func main() {
 
 func usage() {
 	const s = `
-vmbackup performs backups for VictoriaMetrics data from instant snapshots to gcs, s3
+vmbackup performs backups for VictoriaMetrics data from instant snapshots to gcs, s3, azblob
 or local filesystem. Backed up data can be restored with vmrestore.
 
 See the docs at https://docs.victoriametrics.com/vmbackup.html .

app/vmbackupmanager/README.md

@@ -67,7 +67,7 @@ credentials.json
   "project_id": "<project>",
   "private_key_id": "",
   "private_key": "-----BEGIN PRIVATE KEY-----\-----END PRIVATE KEY-----\n",
-  "client_email": “test@<project>.iam.gserviceaccount.com",
+  "client_email": "test@<project>.iam.gserviceaccount.com",
   "client_id": "",
   "auth_uri": "https://accounts.google.com/o/oauth2/auth",
   "token_uri": "https://oauth2.googleapis.com/token",
@@ -158,7 +158,7 @@ The result on the GCS bucket. We see only 3 daily backups:
 * GET `/api/v1/backups` - returns list of backups in remote storage.
   Example output:
   ```json
-  ["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
+  [{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
   ```
 
 * POST `/api/v1/restore` - saves backup name to restore when [performing restore](#restore-commands).
@@ -211,7 +211,7 @@ It can be changed by using flag:
 `vmbackupmanager backup list` lists backups in remote storage:
 ```console
 $ ./vmbackupmanager backup list
-["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
+[{"name":"daily/2022-11-30","size_bytes":26664689,"size":"25.429Mi"},{"name":"daily/2022-12-01","size_bytes":40160965,"size":"38.300Mi"},{"name":"hourly/2022-11-30:12","size_bytes":5846529,"size":"5.576Mi"},{"name":"hourly/2022-11-30:13","size_bytes":17651847,"size":"16.834Mi"},{"name":"hourly/2022-11-30:13:22","size_bytes":8797831,"size":"8.390Mi"},{"name":"hourly/2022-11-30:14","size_bytes":10680454,"size":"10.186Mi"}]
 ```
 
 ### Restore commands
@@ -270,7 +270,15 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
 
 ### How to restore in Kubernetes
 
-1. Enter container running `vmbackupmanager`
+1. Ensure there is an init container with `vmbackupmanager restore` in `vmstorage` or `vmsingle` pod.
+   For [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) deployments it is required to add:
+   ```yaml
+   vmbackup:
+     restore:
+       onStart: "true"
+   ```
+   See operator `VMStorage` schema [here](https://docs.victoriametrics.com/operator/api.html#vmstorage) and `VMSingle` [here](https://docs.victoriametrics.com/operator/api.html#vmsinglespec).
+2. Enter container running `vmbackupmanager`
 2. Use `vmbackupmanager backup list` to get list of available backups:
   ```console
   $ /vmbackupmanager-prod backup list
@@ -287,6 +295,43 @@ If restore mark doesn't exist at `storageDataPath`(restore wasn't requested) `vm
     ```
 4. Restart pod
 
+#### Restore cluster into another cluster
+
+These steps are assuming that [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/VictoriaMetrics-Operator.html) is used to manage `VMCluster`.
+Clusters here are referred to as `source` and `destination`.
+
+1. Create a new cluster with access to *source* cluster `vmbackupmanager` storage and same number of storage nodes.
+   Add the following section in order to enable restore on start (operator `VMStorage` schema can be found [here](https://docs.victoriametrics.com/operator/api.html#vmstorage):
+   ```yaml
+   vmbackup:
+     restore:
+       onStart: "true"
+   ```
+   Note: it is safe to leave this section in the cluster configuration, since it will be ignored if restore mark doesn't exist.
+   > Important! Use different `-dst` for *destination* cluster to avoid overwriting backup data of the *source* cluster.
+2. Enter container running `vmbackupmanager` in *source* cluster
+2. Use `vmbackupmanager backup list` to get list of available backups:
+  ```console
+  $ /vmbackupmanager-prod backup list
+  ["daily/2022-10-06","daily/2022-10-10","hourly/2022-10-04:13","hourly/2022-10-06:12","hourly/2022-10-06:13","hourly/2022-10-10:14","hourly/2022-10-10:16","monthly/2022-10","weekly/2022-40","weekly/2022-41"]
+  ```
+3. Use `vmbackupmanager restore create` to create restore mark at each pod of the *destination* cluster.
+   Each pod in *destination* cluster should be restored from backup of respective pod in *source* cluster.
+   For example: `vmstorage-source-0` in *source* cluster should be restored from `vmstorage-destination-0` in *destination* cluster.
+  ```console
+  $ /vmbackupmanager-prod restore create s3://source_cluster/vmstorage-source-0/daily/2022-10-06
+  ```
+
+## Monitoring
+
+`vmbackupmanager` exports various metrics in Prometheus exposition format at `http://vmbackupmanager:8300/metrics` page. It is recommended setting up regular scraping of this page
+either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
+
+Use the official [Grafana dashboard](https://grafana.com/grafana/dashboards/17798-victoriametrics-backupmanager/) for `vmbackupmanager` overview.
+Graphs on this dashboard contain useful hints - hover the `i` icon in the top left corner of each graph in order to read it.
+If you have suggestions for improvements or have found a bug - please open an issue on github or add
+a review to the dashboard.
+
 ## Configuration
 
 ### Flags
@@ -372,6 +417,8 @@ command-line flags:
      Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
      Format for logs. Possible values: default, json (default "default")
+  -loggerJSONFields string
+     Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg
   -loggerLevel string
      Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string

app/vmctl/flags.go

@@ -353,7 +353,7 @@ var (
 		},
 		&cli.StringFlag{
 			Name:  vmNativeStepInterval,
-			Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are '%s','%s','%s'.", vmNativeFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour),
+			Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are '%s','%s','%s','%s'.", vmNativeFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
 		},
 		&cli.StringFlag{
 			Name: vmNativeSrcAddr,
@@ -410,19 +410,20 @@ var (
 )
 
 const (
-	remoteRead                 = "remote-read"
-	remoteReadUseStream        = "remote-read-use-stream"
-	remoteReadConcurrency      = "remote-read-concurrency"
-	remoteReadFilterTimeStart  = "remote-read-filter-time-start"
-	remoteReadFilterTimeEnd    = "remote-read-filter-time-end"
-	remoteReadFilterLabel      = "remote-read-filter-label"
-	remoteReadFilterLabelValue = "remote-read-filter-label-value"
-	remoteReadStepInterval     = "remote-read-step-interval"
-	remoteReadSrcAddr          = "remote-read-src-addr"
-	remoteReadUser             = "remote-read-user"
-	remoteReadPassword         = "remote-read-password"
-	remoteReadHTTPTimeout      = "remote-read-http-timeout"
-	remoteReadHeaders          = "remote-read-headers"
+	remoteRead                   = "remote-read"
+	remoteReadUseStream          = "remote-read-use-stream"
+	remoteReadConcurrency        = "remote-read-concurrency"
+	remoteReadFilterTimeStart    = "remote-read-filter-time-start"
+	remoteReadFilterTimeEnd      = "remote-read-filter-time-end"
+	remoteReadFilterLabel        = "remote-read-filter-label"
+	remoteReadFilterLabelValue   = "remote-read-filter-label-value"
+	remoteReadStepInterval       = "remote-read-step-interval"
+	remoteReadSrcAddr            = "remote-read-src-addr"
+	remoteReadUser               = "remote-read-user"
+	remoteReadPassword           = "remote-read-password"
+	remoteReadHTTPTimeout        = "remote-read-http-timeout"
+	remoteReadHeaders            = "remote-read-headers"
+	remoteReadInsecureSkipVerify = "remote-read-insecure-skip-verify"
 )
 
 var (
@@ -493,6 +494,11 @@ var (
 				"For example, --remote-read-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding remote source storage. \n" +
 				"Multiple headers must be delimited by '^^': --remote-read-headers='header1:value1^^header2:value2'",
 		},
+		&cli.BoolFlag{
+			Name:  remoteReadInsecureSkipVerify,
+			Usage: "Whether to skip TLS certificate verification when connecting to the remote read address",
+			Value: false,
+		},
 	}
 )
 

app/vmctl/main.go

@@ -121,14 +121,15 @@ func main() {
 				Flags: mergeFlags(globalFlags, remoteReadFlags, vmFlags),
 				Action: func(c *cli.Context) error {
 					rr, err := remoteread.NewClient(remoteread.Config{
-						Addr:       c.String(remoteReadSrcAddr),
-						Username:   c.String(remoteReadUser),
-						Password:   c.String(remoteReadPassword),
-						Timeout:    c.Duration(remoteReadHTTPTimeout),
-						UseStream:  c.Bool(remoteReadUseStream),
-						Headers:    c.String(remoteReadHeaders),
-						LabelName:  c.String(remoteReadFilterLabel),
-						LabelValue: c.String(remoteReadFilterLabelValue),
+						Addr:               c.String(remoteReadSrcAddr),
+						Username:           c.String(remoteReadUser),
+						Password:           c.String(remoteReadPassword),
+						Timeout:            c.Duration(remoteReadHTTPTimeout),
+						UseStream:          c.Bool(remoteReadUseStream),
+						Headers:            c.String(remoteReadHeaders),
+						LabelName:          c.String(remoteReadFilterLabel),
+						LabelValue:         c.String(remoteReadFilterLabelValue),
+						InsecureSkipVerify: c.Bool(remoteReadInsecureSkipVerify),
 					})
 					if err != nil {
 						return fmt.Errorf("error create remote read client: %s", err)

app/vmctl/remote_read_test.go

@@ -0,0 +1,319 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	"github.com/prometheus/prometheus/prompb"
+)
+
+func TestRemoteRead(t *testing.T) {
+
+	var testCases = []struct {
+		name             string
+		remoteReadConfig remoteread.Config
+		vmCfg            vm.Config
+		start            string
+		end              string
+		numOfSamples     int64
+		numOfSeries      int64
+		rrp              remoteReadProcessor
+		chunk            string
+		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
+		expectedSeries   []vm.TimeSeries
+	}{
+		{
+			name:             "step minute on minute time range",
+			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
+			vmCfg:            vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true},
+			start:            "2022-11-26T11:23:05+02:00",
+			end:              "2022-11-26T11:24:05+02:00",
+			numOfSamples:     2,
+			numOfSeries:      3,
+			chunk:            stepper.StepMinute,
+			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
+			expectedSeries: []vm.TimeSeries{
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1669454585000, 1669454615000},
+					Values:     []float64{0, 0},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1669454585000, 1669454615000},
+					Values:     []float64{100, 100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1669454585000, 1669454615000},
+					Values:     []float64{200, 200},
+				},
+			},
+		},
+		{
+			name:             "step month on month time range",
+			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
+			vmCfg:            vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true},
+			start:            "2022-09-26T11:23:05+02:00",
+			end:              "2022-11-26T11:24:05+02:00",
+			numOfSamples:     2,
+			numOfSeries:      3,
+			chunk:            stepper.StepMonth,
+			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
+			expectedSeries: []vm.TimeSeries{
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{0},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{200},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{0},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{200}},
+			},
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			remoteReadServer := remote_read_integration.NewRemoteReadServer(t)
+			defer remoteReadServer.Close()
+			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
+			defer remoteWriteServer.Close()
+
+			tt.remoteReadConfig.Addr = remoteReadServer.URL()
+
+			rr, err := remoteread.NewClient(tt.remoteReadConfig)
+			if err != nil {
+				t.Fatalf("error create remote read client: %s", err)
+			}
+
+			start, err := time.Parse(time.RFC3339, tt.start)
+			if err != nil {
+				t.Fatalf("Error parse start time: %s", err)
+			}
+
+			end, err := time.Parse(time.RFC3339, tt.end)
+			if err != nil {
+				t.Fatalf("Error parse end time: %s", err)
+			}
+
+			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
+
+			remoteReadServer.SetRemoteReadSeries(rrs)
+			remoteWriteServer.ExpectedSeries(tt.expectedSeries)
+
+			tt.vmCfg.Addr = remoteWriteServer.URL()
+
+			importer, err := vm.NewImporter(tt.vmCfg)
+			if err != nil {
+				t.Fatalf("failed to create VM importer: %s", err)
+			}
+			defer importer.Close()
+
+			rmp := remoteReadProcessor{
+				src: rr,
+				dst: importer,
+				filter: remoteReadFilter{
+					timeStart: &start,
+					timeEnd:   &end,
+					chunk:     tt.chunk,
+				},
+				cc: 1,
+			}
+
+			ctx := context.Background()
+			err = rmp.run(ctx, true, false)
+			if err != nil {
+				t.Fatalf("failed to run remote read processor: %s", err)
+			}
+		})
+	}
+}
+
+func TestSteamRemoteRead(t *testing.T) {
+
+	var testCases = []struct {
+		name             string
+		remoteReadConfig remoteread.Config
+		vmCfg            vm.Config
+		start            string
+		end              string
+		numOfSamples     int64
+		numOfSeries      int64
+		rrp              remoteReadProcessor
+		chunk            string
+		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
+		expectedSeries   []vm.TimeSeries
+	}{
+		{
+			name:             "step minute on minute time range",
+			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
+			vmCfg:            vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true},
+			start:            "2022-11-26T11:23:05+02:00",
+			end:              "2022-11-26T11:24:05+02:00",
+			numOfSamples:     2,
+			numOfSeries:      3,
+			chunk:            stepper.StepMinute,
+			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
+			expectedSeries: []vm.TimeSeries{
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1669454585000, 1669454615000},
+					Values:     []float64{0, 0},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1669454585000, 1669454615000},
+					Values:     []float64{100, 100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1669454585000, 1669454615000},
+					Values:     []float64{200, 200},
+				},
+			},
+		},
+		{
+			name:             "step month on month time range",
+			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
+			vmCfg:            vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true},
+			start:            "2022-09-26T11:23:05+02:00",
+			end:              "2022-11-26T11:24:05+02:00",
+			numOfSamples:     2,
+			numOfSeries:      3,
+			chunk:            stepper.StepMonth,
+			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
+			expectedSeries: []vm.TimeSeries{
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{0},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1664184185000},
+					Values:     []float64{200},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{0},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{100},
+				},
+				{
+					Name:       "vm_metric_1",
+					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
+					Timestamps: []int64{1666819415000},
+					Values:     []float64{200}},
+			},
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			remoteReadServer := remote_read_integration.NewRemoteReadStreamServer(t)
+			defer remoteReadServer.Close()
+			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
+			defer remoteWriteServer.Close()
+
+			tt.remoteReadConfig.Addr = remoteReadServer.URL()
+
+			rr, err := remoteread.NewClient(tt.remoteReadConfig)
+			if err != nil {
+				t.Fatalf("error create remote read client: %s", err)
+			}
+
+			start, err := time.Parse(time.RFC3339, tt.start)
+			if err != nil {
+				t.Fatalf("Error parse start time: %s", err)
+			}
+
+			end, err := time.Parse(time.RFC3339, tt.end)
+			if err != nil {
+				t.Fatalf("Error parse end time: %s", err)
+			}
+
+			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
+
+			remoteReadServer.InitMockStorage(rrs)
+			remoteWriteServer.ExpectedSeries(tt.expectedSeries)
+
+			tt.vmCfg.Addr = remoteWriteServer.URL()
+
+			importer, err := vm.NewImporter(tt.vmCfg)
+			if err != nil {
+				t.Fatalf("failed to create VM importer: %s", err)
+			}
+			defer importer.Close()
+
+			rmp := remoteReadProcessor{
+				src: rr,
+				dst: importer,
+				filter: remoteReadFilter{
+					timeStart: &start,
+					timeEnd:   &end,
+					chunk:     tt.chunk,
+				},
+				cc: 1,
+			}
+
+			ctx := context.Background()
+			err = rmp.run(ctx, true, false)
+			if err != nil {
+				t.Fatalf("failed to run remote read processor: %s", err)
+			}
+		})
+	}
+}

app/vmctl/remoteread/remoteread.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/gogo/protobuf/proto"
@@ -60,6 +61,8 @@ type Config struct {
 	// LabelName, LabelValue stands for label=~value pair used for read requests.
 	// Is optional.
 	LabelName, LabelValue string
+	// TLSSkipVerify defines whether to skip TLS certificate verification when connecting to the remote read address.
+	InsecureSkipVerify bool
 }
 
 // Filter defines a list of filters applied to requested data
@@ -100,7 +103,7 @@ func NewClient(cfg Config) (*Client, error) {
 	c := &Client{
 		c: &http.Client{
 			Timeout:   cfg.Timeout,
-			Transport: http.DefaultTransport.(*http.Transport).Clone(),
+			Transport: utils.Transport(cfg.Addr, cfg.InsecureSkipVerify),
 		},
 		addr:      strings.TrimSuffix(cfg.Addr, "/"),
 		user:      cfg.Username,

app/vmctl/testdata/servers_integration_test/remote_read_server.go

@@ -0,0 +1,366 @@
+package remote_read_integration
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/prometheus/prometheus/storage/remote"
+)
+
+const (
+	maxBytesInFrame = 1024 * 1024
+)
+
+type RemoteReadServer struct {
+	server  *httptest.Server
+	series  []*prompb.TimeSeries
+	storage *MockStorage
+}
+
+// NewRemoteReadServer creates a remote read server. It exposes a single endpoint and responds with the
+// passed series based on the request to the read endpoint. It returns a server which should be closed after
+// being used.
+func NewRemoteReadServer(t *testing.T) *RemoteReadServer {
+	rrs := &RemoteReadServer{
+		series: make([]*prompb.TimeSeries, 0),
+	}
+	rrs.server = httptest.NewServer(rrs.getReadHandler(t))
+	return rrs
+}
+
+// Close closes the server.
+func (rrs *RemoteReadServer) Close() {
+	rrs.server.Close()
+}
+
+func (rrs *RemoteReadServer) URL() string {
+	return rrs.server.URL
+}
+
+func (rrs *RemoteReadServer) SetRemoteReadSeries(series []*prompb.TimeSeries) {
+	rrs.series = append(rrs.series, series...)
+}
+
+func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !validateReadHeaders(t, r) {
+			t.Fatalf("invalid read headers")
+		}
+
+		compressed, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("error read body: %s", err)
+		}
+
+		reqBuf, err := snappy.Decode(nil, compressed)
+		if err != nil {
+			t.Fatalf("error decode compressed data:%s", err)
+		}
+
+		var req prompb.ReadRequest
+		if err := proto.Unmarshal(reqBuf, &req); err != nil {
+			t.Fatalf("error unmarshal read request: %s", err)
+		}
+
+		resp := &prompb.ReadResponse{
+			Results: make([]*prompb.QueryResult, len(req.Queries)),
+		}
+
+		for i, r := range req.Queries {
+			startTs := r.StartTimestampMs
+			endTs := r.EndTimestampMs
+			ts := make([]*prompb.TimeSeries, len(rrs.series))
+			for i, s := range rrs.series {
+				var samples []prompb.Sample
+				for _, sample := range s.Samples {
+					if sample.Timestamp >= startTs && sample.Timestamp < endTs {
+						samples = append(samples, sample)
+					}
+				}
+				var series prompb.TimeSeries
+				if len(samples) > 0 {
+					series.Labels = s.Labels
+					series.Samples = samples
+				}
+				ts[i] = &series
+			}
+
+			resp.Results[i] = &prompb.QueryResult{Timeseries: ts}
+			data, err := proto.Marshal(resp)
+			if err != nil {
+				t.Fatalf("error marshal response: %s", err)
+			}
+
+			compressed = snappy.Encode(nil, data)
+
+			w.Header().Set("Content-Type", "application/x-protobuf")
+			w.Header().Set("Content-Encoding", "snappy")
+			w.WriteHeader(http.StatusOK)
+
+			if _, err := w.Write(compressed); err != nil {
+				t.Fatalf("snappy encode error: %s", err)
+			}
+		}
+	})
+}
+
+func NewRemoteReadStreamServer(t *testing.T) *RemoteReadServer {
+	rrs := &RemoteReadServer{
+		series: make([]*prompb.TimeSeries, 0),
+	}
+	rrs.server = httptest.NewServer(rrs.getStreamReadHandler(t))
+	return rrs
+}
+
+func (rrs *RemoteReadServer) InitMockStorage(series []*prompb.TimeSeries) {
+	rrs.storage = NewMockStorage(series)
+}
+
+func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !validateStreamReadHeaders(t, r) {
+			t.Fatalf("invalid read headers")
+		}
+
+		f, ok := w.(http.Flusher)
+		if !ok {
+			t.Fatalf("internal http.ResponseWriter does not implement http.Flusher interface")
+		}
+
+		stream := remote.NewChunkedWriter(w, f)
+
+		data, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("error read body: %s", err)
+		}
+
+		decodedData, err := snappy.Decode(nil, data)
+		if err != nil {
+			t.Fatalf("error decode compressed data:%s", err)
+		}
+
+		var req prompb.ReadRequest
+		if err := proto.Unmarshal(decodedData, &req); err != nil {
+			t.Fatalf("error unmarshal read request: %s", err)
+		}
+
+		var chunks []prompb.Chunk
+		ctx := context.Background()
+		for idx, r := range req.Queries {
+			startTs := r.StartTimestampMs
+			endTs := r.EndTimestampMs
+
+			var matchers []*labels.Matcher
+			cb := func() (int64, error) { return 0, nil }
+
+			c := remote.NewSampleAndChunkQueryableClient(rrs.storage, nil, matchers, true, cb)
+
+			q, err := c.ChunkQuerier(ctx, startTs, endTs)
+			if err != nil {
+				t.Fatalf("error init chunk querier: %s", err)
+			}
+
+			ss := q.Select(false, nil, matchers...)
+			for ss.Next() {
+				series := ss.At()
+				iter := series.Iterator()
+				labels := remote.MergeLabels(labelsToLabelsProto(series.Labels()), nil)
+
+				frameBytesLeft := maxBytesInFrame
+				for _, lb := range labels {
+					frameBytesLeft -= lb.Size()
+				}
+
+				isNext := iter.Next()
+
+				for isNext {
+					chunk := iter.At()
+
+					if chunk.Chunk == nil {
+						t.Fatalf("error found not populated chunk returned by SeriesSet at ref: %v", chunk.Ref)
+					}
+
+					chunks = append(chunks, prompb.Chunk{
+						MinTimeMs: chunk.MinTime,
+						MaxTimeMs: chunk.MaxTime,
+						Type:      prompb.Chunk_Encoding(chunk.Chunk.Encoding()),
+						Data:      chunk.Chunk.Bytes(),
+					})
+
+					frameBytesLeft -= chunks[len(chunks)-1].Size()
+
+					// We are fine with minor inaccuracy of max bytes per frame. The inaccuracy will be max of full chunk size.
+					isNext = iter.Next()
+					if frameBytesLeft > 0 && isNext {
+						continue
+					}
+
+					resp := &prompb.ChunkedReadResponse{
+						ChunkedSeries: []*prompb.ChunkedSeries{
+							{Labels: labels, Chunks: chunks},
+						},
+						QueryIndex: int64(idx),
+					}
+
+					b, err := proto.Marshal(resp)
+					if err != nil {
+						t.Fatalf("error marshal response: %s", err)
+					}
+
+					if _, err := stream.Write(b); err != nil {
+						t.Fatalf("error write to stream: %s", err)
+					}
+					chunks = chunks[:0]
+					rrs.storage.Reset()
+				}
+				if err := iter.Err(); err != nil {
+					t.Fatalf("error iterate over chunk series: %s", err)
+				}
+			}
+		}
+	})
+}
+
+func validateReadHeaders(t *testing.T, r *http.Request) bool {
+	if r.Method != http.MethodPost {
+		t.Fatalf("got %q method, expected %q", r.Method, http.MethodPost)
+	}
+	if r.Header.Get("Content-Encoding") != "snappy" {
+		t.Fatalf("got %q content encoding header, expected %q", r.Header.Get("Content-Encoding"), "snappy")
+	}
+	if r.Header.Get("Content-Type") != "application/x-protobuf" {
+		t.Fatalf("got %q content type header, expected %q", r.Header.Get("Content-Type"), "application/x-protobuf")
+	}
+
+	remoteReadVersion := r.Header.Get("X-Prometheus-Remote-Read-Version")
+	if remoteReadVersion == "" {
+		t.Fatalf("got empty prometheus remote read header")
+	}
+	if !strings.HasPrefix(remoteReadVersion, "0.1.") {
+		t.Fatalf("wrong remote version defined")
+	}
+
+	return true
+}
+
+func validateStreamReadHeaders(t *testing.T, r *http.Request) bool {
+	if r.Method != http.MethodPost {
+		t.Fatalf("got %q method, expected %q", r.Method, http.MethodPost)
+	}
+	if r.Header.Get("Content-Encoding") != "snappy" {
+		t.Fatalf("got %q content encoding header, expected %q", r.Header.Get("Content-Encoding"), "snappy")
+	}
+	if r.Header.Get("Content-Type") != "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" {
+		t.Fatalf("got %q content type header, expected %q", r.Header.Get("Content-Type"), "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse")
+	}
+
+	remoteReadVersion := r.Header.Get("X-Prometheus-Remote-Read-Version")
+	if remoteReadVersion == "" {
+		t.Fatalf("got empty prometheus remote read header")
+	}
+	if !strings.HasPrefix(remoteReadVersion, "0.1.") {
+		t.Fatalf("wrong remote version defined")
+	}
+	return true
+}
+
+func GenerateRemoteReadSeries(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries {
+	var ts []*prompb.TimeSeries
+	j := 0
+	for i := 0; i < int(numOfSeries); i++ {
+		if i%3 == 0 {
+			j++
+		}
+
+		timeSeries := prompb.TimeSeries{
+			Labels: []prompb.Label{
+				{Name: labels.MetricName, Value: fmt.Sprintf("vm_metric_%d", j)},
+				{Name: "job", Value: strconv.Itoa(i)},
+			},
+		}
+
+		ts = append(ts, &timeSeries)
+	}
+
+	for i := range ts {
+		ts[i].Samples = generateRemoteReadSamples(i, start, end, numOfSamples)
+	}
+
+	return ts
+}
+
+func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64) []prompb.Sample {
+	samples := make([]prompb.Sample, 0)
+	delta := (endTime - startTime) / numOfSamples
+
+	t := startTime
+	for t != endTime {
+		v := 100 * int64(idx)
+		samples = append(samples, prompb.Sample{
+			Timestamp: t * 1000,
+			Value:     float64(v),
+		})
+		t = t + delta
+	}
+
+	return samples
+}
+
+type MockStorage struct {
+	query *prompb.Query
+	store []*prompb.TimeSeries
+}
+
+func NewMockStorage(series []*prompb.TimeSeries) *MockStorage {
+	return &MockStorage{store: series}
+}
+
+func (ms *MockStorage) Read(_ context.Context, query *prompb.Query) (*prompb.QueryResult, error) {
+	if ms.query != nil {
+		return nil, fmt.Errorf("expected only one call to remote client got: %v", query)
+	}
+	ms.query = query
+
+	q := &prompb.QueryResult{Timeseries: make([]*prompb.TimeSeries, 0, len(ms.store))}
+	for _, s := range ms.store {
+		var samples []prompb.Sample
+		for _, sample := range s.Samples {
+			if sample.Timestamp >= query.StartTimestampMs && sample.Timestamp < query.EndTimestampMs {
+				samples = append(samples, sample)
+			}
+		}
+		var series prompb.TimeSeries
+		if len(samples) > 0 {
+			series.Labels = s.Labels
+			series.Samples = samples
+		}
+
+		q.Timeseries = append(q.Timeseries, &series)
+	}
+	return q, nil
+}
+
+func (ms *MockStorage) Reset() {
+	ms.query = nil
+}
+
+func labelsToLabelsProto(labels labels.Labels) []prompb.Label {
+	result := make([]prompb.Label, 0, len(labels))
+	for _, l := range labels {
+		result = append(result, prompb.Label{
+			Name:  l.Name,
+			Value: l.Value,
+		})
+	}
+	return result
+}

app/vmctl/testdata/servers_integration_test/remote_write_server.go

@@ -0,0 +1,86 @@
+package remote_read_integration
+
+import (
+	"bufio"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+)
+
+type RemoteWriteServer struct {
+	server *httptest.Server
+	series []vm.TimeSeries
+}
+
+// NewRemoteWriteServer prepares test remote write server
+func NewRemoteWriteServer(t *testing.T) *RemoteWriteServer {
+	rws := &RemoteWriteServer{series: make([]vm.TimeSeries, 0)}
+	mux := http.NewServeMux()
+	mux.Handle("/api/v1/import", rws.getWriteHandler(t))
+	mux.Handle("/health", rws.handlePing())
+	rws.server = httptest.NewServer(mux)
+	return rws
+}
+
+// Close closes the server.
+func (rws *RemoteWriteServer) Close() {
+	rws.server.Close()
+}
+
+func (rws *RemoteWriteServer) ExpectedSeries(series []vm.TimeSeries) {
+	rws.series = append(rws.series, series...)
+}
+
+func (rws *RemoteWriteServer) URL() string {
+	return rws.server.URL
+}
+
+func (rws *RemoteWriteServer) getWriteHandler(t *testing.T) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var tss []vm.TimeSeries
+		scanner := bufio.NewScanner(r.Body)
+		var rows parser.Rows
+		for scanner.Scan() {
+
+			rows.Unmarshal(scanner.Text())
+			for _, row := range rows.Rows {
+				var labelPairs []vm.LabelPair
+				var ts vm.TimeSeries
+				nameValue := ""
+				for _, tag := range row.Tags {
+					if string(tag.Key) == "__name__" {
+						nameValue = string(tag.Value)
+						continue
+					}
+					labelPairs = append(labelPairs, vm.LabelPair{Name: string(tag.Key), Value: string(tag.Value)})
+				}
+
+				ts.Values = append(ts.Values, row.Values...)
+				ts.Timestamps = append(ts.Timestamps, row.Timestamps...)
+				ts.Name = nameValue
+				ts.LabelPairs = labelPairs
+				tss = append(tss, ts)
+			}
+			rows.Reset()
+		}
+
+		if !reflect.DeepEqual(tss, rws.series) {
+			w.WriteHeader(http.StatusInternalServerError)
+			t.Fatalf("datasets not equal, expected: %#v; \n got: %#v", rws.series, tss)
+			return
+		}
+
+		w.WriteHeader(http.StatusNoContent)
+	})
+}
+
+func (rws *RemoteWriteServer) handlePing() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("OK"))
+	})
+}

app/vmctl/utils/tls.go

@@ -0,0 +1,25 @@
+package utils
+
+import (
+	"crypto/tls"
+	"net/http"
+	"strings"
+)
+
+// Transport creates http.Transport object based on provided URL.
+// Returns Transport with TLS configuration if URL contains `https` prefix
+func Transport(URL string, insecureSkipVerify bool) *http.Transport {
+	t := http.DefaultTransport.(*http.Transport).Clone()
+	if !strings.HasPrefix(URL, "https") {
+		return t
+	}
+	t.TLSClientConfig = TLSConfig(insecureSkipVerify)
+	return t
+}
+
+// TLSConfig creates tls.Config object from provided arguments
+func TLSConfig(insecureSkipVerify bool) *tls.Config {
+	return &tls.Config{
+		InsecureSkipVerify: insecureSkipVerify,
+	}
+}

app/vmgateway/README.md

@@ -311,6 +311,8 @@ The shortlist of configuration flags include the following:
      Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
      Format for logs. Possible values: default, json (default "default")
+  -loggerJSONFields string
+     Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg
   -loggerLevel string
      Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string

app/vminsert/common/insert_ctx.go

@@ -19,7 +19,10 @@ type InsertCtx struct {
 	mrs            []storage.MetricRow
 	metricNamesBuf []byte
 
-	relabelCtx relabel.Ctx
+	relabelCtx    relabel.Ctx
+	streamAggrCtx streamAggrCtx
+
+	skipStreamAggr bool
 }
 
 // Reset resets ctx for future fill with rowsLen rows.
@@ -42,6 +45,8 @@ func (ctx *InsertCtx) Reset(rowsLen int) {
 	ctx.mrs = ctx.mrs[:0]
 	ctx.metricNamesBuf = ctx.metricNamesBuf[:0]
 	ctx.relabelCtx.Reset()
+	ctx.streamAggrCtx.Reset()
+	ctx.skipStreamAggr = false
 }
 
 func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) []byte {
@@ -132,6 +137,16 @@ func (ctx *InsertCtx) ApplyRelabeling() {
 
 // FlushBufs flushes buffered rows to the underlying storage.
 func (ctx *InsertCtx) FlushBufs() error {
+	if sa != nil && !ctx.skipStreamAggr {
+		ctx.streamAggrCtx.push(ctx.mrs)
+		if !*streamAggrKeepInput {
+			ctx.Reset(0)
+			return nil
+		}
+	}
+	// There is no need in limiting the number of concurrent calls to vmstorage.AddRows() here,
+	// since the number of concurrent FlushBufs() calls should be already limited via writeconcurrencylimiter
+	// used at every ParseStream() call under lib/protoparser/*/streamparser.go
 	err := vmstorage.AddRows(ctx.mrs)
 	ctx.Reset(0)
 	if err == nil {

app/vminsert/common/streamaggr.go

@@ -0,0 +1,119 @@
+package common
+
+import (
+	"flag"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
+)
+
+var (
+	streamAggrConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
+		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
+		"See also -remoteWrite.streamAggr.keepInput")
+	streamAggrKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep input samples after the aggregation with -streamAggr.config. "+
+		"By default the input is dropped after the aggregation, so only the aggregate data is stored. "+
+		"See https://docs.victoriametrics.com/stream-aggregation.html")
+)
+
+// InitStreamAggr must be called after flag.Parse and before using the common package.
+//
+// MustStopStreamAggr must be called when stream aggr is no longer needed.
+func InitStreamAggr() {
+	if *streamAggrConfig == "" {
+		// Nothing to initialize
+		return
+	}
+	a, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries)
+	if err != nil {
+		logger.Fatalf("cannot load -streamAggr.config=%q: %s", *streamAggrConfig, err)
+	}
+	sa = a
+}
+
+// MustStopStreamAggr stops stream aggregators.
+func MustStopStreamAggr() {
+	sa.MustStop()
+	sa = nil
+}
+
+var sa *streamaggr.Aggregators
+
+type streamAggrCtx struct {
+	mn  storage.MetricName
+	tss [1]prompbmarshal.TimeSeries
+}
+
+func (ctx *streamAggrCtx) Reset() {
+	ctx.mn.Reset()
+	ts := &ctx.tss[0]
+	promrelabel.CleanLabels(ts.Labels)
+}
+
+func (ctx *streamAggrCtx) push(mrs []storage.MetricRow) {
+	mn := &ctx.mn
+	tss := ctx.tss[:]
+	ts := &tss[0]
+	labels := ts.Labels
+	samples := ts.Samples
+	for _, mr := range mrs {
+		if err := mn.UnmarshalRaw(mr.MetricNameRaw); err != nil {
+			logger.Panicf("BUG: cannot unmarshal recently marshaled MetricName: %s", err)
+		}
+
+		labels = append(labels[:0], prompbmarshal.Label{
+			Name:  "__name__",
+			Value: bytesutil.ToUnsafeString(mn.MetricGroup),
+		})
+		for _, tag := range mn.Tags {
+			labels = append(labels, prompbmarshal.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+
+		samples = append(samples[:0], prompbmarshal.Sample{
+			Timestamp: mr.Timestamp,
+			Value:     mr.Value,
+		})
+
+		ts.Labels = labels
+		ts.Samples = samples
+
+		sa.Push(tss)
+	}
+}
+
+func pushAggregateSeries(tss []prompbmarshal.TimeSeries) {
+	currentTimestamp := int64(fasttime.UnixTimestamp()) * 1000
+	var ctx InsertCtx
+	ctx.Reset(len(tss))
+	ctx.skipStreamAggr = true
+	for _, ts := range tss {
+		labels := ts.Labels
+		for _, label := range labels {
+			name := label.Name
+			if name == "__name__" {
+				name = ""
+			}
+			ctx.AddLabel(name, label.Value)
+		}
+		value := ts.Samples[0].Value
+		if err := ctx.WriteDataPoint(nil, ctx.Labels, currentTimestamp, value); err != nil {
+			logger.Errorf("cannot store aggregate series: %s", err)
+			// Do not continue pushing the remaining samples, since it is likely they will return the same error.
+			return
+		}
+	}
+	// There is no need in limiting the number of concurrent calls to vmstorage.AddRows() here,
+	// since the number of concurrent pushAggregateSeries() calls should be already limited by lib/streamaggr.
+	if err := vmstorage.AddRows(ctx.mrs); err != nil {
+		logger.Errorf("cannot flush aggregate series: %s", err)
+	}
+}

app/vminsert/csvimport/request_handler.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -23,10 +22,8 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
-		})
+	return parser.ParseStream(req, func(rows []parser.Row) error {
+		return insertRows(rows, extraLabels)
 	})
 }
 

app/vminsert/datadog/request_handler.go

@@ -1,7 +1,6 @@
 package datadog
 
 import (
-	"fmt"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
@@ -9,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -26,15 +24,9 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		ce := req.Header.Get("Content-Encoding")
-		err := parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
-			return insertRows(series, extraLabels)
-		})
-		if err != nil {
-			return fmt.Errorf("headers: %q; err: %w", req.Header, err)
-		}
-		return nil
+	ce := req.Header.Get("Content-Encoding")
+	return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+		return insertRows(series, extraLabels)
 	})
 }
 

app/vminsert/graphite/request_handler.go

@@ -6,7 +6,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -19,9 +18,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, insertRows)
-	})
+	return parser.ParseStream(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vminsert/influx/request_handler.go

@@ -15,7 +15,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -35,10 +34,8 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(r io.Reader) error {
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
-			return insertRows(db, rows, nil)
-		})
+	return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
+		return insertRows(db, rows, nil)
 	})
 }
 
@@ -50,15 +47,13 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-		q := req.URL.Query()
-		precision := q.Get("precision")
-		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
-		db := q.Get("db")
-		return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
-			return insertRows(db, rows, extraLabels)
-		})
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	q := req.URL.Query()
+	precision := q.Get("precision")
+	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+	db := q.Get("db")
+	return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+		return insertRows(db, rows, extraLabels)
 	})
 }
 

app/vminsert/main.go

@@ -9,6 +9,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	vminsertCommon "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
@@ -34,7 +35,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -66,10 +66,10 @@ var staticServer = http.FileServer(http.FS(staticFiles))
 // Init initializes vminsert.
 func Init() {
 	relabel.Init()
+	vminsertCommon.InitStreamAggr()
 	storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries)
 	storage.SetMaxLabelValueLen(*maxLabelValueLen)
 	common.StartUnmarshalWorkers()
-	writeconcurrencylimiter.Init()
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
 	}
@@ -103,6 +103,7 @@ func Stop() {
 		opentsdbhttpServer.MustStop()
 	}
 	common.StopUnmarshalWorkers()
+	vminsertCommon.MustStopStreamAggr()
 }
 
 // RequestHandler is a handler for Prometheus remote storage write API
@@ -127,7 +128,14 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.WriteHeader(http.StatusNoContent)
+		statusCode := http.StatusNoContent
+		if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus/metrics/job/") ||
+			strings.HasPrefix(path, "/api/v1/import/prometheus/metrics/job/") {
+			// Return 200 status code for pushgateway requests.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
+			statusCode = http.StatusOK
+		}
+		w.WriteHeader(statusCode)
 		return true
 	}
 	if strings.HasPrefix(path, "/datadog/") {
@@ -245,12 +253,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
-			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
-				StatusCode: http.StatusUnauthorized,
-			}
-			httpserver.Errorf(w, r, "%s", err)
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -259,12 +262,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
-			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
-				StatusCode: http.StatusUnauthorized,
-			}
-			httpserver.Errorf(w, r, "%s", err)
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()

app/vminsert/native/request_handler.go

@@ -12,7 +12,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -28,10 +27,8 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
-			return insertRows(block, extraLabels)
-		})
+	return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+		return insertRows(block, extraLabels)
 	})
 }
 

app/vminsert/opentsdb/request_handler.go

@@ -6,7 +6,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -19,9 +18,7 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(r, insertRows)
-	})
+	return parser.ParseStream(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vminsert/opentsdbhttp/request_handler.go

@@ -9,7 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -28,10 +27,8 @@ func InsertHandler(req *http.Request) error {
 		if err != nil {
 			return err
 		}
-		return writeconcurrencylimiter.Do(func() error {
-			return parser.ParseStream(req, func(rows []parser.Row) error {
-				return insertRows(rows, extraLabels)
-			})
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(rows, extraLabels)
 		})
 	default:
 		return fmt.Errorf("unexpected path requested on HTTP OpenTSDB server: %q", path)

app/vminsert/prometheusimport/request_handler.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -27,12 +26,10 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-		return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
-		}, nil)
-	})
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+		return insertRows(rows, extraLabels)
+	}, nil)
 }
 
 func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {

app/vminsert/promremotewrite/request_handler.go

@@ -9,7 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -24,10 +23,8 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
-			return insertRows(tss, extraLabels)
-		})
+	return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
+		return insertRows(tss, extraLabels)
 	})
 }
 

app/vminsert/vmimport/request_handler.go

@@ -12,7 +12,6 @@ import (
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -29,11 +28,9 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return writeconcurrencylimiter.Do(func() error {
-		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-		return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
-			return insertRows(rows, extraLabels)
-		})
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return parser.ParseStream(req.Body, isGzipped, func(rows []parser.Row) error {
+		return insertRows(rows, extraLabels)
 	})
 }
 

app/vmrestore/README.md

@@ -129,6 +129,8 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
      Per-second limit on the number of ERROR messages. If more than the given number of errors are emitted per second, the remaining errors are suppressed. Zero values disable the rate limit
   -loggerFormat string
      Format for logs. Possible values: default, json (default "default")
+  -loggerJSONFields string
+     Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg
   -loggerLevel string
      Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO")
   -loggerOutput string

app/vmselect/graphite/metrics_api.go

@@ -16,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/metricsql"
 )
 
 var maxTagValueSuffixes = flag.Int("search.maxTagValueSuffixesPerSearch", 100e3, "The maximum number of tag value suffixes returned from /metrics/find")
@@ -349,7 +350,7 @@ func getRegexpForQuery(query string, delimiter byte) (*regexp.Regexp, error) {
 	if len(tail) > 0 {
 		return nil, fmt.Errorf("unexpected tail left after parsing query %q; tail: %q", query, tail)
 	}
-	re, err := regexp.Compile(rs)
+	re, err := metricsql.CompileRegexp(rs)
 	regexpCache[k] = &regexpCacheEntry{
 		re:  re,
 		err: err,

app/vmselect/main.go

@@ -62,7 +62,7 @@ func Init() {
 	netstorage.InitTmpBlocksDir(tmpDirPath)
 	promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult")
 
-	concurrencyCh = make(chan struct{}, *maxConcurrentRequests)
+	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
 	initVMAlertProxy()
 }
 
@@ -71,17 +71,17 @@ func Stop() {
 	promql.StopRollupResultCache()
 }
 
-var concurrencyCh chan struct{}
+var concurrencyLimitCh chan struct{}
 
 var (
 	concurrencyLimitReached = metrics.NewCounter(`vm_concurrent_select_limit_reached_total`)
 	concurrencyLimitTimeout = metrics.NewCounter(`vm_concurrent_select_limit_timeout_total`)
 
 	_ = metrics.NewGauge(`vm_concurrent_select_capacity`, func() float64 {
-		return float64(cap(concurrencyCh))
+		return float64(cap(concurrencyLimitCh))
 	})
 	_ = metrics.NewGauge(`vm_concurrent_select_current`, func() float64 {
-		return float64(len(concurrencyCh))
+		return float64(len(concurrencyLimitCh))
 	})
 )
 
@@ -99,8 +99,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	// Limit the number of concurrent queries.
 	select {
-	case concurrencyCh <- struct{}{}:
-		defer func() { <-concurrencyCh }()
+	case concurrencyLimitCh <- struct{}{}:
+		defer func() { <-concurrencyLimitCh }()
 	default:
 		// Sleep for a while until giving up. This should resolve short bursts in requests.
 		concurrencyLimitReached.Inc()
@@ -110,18 +110,18 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		t := timerpool.Get(d)
 		select {
-		case concurrencyCh <- struct{}{}:
-			qt.Printf("wait in queue because -search.maxConcurrentRequests=%d concurrent requests are executed", *maxConcurrentRequests)
+		case concurrencyLimitCh <- struct{}{}:
 			timerpool.Put(t)
-			defer func() { <-concurrencyCh }()
+			qt.Printf("wait in queue because -search.maxConcurrentRequests=%d concurrent requests are executed", *maxConcurrentRequests)
+			defer func() { <-concurrencyLimitCh }()
 		case <-t.C:
 			timerpool.Put(t)
 			concurrencyLimitTimeout.Inc()
 			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("cannot handle more than %d concurrent search requests during %s; possible solutions: "+
-					"increase `-search.maxQueueDuration`; increase `-search.maxQueryDuration`; increase `-search.maxConcurrentRequests`; "+
-					"increase server capacity",
-					*maxConcurrentRequests, d),
+				Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
+					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
+					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
+					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
@@ -145,8 +145,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	if path == "/internal/resetRollupResultCache" {
-		if len(*resetCacheAuthKey) > 0 && r.FormValue("authKey") != *resetCacheAuthKey {
-			sendPrometheusError(w, r, fmt.Errorf("invalid authKey=%q for %q", r.FormValue("authKey"), path))
+		if !httpserver.CheckAuthFlag(w, r, *resetCacheAuthKey, "resetCacheAuthKey") {
 			return true
 		}
 		promql.ResetRollupResultCache()
@@ -162,8 +161,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case strings.HasPrefix(path, "/graphite/"):
 		path = path[len("/graphite"):]
 	}
+
 	// vmui access.
-	if path == "/vmui" || path == "/graph" {
+	switch {
+	case path == "/vmui" || path == "/graph":
 		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
 		// Use relative redirect, since, since the hostname and path prefix may be incorrect if VictoriaMetrics
 		// is hidden behind vmauth or similar proxy.
@@ -172,18 +173,31 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		newURL := path + "/?" + r.Form.Encode()
 		httpserver.Redirect(w, newURL)
 		return true
-	}
-	if strings.HasPrefix(path, "/vmui/") {
+	case strings.HasPrefix(path, "/vmui/"):
+		if path == "/vmui/custom-dashboards" {
+			if err := handleVMUICustomDashboards(w); err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+				return true
+			}
+			return true
+		}
 		r.URL.Path = path
 		vmuiFileServer.ServeHTTP(w, r)
 		return true
-	}
-	if strings.HasPrefix(path, "/graph/") {
+	case strings.HasPrefix(path, "/graph/"):
 		// This is needed for serving /graph URLs from Prometheus datasource in Grafana.
+		if path == "/graph/custom-dashboards" {
+			if err := handleVMUICustomDashboards(w); err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+				return true
+			}
+			return true
+		}
 		r.URL.Path = strings.Replace(path, "/graph/", "/vmui/", 1)
 		vmuiFileServer.ServeHTTP(w, r)
 		return true
 	}
+
 	if strings.HasPrefix(path, "/api/v1/label/") {
 		s := path[len("/api/v1/label/"):]
 		if strings.HasSuffix(s, "/values") {
@@ -412,12 +426,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
-		graphiteTagsDelSeriesRequests.Inc()
-		authKey := r.FormValue("authKey")
-		if authKey != *deleteAuthKey {
-			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -deleteAuthKey command line flag", authKey)
+		if !httpserver.CheckAuthFlag(w, r, *deleteAuthKey, "deleteAuthKey") {
 			return true
 		}
+		graphiteTagsDelSeriesRequests.Inc()
 		if err := graphite.TagsDelSeriesHandler(startTime, w, r); err != nil {
 			graphiteTagsDelSeriesErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
@@ -474,12 +486,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "%s", `{"status":"success","data":[]}`)
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
-		deleteRequests.Inc()
-		authKey := r.FormValue("authKey")
-		if authKey != *deleteAuthKey {
-			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -deleteAuthKey command line flag", authKey)
+		if !httpserver.CheckAuthFlag(w, r, *deleteAuthKey, "deleteAuthKey") {
 			return true
 		}
+		deleteRequests.Inc()
 		if err := prometheus.DeleteHandler(startTime, r); err != nil {
 			deleteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)

app/vmselect/netstorage/netstorage.go

@@ -5,8 +5,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"math/rand"
-	"regexp"
+	"runtime"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -17,11 +16,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastrand"
+	"github.com/VictoriaMetrics/metricsql"
 )
 
 var (
@@ -133,16 +131,66 @@ func (tsw *timeseriesWork) do(r *Result, workerID uint) error {
 	return nil
 }
 
-func timeseriesWorker(tsws []*timeseriesWork, workerID uint) {
+func timeseriesWorker(qt *querytracer.Tracer, workChs []chan *timeseriesWork, workerID uint) {
+	tmpResult := getTmpResult()
+
+	// Perform own work at first.
+	rowsProcessed := 0
+	seriesProcessed := 0
+	ch := workChs[workerID]
+	for tsw := range ch {
+		tsw.err = tsw.do(&tmpResult.rs, workerID)
+		rowsProcessed += tsw.rowsProcessed
+		seriesProcessed++
+	}
+	qt.Printf("own work processed: series=%d, samples=%d", seriesProcessed, rowsProcessed)
+
+	// Then help others with the remaining work.
+	rowsProcessed = 0
+	seriesProcessed = 0
+	idx := int(workerID)
+	for {
+		tsw, idxNext := stealTimeseriesWork(workChs, idx)
+		if tsw == nil {
+			// There is no more work
+			break
+		}
+		tsw.err = tsw.do(&tmpResult.rs, workerID)
+		rowsProcessed += tsw.rowsProcessed
+		seriesProcessed++
+		idx = idxNext
+	}
+	qt.Printf("others work processed: series=%d, samples=%d", seriesProcessed, rowsProcessed)
+
+	putTmpResult(tmpResult)
+}
+
+func stealTimeseriesWork(workChs []chan *timeseriesWork, startIdx int) (*timeseriesWork, int) {
+	for i := startIdx; i < startIdx+len(workChs); i++ {
+		// Give a chance other goroutines to perform their work
+		runtime.Gosched()
+
+		idx := i % len(workChs)
+		ch := workChs[idx]
+		// It is expected that every channel in the workChs is already closed,
+		// so the next line should return immediately.
+		tsw, ok := <-ch
+		if ok {
+			return tsw, idx
+		}
+	}
+	return nil, startIdx
+}
+
+func getTmpResult() *result {
 	v := resultPool.Get()
 	if v == nil {
 		v = &result{}
 	}
-	r := v.(*result)
-	for _, tsw := range tsws {
-		err := tsw.do(&r.rs, workerID)
-		tsw.err = err
-	}
+	return v.(*result)
+}
+
+func putTmpResult(r *result) {
 	currentTime := fasttime.UnixTimestamp()
 	if cap(r.rs.Values) > 1024*1024 && 4*len(r.rs.Values) < cap(r.rs.Values) && currentTime-r.lastResetTime > 10 {
 		// Reset r.rs in order to preseve memory usage after processing big time series with millions of rows.
@@ -170,87 +218,113 @@ func (rss *Results) RunParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 	qt = qt.NewChild("parallel process of fetched data")
 	defer rss.mustClose()
 
-	// Prepare work for workers.
-	tsws := make([]*timeseriesWork, len(rss.packedTimeseries))
+	rowsProcessedTotal, err := rss.runParallel(qt, f)
+	seriesProcessedTotal := len(rss.packedTimeseries)
+	rss.packedTimeseries = rss.packedTimeseries[:0]
+
+	rowsReadPerQuery.Update(float64(rowsProcessedTotal))
+	seriesReadPerQuery.Update(float64(seriesProcessedTotal))
+
+	qt.Donef("series=%d, samples=%d", seriesProcessedTotal, rowsProcessedTotal)
+
+	return err
+}
+
+func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, workerID uint) error) (int, error) {
+	tswsLen := len(rss.packedTimeseries)
+	if tswsLen == 0 {
+		// Nothing to process
+		return 0, nil
+	}
+
 	var mustStop uint32
-	for i := range rss.packedTimeseries {
-		tsw := getTimeseriesWork()
+	initTimeseriesWork := func(tsw *timeseriesWork, pts *packedTimeseries) {
 		tsw.rss = rss
-		tsw.pts = &rss.packedTimeseries[i]
+		tsw.pts = pts
 		tsw.f = f
 		tsw.mustStop = &mustStop
+	}
+	if gomaxprocs == 1 || tswsLen == 1 {
+		// It is faster to process time series in the current goroutine.
+		tsw := getTimeseriesWork()
+		tmpResult := getTmpResult()
+		rowsProcessedTotal := 0
+		var err error
+		for i := range rss.packedTimeseries {
+			initTimeseriesWork(tsw, &rss.packedTimeseries[i])
+			err = tsw.do(&tmpResult.rs, 0)
+			rowsReadPerSeries.Update(float64(tsw.rowsProcessed))
+			rowsProcessedTotal += tsw.rowsProcessed
+			if err != nil {
+				break
+			}
+			tsw.reset()
+		}
+		putTmpResult(tmpResult)
+		putTimeseriesWork(tsw)
+
+		return rowsProcessedTotal, err
+	}
+
+	// Slow path - spin up multiple local workers for parallel data processing.
+	// Do not use global workers pool, since it increases inter-CPU memory ping-poing,
+	// which reduces the scalability on systems with many CPU cores.
+
+	// Prepare the work for workers.
+	tsws := make([]*timeseriesWork, len(rss.packedTimeseries))
+	for i := range rss.packedTimeseries {
+		tsw := getTimeseriesWork()
+		initTimeseriesWork(tsw, &rss.packedTimeseries[i])
 		tsws[i] = tsw
 	}
-	// Shuffle tsws for providing the equal amount of work among workers.
-	r := getRand()
-	r.Shuffle(len(tsws), func(i, j int) {
-		tsws[i], tsws[j] = tsws[j], tsws[i]
-	})
-	putRand(r)
 
-	// Spin up up to gomaxprocs local workers and split work equally among them.
-	// This guarantees linear scalability with the increase of gomaxprocs
-	// (e.g. the number of available CPU cores).
-	itemsPerWorker := 1
-	if len(rss.packedTimeseries) > gomaxprocs {
-		itemsPerWorker = 1 + len(rss.packedTimeseries)/gomaxprocs
+	// Prepare worker channels.
+	workers := len(tsws)
+	if workers > gomaxprocs {
+		workers = gomaxprocs
 	}
-	var start int
-	var i uint
+	itemsPerWorker := (len(tsws) + workers - 1) / workers
+	workChs := make([]chan *timeseriesWork, workers)
+	for i := range workChs {
+		workChs[i] = make(chan *timeseriesWork, itemsPerWorker)
+	}
+
+	// Spread work among workers.
+	for i, tsw := range tsws {
+		idx := i % len(workChs)
+		workChs[idx] <- tsw
+	}
+	// Mark worker channels as closed.
+	for _, workCh := range workChs {
+		close(workCh)
+	}
+
+	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for start < len(tsws) {
-		end := start + itemsPerWorker
-		if end > len(tsws) {
-			end = len(tsws)
-		}
-		chunk := tsws[start:end]
+	for i := range workChs {
 		wg.Add(1)
-		go func(tswsChunk []*timeseriesWork, workerID uint) {
-			defer wg.Done()
-			timeseriesWorker(tswsChunk, workerID)
-		}(chunk, i)
-		start = end
-		i++
+		qtChild := qt.NewChild("worker #%d", i)
+		go func(workerID uint) {
+			timeseriesWorker(qtChild, workChs, workerID)
+			qtChild.Done()
+			wg.Done()
+		}(uint(i))
 	}
-
-	// Wait until work is complete.
 	wg.Wait()
 
 	// Collect results.
 	var firstErr error
 	rowsProcessedTotal := 0
 	for _, tsw := range tsws {
-		if err := tsw.err; err != nil && firstErr == nil {
+		if tsw.err != nil && firstErr == nil {
 			// Return just the first error, since other errors are likely duplicate the first error.
-			firstErr = err
+			firstErr = tsw.err
 		}
 		rowsReadPerSeries.Update(float64(tsw.rowsProcessed))
 		rowsProcessedTotal += tsw.rowsProcessed
 		putTimeseriesWork(tsw)
 	}
-
-	seriesProcessedTotal := len(rss.packedTimeseries)
-	rss.packedTimeseries = rss.packedTimeseries[:0]
-	rowsReadPerQuery.Update(float64(rowsProcessedTotal))
-	seriesReadPerQuery.Update(float64(seriesProcessedTotal))
-
-	qt.Donef("series=%d, samples=%d", seriesProcessedTotal, rowsProcessedTotal)
-
-	return firstErr
-}
-
-var randPool sync.Pool
-
-func getRand() *rand.Rand {
-	v := randPool.Get()
-	if v == nil {
-		v = rand.New(rand.NewSource(int64(fasttime.UnixTimestamp())))
-	}
-	return v.(*rand.Rand)
-}
-
-func putRand(r *rand.Rand) {
-	randPool.Put(r)
+	return rowsProcessedTotal, firstErr
 }
 
 var (
@@ -266,48 +340,30 @@ type packedTimeseries struct {
 	brs        []blockRef
 }
 
-type unpackWorkItem struct {
-	br blockRef
-	tr storage.TimeRange
-}
-
 type unpackWork struct {
-	tbf    *tmpBlocksFile
-	ws     []unpackWorkItem
-	sbs    []*sortBlock
-	doneCh chan error
+	tbf *tmpBlocksFile
+	br  blockRef
+	tr  storage.TimeRange
+	sb  *sortBlock
+	err error
 }
 
 func (upw *unpackWork) reset() {
 	upw.tbf = nil
-	ws := upw.ws
-	for i := range ws {
-		w := &ws[i]
-		w.br = blockRef{}
-		w.tr = storage.TimeRange{}
-	}
-	upw.ws = upw.ws[:0]
-	sbs := upw.sbs
-	for i := range sbs {
-		sbs[i] = nil
-	}
-	upw.sbs = upw.sbs[:0]
-	if n := len(upw.doneCh); n > 0 {
-		logger.Panicf("BUG: upw.doneCh must be empty; it contains %d items now", n)
-	}
+	upw.br = blockRef{}
+	upw.tr = storage.TimeRange{}
+	upw.sb = nil
+	upw.err = nil
 }
 
 func (upw *unpackWork) unpack(tmpBlock *storage.Block) {
-	for _, w := range upw.ws {
-		sb := getSortBlock()
-		if err := sb.unpackFrom(tmpBlock, upw.tbf, w.br, w.tr); err != nil {
-			putSortBlock(sb)
-			upw.doneCh <- fmt.Errorf("cannot unpack block: %w", err)
-			return
-		}
-		upw.sbs = append(upw.sbs, sb)
+	sb := getSortBlock()
+	if err := sb.unpackFrom(tmpBlock, upw.tbf, upw.br, upw.tr); err != nil {
+		putSortBlock(sb)
+		upw.err = fmt.Errorf("cannot unpack block: %w", err)
+		return
 	}
-	upw.doneCh <- nil
+	upw.sb = sb
 }
 
 func getUnpackWork() *unpackWork {
@@ -315,9 +371,7 @@ func getUnpackWork() *unpackWork {
 	if v != nil {
 		return v.(*unpackWork)
 	}
-	return &unpackWork{
-		doneCh: make(chan error, 1),
-	}
+	return &unpackWork{}
 }
 
 func putUnpackWork(upw *unpackWork) {
@@ -327,49 +381,60 @@ func putUnpackWork(upw *unpackWork) {
 
 var unpackWorkPool sync.Pool
 
-func scheduleUnpackWork(workChs []chan *unpackWork, uw *unpackWork) {
-	if len(workChs) == 1 {
-		// Fast path for a single worker
-		workChs[0] <- uw
-		return
-	}
-	attempts := 0
-	for {
-		idx := fastrand.Uint32n(uint32(len(workChs)))
-		select {
-		case workChs[idx] <- uw:
-			return
-		default:
-			attempts++
-			if attempts >= len(workChs) {
-				workChs[idx] <- uw
-				return
-			}
-		}
-	}
-}
+func unpackWorker(workChs []chan *unpackWork, workerID uint) {
+	tmpBlock := getTmpStorageBlock()
 
-func unpackWorker(ch <-chan *unpackWork) {
-	v := tmpBlockPool.Get()
-	if v == nil {
-		v = &storage.Block{}
-	}
-	tmpBlock := v.(*storage.Block)
+	// Deal with own work at first.
+	ch := workChs[workerID]
 	for upw := range ch {
 		upw.unpack(tmpBlock)
 	}
-	tmpBlockPool.Put(v)
+
+	// Then help others with their work.
+	idx := int(workerID)
+	for {
+		upw, idxNext := stealUnpackWork(workChs, idx)
+		if upw == nil {
+			// There is no more work
+			break
+		}
+		upw.unpack(tmpBlock)
+		idx = idxNext
+	}
+
+	putTmpStorageBlock(tmpBlock)
 }
 
-var tmpBlockPool sync.Pool
+func stealUnpackWork(workChs []chan *unpackWork, startIdx int) (*unpackWork, int) {
+	for i := startIdx; i < startIdx+len(workChs); i++ {
+		// Give a chance other goroutines to perform their work
+		runtime.Gosched()
 
-// unpackBatchSize is the maximum number of blocks that may be unpacked at once by a single goroutine.
-//
-// It is better to load a single goroutine for up to one second on a system with many CPU cores
-// in order to reduce inter-CPU memory ping-pong.
-// A single goroutine can unpack up to 40 millions of rows per second, while a single block contains up to 8K rows.
-// So the batch size should be 40M / 8K = 5K.
-var unpackBatchSize = 5000
+		idx := i % len(workChs)
+		ch := workChs[idx]
+		// It is expected that every channel in the workChs is already closed,
+		// so the next line should return immediately.
+		upw, ok := <-ch
+		if ok {
+			return upw, idx
+		}
+	}
+	return nil, startIdx
+}
+
+func getTmpStorageBlock() *storage.Block {
+	v := tmpStorageBlockPool.Get()
+	if v == nil {
+		v = &storage.Block{}
+	}
+	return v.(*storage.Block)
+}
+
+func putTmpStorageBlock(sb *storage.Block) {
+	tmpStorageBlockPool.Put(sb)
+}
+
+var tmpStorageBlockPool sync.Pool
 
 // Unpack unpacks pts to dst.
 func (pts *packedTimeseries) Unpack(dst *Result, tbf *tmpBlocksFile, tr storage.TimeRange) error {
@@ -377,93 +442,131 @@ func (pts *packedTimeseries) Unpack(dst *Result, tbf *tmpBlocksFile, tr storage.
 	if err := dst.MetricName.Unmarshal(bytesutil.ToUnsafeBytes(pts.metricName)); err != nil {
 		return fmt.Errorf("cannot unmarshal metricName %q: %w", pts.metricName, err)
 	}
+	sbh := getSortBlocksHeap()
+	var err error
+	sbh.sbs, err = pts.unpackTo(sbh.sbs[:0], tbf, tr)
+	pts.brs = pts.brs[:0]
+	if err != nil {
+		putSortBlocksHeap(sbh)
+		return err
+	}
+	dedupInterval := storage.GetDedupInterval()
+	mergeSortBlocks(dst, sbh, dedupInterval)
+	putSortBlocksHeap(sbh)
+	return nil
+}
 
-	// Spin up local workers.
+func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr storage.TimeRange) ([]*sortBlock, error) {
+	upwsLen := len(pts.brs)
+	if upwsLen == 0 {
+		// Nothing to do
+		return nil, nil
+	}
+	initUnpackWork := func(upw *unpackWork, br blockRef) {
+		upw.tbf = tbf
+		upw.br = br
+		upw.tr = tr
+	}
+	if gomaxprocs == 1 || upwsLen <= 1000 {
+		// It is faster to unpack all the data in the current goroutine.
+		upw := getUnpackWork()
+		samples := 0
+		tmpBlock := getTmpStorageBlock()
+		var err error
+		for _, br := range pts.brs {
+			initUnpackWork(upw, br)
+			upw.unpack(tmpBlock)
+			if upw.err != nil {
+				return dst, upw.err
+			}
+			samples += len(upw.sb.Timestamps)
+			if *maxSamplesPerSeries > 0 && samples > *maxSamplesPerSeries {
+				putSortBlock(upw.sb)
+				err = fmt.Errorf("cannot process more than %d samples per series; either increase -search.maxSamplesPerSeries "+
+					"or reduce time range for the query", *maxSamplesPerSeries)
+				break
+			}
+			dst = append(dst, upw.sb)
+			upw.reset()
+		}
+		putTmpStorageBlock(tmpBlock)
+		putUnpackWork(upw)
+
+		return dst, err
+	}
+
+	// Slow path - spin up multiple local workers for parallel data unpacking.
 	// Do not use global workers pool, since it increases inter-CPU memory ping-poing,
 	// which reduces the scalability on systems with many CPU cores.
-	brsLen := len(pts.brs)
-	workers := brsLen / unpackBatchSize
+
+	// Prepare the work for workers.
+	upws := make([]*unpackWork, upwsLen)
+	for i, br := range pts.brs {
+		upw := getUnpackWork()
+		initUnpackWork(upw, br)
+		upws[i] = upw
+	}
+
+	// Prepare worker channels.
+	workers := len(upws)
 	if workers > gomaxprocs {
 		workers = gomaxprocs
 	}
 	if workers < 1 {
 		workers = 1
 	}
+	itemsPerWorker := (len(upws) + workers - 1) / workers
 	workChs := make([]chan *unpackWork, workers)
-	var workChsWG sync.WaitGroup
+	for i := range workChs {
+		workChs[i] = make(chan *unpackWork, itemsPerWorker)
+	}
+
+	// Spread work among worker channels.
+	for i, upw := range upws {
+		idx := i % len(workChs)
+		workChs[idx] <- upw
+	}
+	// Mark worker channels as closed.
+	for _, workCh := range workChs {
+		close(workCh)
+	}
+
+	// Start workers and wait until they finish the work.
+	var wg sync.WaitGroup
 	for i := 0; i < workers; i++ {
-		// Use unbuffered channel on purpose, since there are high chances
-		// that only a single unpackWork is needed to unpack.
-		// The unbuffered channel should reduce inter-CPU ping-pong in this case,
-		// which should improve the performance in a system with many CPU cores.
-		workChs[i] = make(chan *unpackWork)
-		workChsWG.Add(1)
-		go func(workerID int) {
-			defer workChsWG.Done()
-			unpackWorker(workChs[workerID])
-		}(i)
+		wg.Add(1)
+		go func(workerID uint) {
+			unpackWorker(workChs, workerID)
+			wg.Done()
+		}(uint(i))
 	}
+	wg.Wait()
 
-	// Feed workers with work
-	upws := make([]*unpackWork, 0, 1+brsLen/unpackBatchSize)
-	upw := getUnpackWork()
-	upw.tbf = tbf
-	for _, br := range pts.brs {
-		if len(upw.ws) >= unpackBatchSize {
-			scheduleUnpackWork(workChs, upw)
-			upws = append(upws, upw)
-			upw = getUnpackWork()
-			upw.tbf = tbf
-		}
-		upw.ws = append(upw.ws, unpackWorkItem{
-			br: br,
-			tr: tr,
-		})
-	}
-	scheduleUnpackWork(workChs, upw)
-	upws = append(upws, upw)
-	pts.brs = pts.brs[:0]
-
-	// Wait until work is complete
+	// Collect results.
 	samples := 0
-	sbs := make([]*sortBlock, 0, brsLen)
 	var firstErr error
 	for _, upw := range upws {
-		if err := <-upw.doneCh; err != nil && firstErr == nil {
+		if upw.err != nil && firstErr == nil {
 			// Return the first error only, since other errors are likely the same.
-			firstErr = err
+			firstErr = upw.err
 		}
 		if firstErr == nil {
-			for _, sb := range upw.sbs {
-				samples += len(sb.Timestamps)
-			}
-			if *maxSamplesPerSeries <= 0 || samples < *maxSamplesPerSeries {
-				sbs = append(sbs, upw.sbs...)
-			} else {
+			sb := upw.sb
+			samples += len(sb.Timestamps)
+			if *maxSamplesPerSeries > 0 && samples > *maxSamplesPerSeries {
+				putSortBlock(sb)
 				firstErr = fmt.Errorf("cannot process more than %d samples per series; either increase -search.maxSamplesPerSeries "+
 					"or reduce time range for the query", *maxSamplesPerSeries)
+			} else {
+				dst = append(dst, sb)
 			}
-		}
-		if firstErr != nil {
-			for _, sb := range upw.sbs {
-				putSortBlock(sb)
-			}
+		} else {
+			putSortBlock(upw.sb)
 		}
 		putUnpackWork(upw)
 	}
 
-	// Shut down local workers
-	for _, workCh := range workChs {
-		close(workCh)
-	}
-	workChsWG.Wait()
-
-	if firstErr != nil {
-		return firstErr
-	}
-	dedupInterval := storage.GetDedupInterval()
-	mergeSortBlocks(dst, sbs, dedupInterval)
-	return nil
+	return dst, firstErr
 }
 
 func getSortBlock() *sortBlock {
@@ -483,24 +586,25 @@ var sbPool sync.Pool
 
 var metricRowsSkipped = metrics.NewCounter(`vm_metric_rows_skipped_total{name="vmselect"}`)
 
-func mergeSortBlocks(dst *Result, sbh sortBlocksHeap, dedupInterval int64) {
+func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
 	// Skip empty sort blocks, since they cannot be passed to heap.Init.
-	src := sbh
-	sbh = sbh[:0]
-	for _, sb := range src {
+	sbs := sbh.sbs[:0]
+	for _, sb := range sbh.sbs {
 		if len(sb.Timestamps) == 0 {
 			putSortBlock(sb)
 			continue
 		}
-		sbh = append(sbh, sb)
+		sbs = append(sbs, sb)
 	}
-	if len(sbh) == 0 {
+	sbh.sbs = sbs
+	if sbh.Len() == 0 {
 		return
 	}
-	heap.Init(&sbh)
+	heap.Init(sbh)
 	for {
-		top := sbh[0]
-		if len(sbh) == 1 {
+		sbs := sbh.sbs
+		top := sbs[0]
+		if len(sbs) == 1 {
 			dst.Timestamps = append(dst.Timestamps, top.Timestamps[top.NextIdx:]...)
 			dst.Values = append(dst.Values, top.Values[top.NextIdx:]...)
 			putSortBlock(top)
@@ -508,21 +612,20 @@ func mergeSortBlocks(dst *Result, sbh sortBlocksHeap, dedupInterval int64) {
 		}
 		sbNext := sbh.getNextBlock()
 		tsNext := sbNext.Timestamps[sbNext.NextIdx]
-		topTimestamps := top.Timestamps
 		topNextIdx := top.NextIdx
-		if n := equalTimestampsPrefix(topTimestamps[topNextIdx:], sbNext.Timestamps[sbNext.NextIdx:]); n > 0 && dedupInterval > 0 {
+		if n := equalSamplesPrefix(top, sbNext); n > 0 && dedupInterval > 0 {
 			// Skip n replicated samples at top if deduplication is enabled.
 			top.NextIdx = topNextIdx + n
 		} else {
 			// Copy samples from top to dst with timestamps not exceeding tsNext.
-			top.NextIdx = topNextIdx + binarySearchTimestamps(topTimestamps[topNextIdx:], tsNext)
-			dst.Timestamps = append(dst.Timestamps, topTimestamps[topNextIdx:top.NextIdx]...)
+			top.NextIdx = topNextIdx + binarySearchTimestamps(top.Timestamps[topNextIdx:], tsNext)
+			dst.Timestamps = append(dst.Timestamps, top.Timestamps[topNextIdx:top.NextIdx]...)
 			dst.Values = append(dst.Values, top.Values[topNextIdx:top.NextIdx]...)
 		}
-		if top.NextIdx < len(topTimestamps) {
-			heap.Fix(&sbh, 0)
+		if top.NextIdx < len(top.Timestamps) {
+			heap.Fix(sbh, 0)
 		} else {
-			heap.Pop(&sbh)
+			heap.Pop(sbh)
 			putSortBlock(top)
 		}
 	}
@@ -535,6 +638,14 @@ func mergeSortBlocks(dst *Result, sbh sortBlocksHeap, dedupInterval int64) {
 
 var dedupsDuringSelect = metrics.NewCounter(`vm_deduplicated_samples_total{type="select"}`)
 
+func equalSamplesPrefix(a, b *sortBlock) int {
+	n := equalTimestampsPrefix(a.Timestamps[a.NextIdx:], b.Timestamps[b.NextIdx:])
+	if n == 0 {
+		return 0
+	}
+	return equalValuesPrefix(a.Values[a.NextIdx:a.NextIdx+n], b.Values[b.NextIdx:b.NextIdx+n])
+}
+
 func equalTimestampsPrefix(a, b []int64) int {
 	for i, v := range a {
 		if i >= len(b) || v != b[i] {
@@ -544,6 +655,15 @@ func equalTimestampsPrefix(a, b []int64) int {
 	return len(a)
 }
 
+func equalValuesPrefix(a, b []float64) int {
+	for i, v := range a {
+		if i >= len(b) || v != b[i] {
+			return i
+		}
+	}
+	return len(a)
+}
+
 func binarySearchTimestamps(timestamps []int64, ts int64) int {
 	// The code has been adapted from sort.Search.
 	n := len(timestamps)
@@ -588,48 +708,73 @@ func (sb *sortBlock) unpackFrom(tmpBlock *storage.Block, tbf *tmpBlocksFile, br
 	return nil
 }
 
-type sortBlocksHeap []*sortBlock
+type sortBlocksHeap struct {
+	sbs []*sortBlock
+}
 
-func (sbh sortBlocksHeap) getNextBlock() *sortBlock {
-	if len(sbh) < 2 {
+func (sbh *sortBlocksHeap) getNextBlock() *sortBlock {
+	sbs := sbh.sbs
+	if len(sbs) < 2 {
 		return nil
 	}
-	if len(sbh) < 3 {
-		return sbh[1]
+	if len(sbs) < 3 {
+		return sbs[1]
 	}
-	a := sbh[1]
-	b := sbh[2]
+	a := sbs[1]
+	b := sbs[2]
 	if a.Timestamps[a.NextIdx] <= b.Timestamps[b.NextIdx] {
 		return a
 	}
 	return b
 }
 
-func (sbh sortBlocksHeap) Len() int {
-	return len(sbh)
+func (sbh *sortBlocksHeap) Len() int {
+	return len(sbh.sbs)
 }
 
-func (sbh sortBlocksHeap) Less(i, j int) bool {
-	a := sbh[i]
-	b := sbh[j]
+func (sbh *sortBlocksHeap) Less(i, j int) bool {
+	sbs := sbh.sbs
+	a := sbs[i]
+	b := sbs[j]
 	return a.Timestamps[a.NextIdx] < b.Timestamps[b.NextIdx]
 }
 
-func (sbh sortBlocksHeap) Swap(i, j int) {
-	sbh[i], sbh[j] = sbh[j], sbh[i]
+func (sbh *sortBlocksHeap) Swap(i, j int) {
+	sbs := sbh.sbs
+	sbs[i], sbs[j] = sbs[j], sbs[i]
 }
 
 func (sbh *sortBlocksHeap) Push(x interface{}) {
-	*sbh = append(*sbh, x.(*sortBlock))
+	sbh.sbs = append(sbh.sbs, x.(*sortBlock))
 }
 
 func (sbh *sortBlocksHeap) Pop() interface{} {
-	a := *sbh
-	v := a[len(a)-1]
-	*sbh = a[:len(a)-1]
+	sbs := sbh.sbs
+	v := sbs[len(sbs)-1]
+	sbs[len(sbs)-1] = nil
+	sbh.sbs = sbs[:len(sbs)-1]
 	return v
 }
 
+func getSortBlocksHeap() *sortBlocksHeap {
+	v := sbhPool.Get()
+	if v == nil {
+		return &sortBlocksHeap{}
+	}
+	return v.(*sortBlocksHeap)
+}
+
+func putSortBlocksHeap(sbh *sortBlocksHeap) {
+	sbs := sbh.sbs
+	for i := range sbs {
+		sbs[i] = nil
+	}
+	sbh.sbs = sbs[:0]
+	sbhPool.Put(sbh)
+}
+
+var sbhPool sync.Pool
+
 // DeleteSeries deletes time series matching the given tagFilterss.
 func DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline searchutils.Deadline) (int, error) {
 	qt = qt.NewChild("delete series: %s", sq)
@@ -1012,7 +1157,8 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
 	indexSearchDuration.UpdateDuration(startTime)
 	type blockRefs struct {
-		brs []blockRef
+		brsPrealloc [4]blockRef
+		brs         []blockRef
 	}
 	m := make(map[string]*blockRefs, maxSeriesCount)
 	orderedMetricNames := make([]string, 0, maxSeriesCount)
@@ -1041,20 +1187,19 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 			putStorageSearch(sr)
 			return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
 		}
-		metricName := sr.MetricBlockRef.MetricName
-		brs := m[string(metricName)]
+		metricName := bytesutil.InternBytes(sr.MetricBlockRef.MetricName)
+		brs := m[metricName]
 		if brs == nil {
 			brs = &blockRefs{}
+			brs.brs = brs.brsPrealloc[:0]
 		}
 		brs.brs = append(brs.brs, blockRef{
 			partRef: br.PartRef(),
 			addr:    addr,
 		})
 		if len(brs.brs) == 1 {
-			// An optimization for big number of time series with long metricName values:
-			// use only a single copy of metricName for both orderedMetricNames and m.
-			orderedMetricNames = append(orderedMetricNames, string(metricName))
-			m[orderedMetricNames[len(orderedMetricNames)-1]] = brs
+			orderedMetricNames = append(orderedMetricNames, metricName)
+			m[metricName] = brs
 		}
 	}
 	if err := sr.Error(); err != nil {
@@ -1127,7 +1272,7 @@ func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
 	// Anchor filter regexp to the beginning of the string as Graphite does.
 	// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157
 	filter = "^(?:" + filter + ")"
-	re, err := regexp.Compile(filter)
+	re, err := metricsql.CompileRegexp(filter)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err)
 	}

app/vmselect/netstorage/netstorage_test.go

@@ -9,7 +9,10 @@ func TestMergeSortBlocks(t *testing.T) {
 	f := func(blocks []*sortBlock, dedupInterval int64, expectedResult *Result) {
 		t.Helper()
 		var result Result
-		mergeSortBlocks(&result, blocks, dedupInterval)
+		sbh := getSortBlocksHeap()
+		sbh.sbs = append(sbh.sbs[:0], blocks...)
+		mergeSortBlocks(&result, sbh, dedupInterval)
+		putSortBlocksHeap(sbh)
 		if !reflect.DeepEqual(result.Values, expectedResult.Values) {
 			t.Fatalf("unexpected values;\ngot\n%v\nwant\n%v", result.Values, expectedResult.Values)
 		}
@@ -102,21 +105,35 @@ func TestMergeSortBlocks(t *testing.T) {
 		Values:     []float64{9, 4.2, 2.1, 5.2, 42, 6.1},
 	})
 
-	// Multiple blocks with identical timestamps.
+	// Multiple blocks with identical timestamps and identical values.
 	f([]*sortBlock{
+		{
+			Timestamps: []int64{1, 2, 4, 5},
+			Values:     []float64{9, 5.2, 6.1, 9},
+		},
 		{
 			Timestamps: []int64{1, 2, 4},
 			Values:     []float64{9, 5.2, 6.1},
 		},
+	}, 1, &Result{
+		Timestamps: []int64{1, 2, 4, 5},
+		Values:     []float64{9, 5.2, 6.1, 9},
+	})
+
+	// Multiple blocks with identical timestamps.
+	f([]*sortBlock{
+		{
+			Timestamps: []int64{1, 2, 4, 5},
+			Values:     []float64{9, 5.2, 6.1, 9},
+		},
 		{
 			Timestamps: []int64{1, 2, 4},
 			Values:     []float64{4.2, 2.1, 42},
 		},
 	}, 1, &Result{
-		Timestamps: []int64{1, 2, 4},
-		Values:     []float64{4.2, 2.1, 42},
+		Timestamps: []int64{1, 2, 4, 5},
+		Values:     []float64{9, 5.2, 42, 9},
 	})
-
 	// Multiple blocks with identical timestamps, disabled deduplication.
 	f([]*sortBlock{
 		{

app/vmselect/netstorage/netstorage_timing_test.go

@@ -90,16 +90,18 @@ func benchmarkMergeSortBlocks(b *testing.B, blocks []*sortBlock) {
 	b.ReportAllocs()
 	b.RunParallel(func(pb *testing.PB) {
 		var result Result
-		sbs := make(sortBlocksHeap, len(blocks))
+		sbh := getSortBlocksHeap()
 		for pb.Next() {
 			result.reset()
-			for i, b := range blocks {
+			sbs := sbh.sbs[:0]
+			for _, b := range blocks {
 				sb := getSortBlock()
 				sb.Timestamps = b.Timestamps
 				sb.Values = b.Values
-				sbs[i] = sb
+				sbs = append(sbs, sb)
 			}
-			mergeSortBlocks(&result, sbs, dedupInterval)
+			sbh.sbs = sbs
+			mergeSortBlocks(&result, sbh, dedupInterval)
 		}
 	})
 }

app/vmselect/prometheus/expand-with-exprs.qtpl

@@ -59,7 +59,7 @@ textarea { margin: 1em }
 <h3>Tutorial for WITH expressions in <a href="https://docs.victoriametrics.com/MetricsQL.html">MetricsQL</a></h3>
 
 <p>
-    Let's look at the following real query from <a href="https://grafana.com/dashboards/1860">Node Exporter Full</a> dashboard:
+    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>
@@ -123,7 +123,7 @@ my_resource_utilization(
 </p>
 
 <p>
-    Let's take another nice query from <a href="https://grafana.com/dashboards/1860">Node Exporter Full</a> dashboard:
+    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>

app/vmselect/prometheus/expand-with-exprs.qtpl.go

@@ -131,7 +131,7 @@ func streamwithExprsTutorial(qw422016 *qt422016.Writer) {
 <h3>Tutorial for WITH expressions in <a href="https://docs.victoriametrics.com/MetricsQL.html">MetricsQL</a></h3>
 
 <p>
-    Let's look at the following real query from <a href="https://grafana.com/dashboards/1860">Node Exporter Full</a> dashboard:
+    Let's look at the following real query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>
@@ -195,7 +195,7 @@ my_resource_utilization(
 </p>
 
 <p>
-    Let's take another nice query from <a href="https://grafana.com/dashboards/1860">Node Exporter Full</a> dashboard:
+    Let's take another nice query from <a href="https://grafana.com/grafana/dashboards/1860-node-exporter-full/">Node Exporter Full</a> dashboard:
 </p>
 
 <pre>

app/vmselect/prometheus/prometheus.go

@@ -716,7 +716,6 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		if err := exportHandler(qt, w, cp, "promapi", 0, false); err != nil {
 			return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %w", childQuery, start, end, err)
 		}
-		queryDuration.UpdateDuration(startTime)
 		return nil
 	}
 	if childQuery, windowExpr, stepExpr, offsetExpr := promql.IsRollup(query); childQuery != "" {
@@ -732,7 +731,6 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		if err := queryRangeHandler(qt, startTime, w, childQuery, start, end, step, r, ct, etfs); err != nil {
 			return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %w", childQuery, start, end, step, err)
 		}
-		queryDuration.UpdateDuration(startTime)
 		return nil
 	}
 
@@ -768,10 +766,14 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	}
 	if queryOffset > 0 {
 		for i := range result {
-			timestamps := result[i].Timestamps
+			r := &result[i]
+			// Do not modify r.Timestamps, since they may be shared among multiple series.
+			// Make a copy instead.
+			timestamps := append([]int64{}, r.Timestamps...)
 			for j := range timestamps {
 				timestamps[j] += queryOffset
 			}
+			r.Timestamps = timestamps
 		}
 	}
 
@@ -912,7 +914,9 @@ func removeEmptyValuesAndTimeseries(tss []netstorage.Result) []netstorage.Result
 		// Slow path: remove NaNs.
 		srcTimestamps := ts.Timestamps
 		dstValues := ts.Values[:0]
-		dstTimestamps := ts.Timestamps[:0]
+		// Do not re-use ts.Timestamps for dstTimestamps, since ts.Timestamps
+		// may be shared among multiple time series.
+		dstTimestamps := make([]int64, 0, len(ts.Timestamps))
 		for j, v := range ts.Values {
 			if math.IsNaN(v) {
 				continue

app/vmselect/promql/aggr.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metrics"
@@ -115,16 +116,17 @@ func aggrFuncExt(afe func(tss []*timeseries, modifier *metricsql.ModifierExpr) [
 	for i, ts := range arg {
 		removeGroupTags(&ts.MetricName, modifier)
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
+		k := bytesutil.InternBytes(bb.B)
 		if keepOriginal {
 			ts = argOrig[i]
 		}
-		tss := m[string(bb.B)]
+		tss := m[k]
 		if tss == nil && maxSeries > 0 && len(m) >= maxSeries {
 			// We already reached time series limit after grouping. Skip other time series.
 			continue
 		}
 		tss = append(tss, ts)
-		m[string(bb.B)] = tss
+		m[k] = tss
 	}
 	bbPool.Put(bb)
 
@@ -817,7 +819,7 @@ func lastValue(values []float64) float64 {
 // quantiles calculates the given phis from originValues without modifying originValues, appends them to qs and returns the result.
 func quantiles(qs, phis []float64, originValues []float64) []float64 {
 	a := getFloat64s()
-	a.A = prepareForQuantileFloat64(a.A[:0], originValues)
+	a.prepareForQuantileFloat64(originValues)
 	qs = quantilesSorted(qs, phis, a.A)
 	putFloat64s(a)
 	return qs
@@ -826,22 +828,38 @@ func quantiles(qs, phis []float64, originValues []float64) []float64 {
 // quantile calculates the given phi from originValues without modifying originValues
 func quantile(phi float64, originValues []float64) float64 {
 	a := getFloat64s()
-	a.A = prepareForQuantileFloat64(a.A[:0], originValues)
+	a.prepareForQuantileFloat64(originValues)
 	q := quantileSorted(phi, a.A)
 	putFloat64s(a)
 	return q
 }
 
-// prepareForQuantileFloat64 copies items from src to dst but removes NaNs and sorts the dst
-func prepareForQuantileFloat64(dst, src []float64) []float64 {
+// prepareForQuantileFloat64 copies items from src to a but removes NaNs and sorts items in a.
+func (a *float64s) prepareForQuantileFloat64(src []float64) {
+	dst := a.A[:0]
 	for _, v := range src {
 		if math.IsNaN(v) {
 			continue
 		}
 		dst = append(dst, v)
 	}
-	sort.Float64s(dst)
-	return dst
+	a.A = dst
+	// Use sort.Sort instead of sort.Float64s in order to avoid a memory allocation
+	sort.Sort(a)
+}
+
+func (a *float64s) Len() int {
+	return len(a.A)
+}
+
+func (a *float64s) Swap(i, j int) {
+	x := a.A
+	x[i], x[j] = x[j], x[i]
+}
+
+func (a *float64s) Less(i, j int) bool {
+	x := a.A
+	return x[i] < x[j]
 }
 
 // quantilesSorted calculates the given phis over a sorted list of values, appends them to qs and returns the result.

app/vmselect/promql/aggr_incremental.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/metricsql"
 )
 
@@ -98,7 +99,8 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 	removeGroupTags(&ts.MetricName, &iafc.ae.Modifier)
 	bb := bbPool.Get()
 	bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-	iac := m[string(bb.B)]
+	k := bytesutil.InternBytes(bb.B)
+	iac := m[k]
 	if iac == nil {
 		if iafc.ae.Limit > 0 && len(m) >= iafc.ae.Limit {
 			// Skip this time series, since the limit on the number of output time series has been already reached.
@@ -117,7 +119,7 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 			ts:     tsAggr,
 			values: make([]float64, len(ts.Values)),
 		}
-		m[string(bb.B)] = iac
+		m[k] = iac
 	}
 	bbPool.Put(bb)
 	iafc.callbacks.updateAggrFunc(iac, ts.Values)

app/vmselect/promql/binary_op.go

@@ -5,6 +5,7 @@ import (
 	"math"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/metricsql"
@@ -248,7 +249,8 @@ func groupJoin(singleTimeseriesSide string, be *metricsql.BinaryOpExpr, rvsLeft,
 			bb.B = marshalMetricTagsSorted(bb.B[:0], &tsCopy.MetricName)
 			pair, ok := m[string(bb.B)]
 			if !ok {
-				m[string(bb.B)] = &tsPair{
+				k := bytesutil.InternBytes(bb.B)
+				m[k] = &tsPair{
 					left:  &tsCopy,
 					right: tsRight,
 				}
@@ -504,7 +506,8 @@ func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*time
 				logger.Panicf("BUG: unexpected binary op modifier %q", groupOp)
 			}
 			bb.B = marshalMetricTagsSorted(bb.B[:0], mn)
-			m[string(bb.B)] = append(m[string(bb.B)], ts)
+			k := bytesutil.InternBytes(bb.B)
+			m[k] = append(m[k], ts)
 		}
 		storage.PutMetricName(mn)
 		bbPool.Put(bb)

app/vmselect/promql/eval.go

@@ -505,26 +505,54 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 }
 
 func getCommonLabelFilters(tss []*timeseries) []metricsql.LabelFilter {
-	m := make(map[string][]string)
+	if len(tss) == 0 {
+		return nil
+	}
+	type valuesCounter struct {
+		values map[string]struct{}
+		count  int
+	}
+	m := make(map[string]*valuesCounter, len(tss[0].MetricName.Tags))
 	for _, ts := range tss {
 		for _, tag := range ts.MetricName.Tags {
-			m[string(tag.Key)] = append(m[string(tag.Key)], string(tag.Value))
+			vc, ok := m[string(tag.Key)]
+			if !ok {
+				k := bytesutil.InternBytes(tag.Key)
+				v := bytesutil.InternBytes(tag.Value)
+				m[k] = &valuesCounter{
+					values: map[string]struct{}{
+						v: {},
+					},
+					count: 1,
+				}
+				continue
+			}
+			if len(vc.values) > 100 {
+				// Too many unique values found for the given tag.
+				// Do not make a filter on such values, since it may slow down
+				// search for matching time series.
+				continue
+			}
+			vc.count++
+			if _, ok := vc.values[string(tag.Value)]; !ok {
+				v := bytesutil.InternBytes(tag.Value)
+				vc.values[v] = struct{}{}
+			}
 		}
 	}
 	lfs := make([]metricsql.LabelFilter, 0, len(m))
-	for key, values := range m {
-		if len(values) != len(tss) {
+	var values []string
+	for k, vc := range m {
+		if vc.count != len(tss) {
 			// Skip the tag, since it doesn't belong to all the time series.
 			continue
 		}
-		values = getUniqueValues(values)
-		if len(values) > 1000 {
-			// Skip the filter on the given tag, since it needs to enumerate too many unique values.
-			// This may slow down the search for matching time series.
-			continue
+		values = values[:0]
+		for s := range vc.values {
+			values = append(values, s)
 		}
 		lf := metricsql.LabelFilter{
-			Label: key,
+			Label: k,
 		}
 		if len(values) == 1 {
 			lf.Value = values[0]
@@ -541,18 +569,6 @@ func getCommonLabelFilters(tss []*timeseries) []metricsql.LabelFilter {
 	return lfs
 }
 
-func getUniqueValues(a []string) []string {
-	m := make(map[string]struct{}, len(a))
-	results := make([]string, 0, len(a))
-	for _, s := range a {
-		if _, ok := m[s]; !ok {
-			results = append(results, s)
-			m[s] = struct{}{}
-		}
-	}
-	return results
-}
-
 func joinRegexpValues(a []string) string {
 	var b []byte
 	for i, s := range a {
@@ -652,9 +668,9 @@ func evalExprsInParallel(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.
 	}
 	rvs := make([][]*timeseries, len(es))
 	errs := make([]error, len(es))
+	qt.Printf("eval function args in parallel")
 	var wg sync.WaitGroup
 	for i, e := range es {
-		qt.Printf("eval function args in parallel")
 		wg.Add(1)
 		qtChild := qt.NewChild("eval arg %d", i)
 		go func(e metricsql.Expr, i int) {

app/vmselect/promql/exec.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -121,15 +122,18 @@ func timeseriesToResult(tss []*timeseries, maySort bool) ([]netstorage.Result, e
 	bb := bbPool.Get()
 	for i, ts := range tss {
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-		if _, ok := m[string(bb.B)]; ok {
+		k := bytesutil.InternBytes(bb.B)
+		if _, ok := m[k]; ok {
 			return nil, fmt.Errorf(`duplicate output timeseries: %s`, stringMetricName(&ts.MetricName))
 		}
-		m[string(bb.B)] = struct{}{}
+		m[k] = struct{}{}
 
 		rs := &result[i]
-		rs.MetricName.CopyFrom(&ts.MetricName)
-		rs.Values = append(rs.Values[:0], ts.Values...)
-		rs.Timestamps = append(rs.Timestamps[:0], ts.Timestamps...)
+		rs.MetricName.MoveFrom(&ts.MetricName)
+		rs.Values = ts.Values
+		ts.Values = nil
+		rs.Timestamps = ts.Timestamps
+		ts.Timestamps = nil
 	}
 	bbPool.Put(bb)
 

app/vmselect/promql/exec_test.go

@@ -86,6 +86,61 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("duration-constant", func(t *testing.T) {
+		t.Parallel()
+		q := `1h23m5S`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4985, 4985, 4985, 4985, 4985, 4985},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("num-with-suffix-1", func(t *testing.T) {
+		t.Parallel()
+		q := `123M`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{123e6, 123e6, 123e6, 123e6, 123e6, 123e6},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("num-with-suffix-2", func(t *testing.T) {
+		t.Parallel()
+		q := `1.23TB`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1.23e12, 1.23e12, 1.23e12, 1.23e12, 1.23e12, 1.23e12},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("num-with-suffix-3", func(t *testing.T) {
+		t.Parallel()
+		q := `1.23Mib`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1.23 * (1 << 20), 1.23 * (1 << 20), 1.23 * (1 << 20), 1.23 * (1 << 20), 1.23 * (1 << 20), 1.23 * (1 << 20)},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("num-with-suffix-4", func(t *testing.T) {
+		t.Parallel()
+		q := `1.23mib`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1.23 * (1 << 20), 1.23 * (1 << 20), 1.23 * (1 << 20), 1.23 * (1 << 20), 1.23 * (1 << 20), 1.23 * (1 << 20)},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("simple-arithmetic", func(t *testing.T) {
 		t.Parallel()
 		q := `-1+2 *3 ^ 4+5%6`
@@ -935,7 +990,7 @@ func TestExecSuccess(t *testing.T) {
 		))`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 1, nan, nan},
+			Values:     []float64{nan, nan, 1, 1, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -2149,6 +2204,34 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`label_replace(nonexisting_src_match)`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_replace(time(), "foo", "x", "bar", "")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("x"),
+			},
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`label_replace(nonexisting_src_mismatch)`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_replace(time(), "foo", "x", "bar", "y")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`label_replace(mismatch)`, func(t *testing.T) {
 		t.Parallel()
 		q := `label_replace(label_set(time(), "foo", "foobar"), "__name__", "x${1}y", "foo", "bar(.+)")`
@@ -6692,7 +6775,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `rate((2000-time())[100s])`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{5.5, 4.5, 3.5, 2.5, 1.5, 0.5},
+			Values:     []float64{5, 4, 3, 2, 1, 0},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -6703,7 +6786,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `rate((2000-time())[100s:])`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{5.5, 4.5, 3.5, 2.5, 1.5, 0.5},
+			Values:     []float64{5, 4, 3, 2, 1, 0},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -6714,7 +6797,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `rate((2000-time())[100s:100s])`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 6.5, 4.5, 2.5, 0.5},
+			Values:     []float64{0, 0, 6, 4, 2, 0},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -6725,7 +6808,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `rate((2000-time())[100s:100s] offset 100s)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 3.5, 5.5, 3.5, 1.5},
+			Values:     []float64{0, 0, 7, 5, 3, 1},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -6736,7 +6819,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `rate((2000-time())[100s:100s] offset 100s)[:] offset 100s`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 0, 3.5, 5.5, 3.5},
+			Values:     []float64{0, 0, 0, 7, 5, 3},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r}
@@ -7329,7 +7412,7 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`rollup_scrape_interval()`, func(t *testing.T) {
 		t.Parallel()
-		q := `sort_by_label(rollup_scrape_interval(1[5m:10s]), "rollup")`
+		q := `sort_by_label(rollup_scrape_interval(1[5m:10S]), "rollup")`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{10, 10, 10, 10, 10, 10},

app/vmselect/promql/rollup.go

@@ -145,11 +145,11 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"zscore_over_time":        rollupZScoreOverTime,
 }
 
-// VictoriaMetrics can increase lookbehind window in square brackets for these functions
-// if the given window doesn't contain enough samples for calculations.
+// VictoriaMetrics can extends lookbehind window for these functions
+// in order to make sure it contains enough points for returning non-empty results.
 //
-// This is needed in order to return the expected non-empty graphs when zooming in the graph in Grafana,
-// which is built with `func_name(metric[$__interval])` query.
+// This is needed for returning the expected non-empty graphs when zooming in the graph in Grafana,
+// which is built with `func_name(metric)` query.
 var rollupFuncsCanAdjustWindow = map[string]bool{
 	"default_rollup":         true,
 	"deriv":                  true,
@@ -584,15 +584,23 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	window := rc.Window
 	if window <= 0 {
 		window = rc.Step
+		if rc.MayAdjustWindow && window < maxPrevInterval {
+			// Adjust lookbehind window only if it isn't set explicilty, e.g. rate(foo).
+			// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
+			// when the window doesn't cover at least two raw samples (this is what most users expect).
+			//
+			// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
+			// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
+			//
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
+			window = maxPrevInterval
+		}
 		if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
 			// Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
 			// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 			window = rc.LookbackDelta
 		}
 	}
-	if rc.MayAdjustWindow && window < maxPrevInterval {
-		window = maxPrevInterval
-	}
 	rfa := getRollupFuncArg()
 	rfa.idx = 0
 	rfa.window = window

app/vmselect/promql/rollup_result_cache.go

@@ -488,7 +488,8 @@ func mergeTimeseries(a, b []*timeseries, bStart int64, ec *EvalConfig) []*timese
 	defer bbPool.Put(bb)
 	for _, ts := range a {
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-		m[string(bb.B)] = ts
+		k := bytesutil.InternBytes(bb.B)
+		m[k] = ts
 	}
 
 	rvs := make([]*timeseries, 0, len(a))
@@ -497,12 +498,11 @@ func mergeTimeseries(a, b []*timeseries, bStart int64, ec *EvalConfig) []*timese
 		tmp.denyReuse = true
 		tmp.Timestamps = sharedTimestamps
 		tmp.Values = make([]float64, 0, len(tmp.Timestamps))
-		// Do not use MetricName.CopyFrom for performance reasons.
-		// It is safe to make shallow copy, since tsB must no longer used.
-		tmp.MetricName = tsB.MetricName
+		tmp.MetricName.MoveFrom(&tsB.MetricName)
 
-		bb.B = marshalMetricNameSorted(bb.B[:0], &tsB.MetricName)
-		tsA := m[string(bb.B)]
+		bb.B = marshalMetricNameSorted(bb.B[:0], &tmp.MetricName)
+		k := bytesutil.InternBytes(bb.B)
+		tsA := m[k]
 		if tsA == nil {
 			tStart := ec.Start
 			for tStart < bStart {
@@ -511,7 +511,7 @@ func mergeTimeseries(a, b []*timeseries, bStart int64, ec *EvalConfig) []*timese
 			}
 		} else {
 			tmp.Values = append(tmp.Values, tsA.Values...)
-			delete(m, string(bb.B))
+			delete(m, k)
 		}
 		tmp.Values = append(tmp.Values, tsB.Values...)
 		if len(tmp.Values) != len(tmp.Timestamps) {
@@ -525,9 +525,8 @@ func mergeTimeseries(a, b []*timeseries, bStart int64, ec *EvalConfig) []*timese
 		var tmp timeseries
 		tmp.denyReuse = true
 		tmp.Timestamps = sharedTimestamps
-		// Do not use MetricName.CopyFrom for performance reasons.
-		// It is safe to make shallow copy, since tsA must no longer used.
-		tmp.MetricName = tsA.MetricName
+		tmp.Values = make([]float64, 0, len(tmp.Timestamps))
+		tmp.MetricName.MoveFrom(&tsA.MetricName)
 		tmp.Values = append(tmp.Values, tsA.Values...)
 
 		tStart := bStart

app/vmselect/promql/timeseries.go

@@ -320,26 +320,16 @@ func marshalMetricTagsFast(dst []byte, tags []storage.Tag) []byte {
 
 func marshalMetricNameSorted(dst []byte, mn *storage.MetricName) []byte {
 	dst = marshalBytesFast(dst, mn.MetricGroup)
-	sortMetricTags(mn.Tags)
+	sortMetricTags(mn)
 	dst = marshalMetricTagsFast(dst, mn.Tags)
 	return dst
 }
 
 func marshalMetricTagsSorted(dst []byte, mn *storage.MetricName) []byte {
-	sortMetricTags(mn.Tags)
+	sortMetricTags(mn)
 	return marshalMetricTagsFast(dst, mn.Tags)
 }
 
-func sortMetricTags(tags []storage.Tag) {
-	less := func(i, j int) bool {
-		return string(tags[i].Key) < string(tags[j].Key)
-	}
-	if sort.SliceIsSorted(tags, less) {
-		return
-	}
-	sort.Slice(tags, less)
-}
-
 func marshalBytesFast(dst []byte, s []byte) []byte {
 	dst = encoding.MarshalUint16(dst, uint16(len(s)))
 	dst = append(dst, s...)
@@ -361,14 +351,14 @@ func unmarshalBytesFast(src []byte) ([]byte, []byte, error) {
 func stringMetricName(mn *storage.MetricName) string {
 	var dst []byte
 	dst = append(dst, mn.MetricGroup...)
-	sortMetricTags(mn.Tags)
+	sortMetricTags(mn)
 	dst = appendStringMetricTags(dst, mn.Tags)
 	return string(dst)
 }
 
 func stringMetricTags(mn *storage.MetricName) string {
 	var dst []byte
-	sortMetricTags(mn.Tags)
+	sortMetricTags(mn)
 	dst = appendStringMetricTags(dst, mn.Tags)
 	return string(dst)
 }
@@ -428,3 +418,26 @@ func assertIdenticalTimestamps(tss []*timeseries, step int64) {
 		}
 	}
 }
+
+func sortMetricTags(mn *storage.MetricName) {
+	mts := (*metricTagsSorter)(mn)
+	if !sort.IsSorted(mts) {
+		sort.Sort(mts)
+	}
+}
+
+type metricTagsSorter storage.MetricName
+
+func (mts *metricTagsSorter) Len() int {
+	return len(mts.Tags)
+}
+
+func (mts *metricTagsSorter) Less(i, j int) bool {
+	a := mts.Tags
+	return string(a[i].Key) < string(a[j].Key)
+}
+
+func (mts *metricTagsSorter) Swap(i, j int) {
+	a := mts.Tags
+	a[i], a[j] = a[j], a[i]
+}

app/vmselect/promql/transform.go

@@ -411,7 +411,8 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 		mn.CopyFrom(&ts.MetricName)
 		mn.RemoveTag("le")
 		b = marshalMetricNameSorted(b[:0], &mn)
-		m[string(b)] = append(m[string(b)], x{
+		k := bytesutil.InternBytes(b)
+		m[k] = append(m[k], x{
 			le: le,
 			ts: ts,
 		})
@@ -513,7 +514,8 @@ func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
 		ts.MetricName.RemoveTag("le")
 		ts.MetricName.RemoveTag("vmrange")
 		bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-		m[string(bb.B)] = append(m[string(bb.B)], x{
+		k := bytesutil.InternBytes(bb.B)
+		m[k] = append(m[k], x{
 			startStr: startStr,
 			endStr:   endStr,
 			start:    start,
@@ -1002,7 +1004,8 @@ func groupLeTimeseries(tss []*timeseries) map[string][]leTimeseries {
 		ts.MetricName.ResetMetricGroup()
 		ts.MetricName.RemoveTag("le")
 		bb.B = marshalMetricTagsSorted(bb.B[:0], &ts.MetricName)
-		m[string(bb.B)] = append(m[string(bb.B)], leTimeseries{
+		k := bytesutil.InternBytes(bb.B)
+		m[k] = append(m[k], leTimeseries{
 			le: le,
 			ts: ts,
 		})
@@ -1532,10 +1535,11 @@ func transformUnion(tfa *transformFuncArg) ([]*timeseries, error) {
 	for _, arg := range args {
 		for _, ts := range arg {
 			bb.B = marshalMetricNameSorted(bb.B[:0], &ts.MetricName)
-			if m[string(bb.B)] {
+			k := bytesutil.InternBytes(bb.B)
+			if m[k] {
 				continue
 			}
-			m[string(bb.B)] = true
+			m[k] = true
 			rvs = append(rvs, ts)
 		}
 	}
@@ -1873,12 +1877,12 @@ func labelReplace(tss []*timeseries, srcLabel string, r *regexp.Regexp, dstLabel
 	replacementBytes := []byte(replacement)
 	for _, ts := range tss {
 		mn := &ts.MetricName
-		dstValue := getDstValue(mn, dstLabel)
 		srcValue := mn.GetTagValue(srcLabel)
 		if !r.Match(srcValue) {
 			continue
 		}
 		b := r.ReplaceAll(srcValue, replacementBytes)
+		dstValue := getDstValue(mn, dstLabel)
 		*dstValue = append((*dstValue)[:0], b...)
 		if len(b) == 0 {
 			mn.RemoveTag(dstLabel)

app/vmselect/searchutils/searchutils.go

@@ -48,30 +48,18 @@ func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) {
 	if len(argValue) == 0 {
 		return roundToSeconds(defaultMs), nil
 	}
-	secs, err := strconv.ParseFloat(argValue, 64)
+	// Handle Prometheus'-provided minTime and maxTime.
+	// See https://github.com/prometheus/client_golang/issues/614
+	switch argValue {
+	case prometheusMinTimeFormatted:
+		return minTimeMsecs, nil
+	case prometheusMaxTimeFormatted:
+		return maxTimeMsecs, nil
+	}
+	// Parse argValue
+	secs, err := parseTime(argValue)
 	if err != nil {
-		// Try parsing string format
-		t, err := time.Parse(time.RFC3339, argValue)
-		if err != nil {
-			// Handle Prometheus'-provided minTime and maxTime.
-			// See https://github.com/prometheus/client_golang/issues/614
-			switch argValue {
-			case prometheusMinTimeFormatted:
-				return minTimeMsecs, nil
-			case prometheusMaxTimeFormatted:
-				return maxTimeMsecs, nil
-			}
-			// Try parsing duration relative to the current time
-			d, err1 := promutils.ParseDuration(argValue)
-			if err1 != nil {
-				return 0, fmt.Errorf("cannot parse %q=%q: %w", argKey, argValue, err)
-			}
-			if d > 0 {
-				d = -d
-			}
-			t = time.Now().Add(d)
-		}
-		secs = float64(t.UnixNano()) / 1e9
+		return 0, fmt.Errorf("cannot parse %s=%s: %w", argKey, argValue, err)
 	}
 	msecs := int64(secs * 1e3)
 	if msecs < minTimeMsecs {
@@ -83,6 +71,78 @@ func GetTime(r *http.Request, argKey string, defaultMs int64) (int64, error) {
 	return msecs, nil
 }
 
+func parseTime(s string) (float64, error) {
+	if len(s) > 0 && (s[len(s)-1] != 'Z' && s[len(s)-1] > '9' || s[0] == '-') {
+		// Parse duration relative to the current time
+		d, err := promutils.ParseDuration(s)
+		if err != nil {
+			return 0, err
+		}
+		if d > 0 {
+			d = -d
+		}
+		t := time.Now().Add(d)
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 4 {
+		// Parse YYYY
+		t, err := time.Parse("2006", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if !strings.Contains(s, "-") {
+		// Parse the timestamp in milliseconds
+		return strconv.ParseFloat(s, 64)
+	}
+	if len(s) == 7 {
+		// Parse YYYY-MM
+		t, err := time.Parse("2006-01", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 10 {
+		// Parse YYYY-MM-DD
+		t, err := time.Parse("2006-01-02", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 13 {
+		// Parse YYYY-MM-DDTHH
+		t, err := time.Parse("2006-01-02T15", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 16 {
+		// Parse YYYY-MM-DDTHH:MM
+		t, err := time.Parse("2006-01-02T15:04", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	if len(s) == 19 {
+		// Parse YYYY-MM-DDTHH:MM:SS
+		t, err := time.Parse("2006-01-02T15:04:05", s)
+		if err != nil {
+			return 0, err
+		}
+		return float64(t.UnixNano()) / 1e9, nil
+	}
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		return 0, err
+	}
+	return float64(t.UnixNano()) / 1e9, nil
+}
+
 var (
 	// These constants were obtained from https://github.com/prometheus/prometheus/blob/91d7175eaac18b00e370965f3a8186cc40bf9f55/web/api/v1/api.go#L442
 	// See https://github.com/prometheus/client_golang/issues/614 for details.

app/vmselect/searchutils/searchutils_test.go

@@ -11,6 +11,69 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
+func TestGetDurationSuccess(t *testing.T) {
+	f := func(s string, dExpected int64) {
+		t.Helper()
+		urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
+		r, err := http.NewRequest("GET", urlStr, nil)
+		if err != nil {
+			t.Fatalf("unexpected error in NewRequest: %s", err)
+		}
+
+		// Verify defaultValue
+		d, err := GetDuration(r, "foo", 123456)
+		if err != nil {
+			t.Fatalf("unexpected error when obtaining default time from GetDuration(%q): %s", s, err)
+		}
+		if d != 123456 {
+			t.Fatalf("unexpected default value for GetDuration(%q); got %d; want %d", s, d, 123456)
+		}
+
+		// Verify dExpected
+		d, err = GetDuration(r, "s", 123)
+		if err != nil {
+			t.Fatalf("unexpected error in GetDuration(%q): %s", s, err)
+		}
+		if d != dExpected {
+			t.Fatalf("unexpected timestamp for GetDuration(%q); got %d; want %d", s, d, dExpected)
+		}
+	}
+
+	f("1.234", 1234)
+	f("1.23ms", 1)
+	f("1.23s", 1230)
+	f("2s56ms", 2056)
+	f("2s-5ms", 1995)
+	f("5m3.5s", 303500)
+	f("2h", 7200000)
+	f("1d", 24*3600*1000)
+	f("7d5h4m3s534ms", 623043534)
+}
+
+func TestGetDurationError(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		urlStr := fmt.Sprintf("http://foo.bar/baz?s=%s", url.QueryEscape(s))
+		r, err := http.NewRequest("GET", urlStr, nil)
+		if err != nil {
+			t.Fatalf("unexpected error in NewRequest: %s", err)
+		}
+
+		if _, err := GetDuration(r, "s", 123); err == nil {
+			t.Fatalf("expecting non-nil error in GetDuration(%q)", s)
+		}
+	}
+
+	// Negative durations aren't supported
+	f("-1.234")
+
+	// Invalid duration
+	f("foo")
+
+	// Invalid suffix
+	f("1md")
+}
+
 func TestGetTimeSuccess(t *testing.T) {
 	f := func(s string, timestampExpected int64) {
 		t.Helper()
@@ -39,6 +102,17 @@ func TestGetTimeSuccess(t *testing.T) {
 		}
 	}
 
+	f("2019", 1546300800000)
+	f("2019-01", 1546300800000)
+	f("2019-02", 1548979200000)
+	f("2019-02-01", 1548979200000)
+	f("2019-02-02", 1549065600000)
+	f("2019-02-02T00", 1549065600000)
+	f("2019-02-02T01", 1549069200000)
+	f("2019-02-02T01:00", 1549069200000)
+	f("2019-02-02T01:01", 1549069260000)
+	f("2019-02-02T01:01:00", 1549069260000)
+	f("2019-02-02T01:01:01", 1549069261000)
 	f("2019-07-07T20:01:02Z", 1562529662000)
 	f("2019-07-07T20:47:40+03:00", 1562521660000)
 	f("-292273086-05-16T16:47:06Z", minTimeMsecs)
@@ -58,27 +132,26 @@ func TestGetTimeError(t *testing.T) {
 			t.Fatalf("unexpected error in NewRequest: %s", err)
 		}
 
-		// Verify defaultValue
-		ts, err := GetTime(r, "foo", 123456)
-		if err != nil {
-			t.Fatalf("unexpected error when obtaining default time from GetTime(%q): %s", s, err)
-		}
-		if ts != 123000 {
-			t.Fatalf("unexpected default value for GetTime(%q); got %d; want %d", s, ts, 123000)
-		}
-
-		// Verify timestampExpected
-		_, err = GetTime(r, "s", 123)
-		if err == nil {
+		if _, err := GetTime(r, "s", 123); err == nil {
 			t.Fatalf("expecting non-nil error in GetTime(%q)", s)
 		}
 	}
 
 	f("foo")
+	f("foo1")
+	f("1245-5")
+	f("2022-x7")
+	f("2022-02-x7")
+	f("2022-02-02Tx7")
+	f("2022-02-02T00:x7")
+	f("2022-02-02T00:00:x7")
+	f("2022-02-02T00:00:00a")
 	f("2019-07-07T20:01:02Zisdf")
 	f("2019-07-07T20:47:40+03:00123")
 	f("-292273086-05-16T16:47:07Z")
 	f("292277025-08-18T07:12:54.999999998Z")
+	f("123md")
+	f("-12.3md")
 }
 
 func TestGetExtraTagFilters(t *testing.T) {