docs: add vmanomaly docs about licensing

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

.github/dependabot.yml

@@ -8,10 +8,12 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    open-pull-requests-limit: 0
   - package-ecosystem: "bundler"
     directory: "/docs"
     schedule:
-      interval: "daily"
+      interval: "weekly"
+    open-pull-requests-limit: 0
   - package-ecosystem: "gomod"
     directory: "/app/vmui/packages/vmui/web"
     schedule:

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.20.4
+          go-version: 1.20.7
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/codeql-analysis.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.4
+          go-version: 1.20.7
           check-latest: true
           cache: true
         if: ${{ matrix.language == 'go' }}

.github/workflows/main.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.4
+          go-version: 1.20.7
           check-latest: true
           cache: true
 
@@ -56,7 +56,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.4
+          go-version: 1.20.7
           check-latest: true
           cache: true
 
@@ -81,7 +81,7 @@ jobs:
         id: go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.4
+          go-version: 1.20.7
           check-latest: true
           cache: true
 

.github/workflows/nightly-build.yml

@@ -1,48 +0,0 @@
-name: nightly-build
-on:
-  schedule:
-    # Daily at 2:48am
-    - cron: '48 2 * * *'
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Setup Go
-        uses: actions/setup-go@main
-        with:
-          go-version: 1.20.4
-        id: go
-
-      - name: Setup docker scan
-        run: |
-          mkdir -p ~/.docker/cli-plugins && \
-          curl https://github.com/docker/scan-cli-plugin/releases/latest/download/docker-scan_linux_amd64 -L -s -S -o ~/.docker/cli-plugins/docker-scan &&\
-          chmod +x ~/.docker/cli-plugins/docker-scan
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      
-      - name: Code checkout
-        uses: actions/checkout@master
-
-      - uses: actions/cache@v3
-        with:
-          path: gocache-for-docker
-          key: gocache-docker-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}
-
-      - name: build & publish
-        run: |
-          docker scan --severity=medium --login --token "$SNYK_TOKEN" --accept-license
-          LATEST_TAG=nightly PKG_TAG=nightly make publish
-        env: 
-          SNYK_TOKEN: ${{ secrets.SNYK_AUTH_TOKEN }}

.github/workflows/sync-docs.yml

@@ -0,0 +1,53 @@
+name: publish-docs
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'docs/**'
+  workflow_dispatch: {}
+permissions:
+  contents: read  # This is required for actions/checkout and to commit back image update
+  deployments: write
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+        with:
+          path: main
+      - name: Checkout private code
+        uses: actions/checkout@v3
+        with:
+          repository: VictoriaMetrics/vmdocs
+          token: ${{ secrets.VM_BOT_GH_TOKEN }}
+          path: docs
+
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@v5
+        with:
+          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+          workdir: docs
+      - name: Set short git commit SHA
+        id: vars
+        run: |
+          calculatedSha=$(git rev-parse --short ${{ github.sha }})
+          echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
+        working-directory: main
+
+      - name: update code and commit
+        run: |
+          rm -rf content
+          cp -r ../main/docs content
+          make clean-after-copy
+          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
+          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
+          git add .
+          git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.vars.outputs.short_sha }}"
+          git push
+        working-directory: docs

.github/workflows/update-sandbox.yml

@@ -0,0 +1,80 @@
+name: sandbox-release
+on:
+  release:
+    types: [published]
+permissions:
+  contents: write
+jobs:
+  deploy-sandbox:
+    runs-on: ubuntu-latest
+    steps:
+      - name: check inputs
+        if: github.event.release.tag_name == ''
+        run: exit 1
+    
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          repository: VictoriaMetrics/ops
+          token: ${{ secrets.VM_BOT_GH_TOKEN }}
+
+      - name: Import GPG key
+        id: import-gpg
+        uses: crazy-max/ghaction-import-gpg@v5
+        with:
+          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+
+      - name: update image tag
+        uses: fjogeleit/yaml-update-action@main
+        with:
+          valueFile: 'gcp-test/sandbox/manifests/benchmark-vm/vmcluster.yaml'
+          commitChange: false
+          createPR: false
+          changes: |
+            {
+              "gcp-test/sandbox/manifests/benchmark-vm/vmcluster.yaml": {
+                "spec.vminsert.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster",
+                "spec.vmselect.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster",
+                "spec.vmstorage.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster"
+              },
+              "gcp-test/sandbox/manifests/benchmark-vm/vmsingle.yaml": {
+                "spec.image.tag": "${{ github.event.release.tag_name }}-enterprise"
+              },
+              "gcp-test/sandbox/manifests/monitoring/monitoring-vmagent.yaml": {
+                "spec.image.tag": "${{ github.event.release.tag_name }}"
+              },
+              "gcp-test/sandbox/manifests/monitoring/monitoring-vmcluster.yaml": {
+                "spec.vminsert.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster",
+                "spec.vmselect.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster",
+                "spec.vmstorage.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster"
+              },
+              "gcp-test/sandbox/manifests/monitoring/vmalert.yaml": {
+                "spec.image.tag": "${{ github.event.release.tag_name }}-enterprise"
+              }
+            }
+
+      - name: commit changes
+        run: |
+          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
+          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
+          git add .
+          git commit -S -m "Deploy image tag ${RELEASE_TAG} to sandbox"
+        env:
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          branch: release-automation
+          token: ${{ secrets.VM_BOT_GH_TOKEN }}
+          delete-branch: true
+          title: "release ${{ github.event.release.tag_name }}"
+          body: |
+            Release [${{ github.event.release.tag_name }}](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/${{ github.event.release.tag_name }}) to sandbox
+        
+            > Auto-generated by `Github Actions Bot`
+    

.gitignore

@@ -8,6 +8,7 @@
 *.test
 *.swp
 /gocache-for-docker
+/victoria-logs-data
 /victoria-metrics-data
 /vmagent-remotewrite-data
 /vmstorage-data
@@ -20,4 +21,4 @@
 Gemfile.lock
 /_site
 _site
-*.tmp
+*.tmp

Makefile

@@ -21,6 +21,7 @@ include package/release/Makefile
 
 all: \
 	victoria-metrics-prod \
+	victoria-logs-prod \
 	vmagent-prod \
 	vmalert-prod \
 	vmauth-prod \
@@ -31,8 +32,9 @@ all: \
 clean:
 	rm -rf bin/*
 
-publish: docker-scan \
+publish: package-base \
 	publish-victoria-metrics \
+	publish-victoria-logs \
 	publish-vmagent \
 	publish-vmalert \
 	publish-vmauth \
@@ -42,6 +44,7 @@ publish: docker-scan \
 
 package: \
 	package-victoria-metrics \
+	package-victoria-logs \
 	package-vmagent \
 	package-vmalert \
 	package-vmauth \
@@ -178,6 +181,7 @@ publish-release:
 
 release: \
 	release-victoria-metrics \
+	release-victoria-logs \
 	release-vmutils
 
 release-victoria-metrics: \
@@ -191,7 +195,6 @@ release-victoria-metrics: \
 	release-victoria-metrics-openbsd-amd64 \
 	release-victoria-metrics-windows-amd64
 
-# adds i386 arch
 release-victoria-metrics-linux-386:
 	GOOS=linux GOARCH=386 $(MAKE) release-victoria-metrics-goos-goarch
 
@@ -238,6 +241,63 @@ release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
 	cd bin && rm -rf \
 		victoria-metrics-windows-$(GOARCH)-prod.exe
 
+release-victoria-logs: \
+	release-victoria-logs-linux-386 \
+	release-victoria-logs-linux-amd64 \
+	release-victoria-logs-linux-arm \
+	release-victoria-logs-linux-arm64 \
+	release-victoria-logs-darwin-amd64 \
+	release-victoria-logs-darwin-arm64 \
+	release-victoria-logs-freebsd-amd64 \
+	release-victoria-logs-openbsd-amd64 \
+	release-victoria-logs-windows-amd64
+
+release-victoria-logs-linux-386:
+	GOOS=linux GOARCH=386 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-linux-arm:
+	GOOS=linux GOARCH=arm $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-darwin-amd64:
+	GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-darwin-arm64:
+	GOOS=darwin GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-freebsd-amd64:
+	GOOS=freebsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-openbsd-amd64:
+	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
+
+release-victoria-logs-windows-amd64:
+	GOARCH=amd64 $(MAKE) release-victoria-logs-windows-goarch
+
+release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
+	cd bin && \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-logs-$(GOOS)-$(GOARCH)-prod \
+		&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+			victoria-logs-$(GOOS)-$(GOARCH)-prod \
+			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf victoria-logs-$(GOOS)-$(GOARCH)-prod
+
+release-victoria-logs-windows-goarch: victoria-logs-windows-$(GOARCH)-prod
+	cd bin && \
+		zip victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
+			victoria-logs-windows-$(GOARCH)-prod.exe \
+		&& sha256sum victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
+			victoria-logs-windows-$(GOARCH)-prod.exe \
+			> victoria-logs-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
+	cd bin && rm -rf \
+		victoria-logs-windows-$(GOARCH)-prod.exe
+
 release-vmutils: \
 	release-vmutils-linux-386 \
 	release-vmutils-linux-amd64 \
@@ -418,16 +478,18 @@ check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
 
 copy-docs:
+# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
+# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
 	echo "---" > ${DST}
 	@if [ ${ORDER} -ne 0 ]; then \
 		echo "sort: ${ORDER}" >> ${DST}; \
 		echo "weight: ${ORDER}" >> ${DST}; \
-		echo "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}" >> ${DST}; \
+		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
 	fi
 
 	echo "title: ${TITLE}" >> ${DST}
 	@if [ ${OLD_URL} ]; then \
-		echo "aliases:\n  - ${OLD_URL}" >> ${DST}; \
+		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
 	fi
 	echo "---" >> ${DST}
 	cat ${SRC} >> ${DST}
@@ -437,7 +499,7 @@ copy-docs:
 # Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
 # For ORDER=0 it adds no order tag/weight tag.
 # FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
-# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links. 
+# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
 # Images starting with <img src="docs/ are replaced with <img src="
 # Cluster docs are supposed to be ordered as 2nd.
 # The rest of docs is ordered manually.

README.md

@@ -16,14 +16,16 @@ VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and t
 VictoriaMetrics is available in [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases),
 [Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Snap packages](https://snapcraft.io/victoriametrics)
 and [source code](https://github.com/VictoriaMetrics/VictoriaMetrics). 
-Just download [the latest version of VictoriaMetrics](https://docs.victoriametrics.com/CHANGELOG.html)
-and follow [these instructions](https://docs.victoriametrics.com/Quick-Start.html).
 
 The cluster version of VictoriaMetrics is available [here](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
 
 Learn more about [key concepts](https://docs.victoriametrics.com/keyConcepts.html) of VictoriaMetrics and follow the 
 [quick start guide](https://docs.victoriametrics.com/Quick-Start.html) for a better experience.
 
+There is also user-friendly database for logs - [VictoriaLogs](https://docs.victoriametrics.com/VictoriaLogs/).
+
+If you have questions about VictoriaMetrics, then feel free asking them at [VictoriaMetrics community Slack chat](https://slack.victoriametrics.com/).
+
 [Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics. 
 See [features available in enterprise package](https://docs.victoriametrics.com/enterprise.html).
 Enterprise binaries can be downloaded and evaluated for free 
@@ -84,6 +86,7 @@ VictoriaMetrics has the following prominent features:
   * [Arbitrary CSV data](#how-to-import-csv-data).
   * [Native binary format](#how-to-import-data-in-native-format).
   * [DataDog agent or DogStatsD](#how-to-send-data-from-datadog-agent).
+  * [OpenTelemetry metrics format](#sending-data-via-opentelemetry).
 * It supports powerful [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html), which can be used as a [statsd](https://github.com/statsd/statsd) alternative.
 * It supports metrics [relabeling](#relabeling).
 * It can deal with [high cardinality issues](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) and
@@ -114,8 +117,10 @@ Case studies:
 * [Groove X](https://docs.victoriametrics.com/CaseStudies.html#groove-x)
 * [Idealo.de](https://docs.victoriametrics.com/CaseStudies.html#idealode)
 * [MHI Vestas Offshore Wind](https://docs.victoriametrics.com/CaseStudies.html#mhi-vestas-offshore-wind)
+* [Naver](https://docs.victoriametrics.com/CaseStudies.html#naver)
 * [Razorpay](https://docs.victoriametrics.com/CaseStudies.html#razorpay)
 * [Percona](https://docs.victoriametrics.com/CaseStudies.html#percona)
+* [Roblox](https://docs.victoriametrics.com/CaseStudies.html#roblox)
 * [Sensedia](https://docs.victoriametrics.com/CaseStudies.html#sensedia)
 * [Smarkets](https://docs.victoriametrics.com/CaseStudies.html#smarkets)
 * [Synthesio](https://docs.victoriametrics.com/CaseStudies.html#synthesio)
@@ -147,7 +152,7 @@ VictoriaMetrics can also be installed via these installation methods:
 The following command-line flags are used the most:
 
 * `-storageDataPath` - VictoriaMetrics stores all the data in this directory. Default path is `victoria-metrics-data` in the current working directory.
-* `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 1 month. See [the Retention section](#retention) for more details.
+* `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 1 month. The minimum retention period is 24h or 1d. See [the Retention section](#retention) for more details.
 
 Other flags have good enough default values, so set them only if you really need this. Pass `-help` to see [all the available flags with description and default values](#list-of-command-line-flags).
 
@@ -313,6 +318,7 @@ The UI allows exploring query results via graphs and tables. It also provides th
   - [Metrics explorer](#metrics-explorer) - automatically builds graphs for selected metrics; 
   - [Cardinality explorer](#cardinality-explorer) - stats about existing metrics in TSDB;
   - [Top queries](#top-queries) - shows most frequently executed queries;
+  - [Active queries](#active-queries) - shows currently executed queries;
 - Tools:
   - [Trace analyzer](#query-tracing) - playground for loading query traces in JSON format; 
   - [WITH expressions playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/expand-with-exprs) - test how WITH expressions work; 
@@ -357,15 +363,24 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric
 * queries with the biggest average execution duration;
 * queries that took the most summary time for execution.
 
+## Active queries
+
+[VMUI](#vmui) provides `active queries` tab, which shows currently execute queries.
+It provides the following information per each query:
+
+- The query itself, together with the time range and step args passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query).
+- The duration of the query execution.
+- The client address, who initiated the query execution.
+
 ## Metrics explorer
 
 [VMUI](#vmui) provides an ability to explore metrics exported by a particular `job` / `instance` in the following way:
 
 1. Open the `vmui` at `http://victoriametrics:8428/vmui/`.
-2. Click the `Explore metrics` tab.
-3. Select the `job` you want to explore.
-4. Optionally select the `instance` for the selected job to explore.
-5. Select metrics you want to explore and compare.
+1. Click the `Explore metrics` tab.
+1. Select the `job` you want to explore.
+1. Optionally select the `instance` for the selected job to explore.
+1. Select metrics you want to explore and compare.
 
 It is possible to change the selected time range for the graphs in the top right corner.
 
@@ -388,6 +403,11 @@ matching the specified [series selector](https://prometheus.io/docs/prometheus/l
 
 Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
 
+In [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) each vmstorage tracks the stored time series individually.
+vmselect requests stats via [/api/v1/status/tsdb](#tsdb-stats) API from each vmstorage node and merges the results by summing per-series stats.
+This may lead to inflated values when samples for the same time series are spread across multiple vmstorage nodes
+due to [replication](#replication) or [rerouting](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html?highlight=re-routes#cluster-availability).
+
 See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).
 See the example of using the cardinality explorer [here](https://victoriametrics.com/blog/cardinality-explorer/).
 
@@ -403,7 +423,7 @@ Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](
 
 ## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
 
-VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets.
+VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. If the provided configuration file contains [unsupported options](https://docs.victoriametrics.com/vmagent.html#unsupported-prometheus-config-sections), then either delete them from the file or just pass `-promscrape.config.strictParse=false` command-line flag to VictoriaMetrics, so it will ignore unsupported options.
 
 The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 
@@ -440,7 +460,7 @@ DD_DD_URL=http://victoriametrics:8428/datadog
 
 _Choose correct URL for VictoriaMetrics [here](https://docs.victoriametrics.com/url-examples.html#datadog)._
 
-To configure DataDog agent via [configuration file](https://docs.datadoghq.com/agent/guide/agent-configuration-files)
+To configure DataDog agent via [configuration file](https://github.com/DataDog/datadog-agent/blob/878600ef7a55c5ef0efb41ed0915f020cf7e3bd0/pkg/config/config_template.yaml#L33)
 add the following line:
 
 <div class="with-copy" markdown="1">
@@ -468,7 +488,8 @@ Run DataDog using the following ENV variable with VictoriaMetrics as additional
 <div class="with-copy" markdown="1">
 
 ```
-DD_ADDITIONAL_ENDPOINTS='{\"http://victoriametrics:8428/datadog\"}'
+DD_ADDITIONAL_ENDPOINTS='{\"http://victoriametrics:8428/datadog\": [\"apikey\"]}'
+
 ```
 
 </div>
@@ -482,7 +503,9 @@ add the following line:
 <div class="with-copy" markdown="1">
 
 ```
-additional_endpoints: http://victoriametrics:8428/datadog
+additional_endpoints:
+  "http://victoriametrics:8428/datadog":
+  - apikey
 ```
 
 </div>
@@ -823,6 +846,7 @@ in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage)
 in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series).
 
 - Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`.
+- Unix timestamps in milliseconds. For example, `1562529662678`.
 - [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, `2022-03-29T01:02:03Z` or `2022-03-29T01:02:03+02:30`.
 - Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`, `2022-03-29T01:02:03`.
   The partial RFC3339 time is in UTC timezone by default. It is possible to specify timezone there by adding `+hh:mm` or `-hh:mm` suffix to partial time.
@@ -891,13 +915,13 @@ to your needs or when testing bugfixes.
 ### Development build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
-2. Run `make victoria-metrics` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make victoria-metrics` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics` binary and puts it into the `bin` folder.
 
 ### Production build
 
 1. [Install docker](https://docs.docker.com/install/).
-2. Run `make victoria-metrics-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make victoria-metrics-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics-prod` binary and puts it into the `bin` folder.
 
 ### ARM build
@@ -907,13 +931,13 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 ### Development ARM build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
-2. Run `make victoria-metrics-linux-arm` or `make victoria-metrics-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make victoria-metrics-linux-arm` or `make victoria-metrics-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics-linux-arm` or `victoria-metrics-linux-arm64` binary respectively and puts it into the `bin` folder.
 
 ### Production ARM build
 
 1. [Install docker](https://docs.docker.com/install/).
-2. Run `make victoria-metrics-linux-arm-prod` or `make victoria-metrics-linux-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make victoria-metrics-linux-arm-prod` or `make victoria-metrics-linux-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics-linux-arm-prod` or `victoria-metrics-linux-arm64-prod` binary respectively and puts it into the `bin` folder.
 
 ### Pure Go build (CGO_ENABLED=0)
@@ -921,7 +945,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 `Pure Go` mode builds only Go code without [cgo](https://golang.org/cmd/cgo/) dependencies.
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
-2. Run `make victoria-metrics-pure` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make victoria-metrics-pure` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics-pure` binary and puts it into the `bin` folder.
 
 ### Building docker images
@@ -974,9 +998,9 @@ Navigate to `http://<victoriametrics-addr>:8428/snapshot/delete_all` in order to
 Steps for restoring from a snapshot:
 
 1. Stop VictoriaMetrics with `kill -INT`.
-2. Restore snapshot contents from backup with [vmrestore](https://docs.victoriametrics.com/vmrestore.html)
+1. Restore snapshot contents from backup with [vmrestore](https://docs.victoriametrics.com/vmrestore.html)
    to the directory pointed by `-storageDataPath`.
-3. Start VictoriaMetrics.
+1. Start VictoriaMetrics.
 
 ## How to delete time series
 
@@ -1041,7 +1065,7 @@ VictoriaMetrics provides the following handlers for exporting data:
 Send a request to `http://<victoriametrics-addr>:8428/api/v1/export?match[]=<timeseries_selector_for_export>`,
 where `<timeseries_selector_for_export>` may contain any [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors)
 for metrics to export. Use `{__name__!=""}` selector for fetching all the time series.
-The response would contain all the data for the selected time series in [JSON streaming format](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON).
+The response would contain all the data for the selected time series in [JSON streaming format](http://ndjson.org/).
 Each JSON line contains samples for a single time series. An example output:
 
 ```json
@@ -1151,6 +1175,7 @@ Additionally, VictoriaMetrics can accept metrics via the following popular data
 * DataDog `submit metrics` API. See [these docs](#how-to-send-data-from-datadog-agent) for details.
 * InfluxDB line protocol. See [these docs](#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) for details.
 * Graphite plaintext protocol. See [these docs](#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) for details.
+* OpenTelemetry http API. See [these docs](#sending-data-via-opentelemetry) for details.
 * OpenTSDB telnet put protocol. See [these docs](#sending-data-via-telnet-put-protocol) for details.
 * OpenTSDB http `/api/put` protocol. See [these docs](#sending-opentsdb-data-via-http-apiput-requests) for details.
 * `/api/v1/import` for importing data obtained from [/api/v1/export](#how-to-export-data-in-json-line-format).
@@ -1161,6 +1186,13 @@ Additionally, VictoriaMetrics can accept metrics via the following popular data
 * `/api/v1/import/prometheus` for importing data in Prometheus exposition format and in [Pushgateway format](https://github.com/prometheus/pushgateway#url).
   See [these docs](#how-to-import-data-in-prometheus-exposition-format) for details.
 
+Please note, most of the ingestion APIs (except [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write))
+are optimized for performance and processes data in a streaming fashion.
+It means that client can transfer unlimited amount of data through the open connection. Because of this, import APIs
+may not return parsing errors to the client, as it is expected for data stream to be not interrupted. 
+Instead, look for parsing errors on the server side (VictoriaMetrics single-node or vminsert) or
+check for changes in `vm_rows_invalid_total` (exported by server side) metric.
+
 ### How to import data in JSON line format
 
 Example for importing data obtained via [/api/v1/export](#how-to-export-data-in-json-line-format):
@@ -1327,6 +1359,13 @@ Note that it could be required to flush response cache after importing historica
 
 VictoriaMetrics also may scrape Prometheus targets - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
 
+## Sending data via OpenTelemetry
+
+VictoriaMetrics supports data ingestion via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/ffddc289462dfe0c2041e3ca42a7b1df805706de/specification/metrics/data-model.md) at `/opentelemetry/api/v1/push` path.
+
+VictoriaMetrics expects `protobuf`-encoded requests at `/opentelemetry/api/v1/push`.
+Set HTTP request header `Content-Encoding: gzip` when sending gzip-compressed data to `/opentelemetry/api/v1/push`.
+
 ## Relabeling
 
 VictoriaMetrics supports Prometheus-compatible relabeling for all the ingested metrics if `-relabelConfig` command-line flag points
@@ -1544,7 +1583,13 @@ occurs in the middle of writing the `part` to disk - such incompletely written `
 are automatically deleted on the next VictoriaMetrics start.
 
 The same applies to merge process — `parts` are either fully merged into a new `part` or fail to merge,
-leaving the source `parts` untouched.
+leaving the source `parts` untouched. However, due to hardware issues data on disk may be corrupted regardless of
+VictoriaMetrics process. VictoriaMetrics can detect corruption during decompressing, decoding or sanity checking
+of the data blocks. But **it cannot fix the corrupted data**. Data parts that fail to load on startup need to be deleted
+or restored from backups. This is why it is recommended performing
+[regular backups](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#backups).
+
+VictoriaMetrics doesn't use checksums for stored data blocks. See why [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3011).
 
 VictoriaMetrics doesn't merge parts if their summary size exceeds free disk space.
 This prevents from potential out of disk space errors during merge.
@@ -1563,19 +1608,22 @@ See also [how to work with snapshots](#how-to-work-with-snapshots).
 
 ## Retention
 
-Retention is configured with the `-retentionPeriod` command-line flag, which takes a number followed by a time unit character - `h(ours)`, `d(ays)`, `w(eeks)`, `y(ears)`. If the time unit is not specified, a month is assumed. For instance, `-retentionPeriod=3` means that the data will be stored for 3 months and then deleted. The default retention period is one month.
+Retention is configured with the `-retentionPeriod` command-line flag, which takes a number followed by a time unit 
+character - `h(ours)`, `d(ays)`, `w(eeks)`, `y(ears)`. If the time unit is not specified, a month is assumed. 
+For instance, `-retentionPeriod=3` means that the data will be stored for 3 months and then deleted. 
+The default retention period is one month. The **minimum retention** period is 24h or 1d.
 
 Data is split in per-month partitions inside `<-storageDataPath>/data/{small,big}` folders.
-Data partitions outside the configured retention are deleted on the first day of the new month.
-Each partition consists of one or more data parts. Data parts outside the configured retention are eventually deleted during
-[background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+**Data partitions** outside the configured retention are deleted **on the first day of the new month**.
+Each partition consists of one or more **data parts**. Data parts outside the configured retention 
+are **eventually deleted** during [background merge](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
+The time range covered by data part is **not limited by retention period unit**. One data part can cover hours or days of 
+data. Hence, a data part can be deleted only **when fully outside the configured retention**.
+See more about partitions and parts [here](#storage).
 
 The maximum disk space usage for a given `-retentionPeriod` is going to be (`-retentionPeriod` + 1) months.
 For example, if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
 
-Please note, the time range covered by data part is not limited by retention period unit. Hence, data part may contain data
-for multiple days and will be deleted only when fully outside the configured retention.
-
 It is safe to extend `-retentionPeriod` on existing data. If `-retentionPeriod` is set to a lower
 value than before, then data outside the configured period will be eventually deleted.
 
@@ -1638,7 +1686,8 @@ Retention filters can be evaluated for free by downloading and using enterprise
 
 * `-downsampling.period=30d:5m,180d:1h` instructs VictoriaMetrics to deduplicate samples older than 30 days with 5 minutes interval and to deduplicate samples older than 180 days with 1 hour interval.
 
-Downsampling is applied independently per each time series. It can reduce disk space usage and improve query performance if it is applied to time series with big number of samples per each series. The downsampling doesn't improve query performance if the database contains big number of time series with small number of samples per each series (aka [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate)), since downsampling doesn't reduce the number of time series. So the majority of time is spent on searching for the matching time series. It is possible to use recording rules in [vmalert](https://docs.victoriametrics.com/vmalert.html) in order to reduce the number of time series. See [these docs](https://docs.victoriametrics.com/vmalert.html#downsampling-and-aggregation-via-vmalert).
+Downsampling is applied independently per each time series. It can reduce disk space usage and improve query performance if it is applied to time series with big number of samples per each series. The downsampling doesn't improve query performance if the database contains big number of time series with small number of samples per each series (aka [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate)), since downsampling doesn't reduce the number of time series. So the majority of time is spent on searching for the matching time series. 
+It is possible to use [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) in vmagent or recording rules in [vmalert](https://docs.victoriametrics.com/vmalert.html) in order to [reduce the number of time series](https://docs.victoriametrics.com/vmalert.html#downsampling-and-aggregation-via-vmalert).
 
 Downsampling happens during [background merges](https://docs.victoriametrics.com/#storage) 
 and can't be performed if there is not enough of free disk space or if vmstorage 
@@ -1733,7 +1782,7 @@ created by community.
 
 Graphs on the dashboards contain useful hints - hover the `i` icon in the top left corner of each graph to read it.
 
-We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts.yml)
+We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
 via [vmalert](https://docs.victoriametrics.com/vmalert.html) or via Prometheus.
 
 VictoriaMetrics exposes currently running queries and their execution times at `/api/v1/status/active_queries` page.
@@ -1753,6 +1802,11 @@ VictoriaMetrics returns TSDB stats at `/api/v1/status/tsdb` page in the way simi
 * `match[]=SELECTOR` where `SELECTOR` is an arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) for series to take into account during stats calculation. By default all the series are taken into account.
 * `extra_label=LABEL=VALUE`. See [these docs](#prometheus-querying-api-enhancements) for more details.
 
+In [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) each vmstorage tracks the stored time series individually.
+vmselect requests stats via [/api/v1/status/tsdb](#tsdb-stats) API from each vmstorage node and merges the results by summing per-series stats.
+This may lead to inflated values when samples for the same time series are spread across multiple vmstorage nodes
+due to [replication](#replication) or [rerouting](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html?highlight=re-routes#cluster-availability).
+
 VictoriaMetrics provides an UI on top of `/api/v1/status/tsdb` - see [cardinality explorer docs](#cardinality-explorer).
 
 ## Query tracing
@@ -2006,16 +2060,16 @@ The simplest way to migrate data from one single-node (source) to another (desti
 to another do the following:
 
 1. Stop the VictoriaMetrics (source) with `kill -INT`;
-2. Copy (via [rsync](https://en.wikipedia.org/wiki/Rsync) or any other tool) the entire folder specified
+1. Copy (via [rsync](https://en.wikipedia.org/wiki/Rsync) or any other tool) the entire folder specified
 via `-storageDataPath` from the source node to the empty folder at the destination node.
-3. Once copy is done, stop the VictoriaMetrics (destination) with `kill -INT` and verify that
+1. Once copy is done, stop the VictoriaMetrics (destination) with `kill -INT` and verify that
 its `-storageDataPath` points to the copied folder from p.2;
-4. Start the VictoriaMetrics (destination). The copied data should be now available.
+1. Start the VictoriaMetrics (destination). The copied data should be now available.
 
 Things to consider when copying data:
 
 1. Data formats between single-node and vmstorage node aren't compatible and can't be copied.
-2. Copying data folder means complete replacement of the previous data on destination VictoriaMetrics.
+1. Copying data folder means complete replacement of the previous data on destination VictoriaMetrics.
 
 For more complex scenarios like single-to-cluster, cluster-to-single, re-sharding or migrating only a fraction
 of data - see [vmctl. Migrating data from VictoriaMetrics](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics).
@@ -2041,8 +2095,7 @@ It is recommended disabling query cache with `-search.disableCache` command-line
 historical data with timestamps from the past, since the cache assumes that the data is written with
 the current timestamps. Query cache can be enabled after the backfilling is complete.
 
-An alternative solution is to query `/internal/resetRollupResultCache` url after backfilling is complete. This will reset
-the query cache, which could contain incomplete data cached during the backfilling.
+An alternative solution is to query [/internal/resetRollupResultCache](https://docs.victoriametrics.com/url-examples.html#internalresetRollupResultCache) handler after the backfilling is complete. This will reset the query cache, which could contain incomplete data cached during the backfilling.
 
 Yet another solution is to increase `-search.cacheTimestampOffset` flag value in order to disable caching
 for data with timestamps close to the current time. Single-node VictoriaMetrics automatically resets response
@@ -2149,7 +2202,6 @@ Feel free asking any questions regarding VictoriaMetrics:
 * [reddit](https://www.reddit.com/r/VictoriaMetrics/)
 * [telegram-en](https://t.me/VictoriaMetrics_en)
 * [telegram-ru](https://t.me/VictoriaMetrics_ru1)
-* [articles and talks about VictoriaMetrics in Russian](https://github.com/denisgolius/victoriametrics-ru-links)
 * [google groups](https://groups.google.com/forum/#!forum/victorametrics-users)
 
 If you like VictoriaMetrics and want to contribute, then we need the following:
@@ -2230,14 +2282,14 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -denyQueryTracing
      Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/#query-tracing
   -downsampling.period array
-     Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. When setting multiple downsampling periods, it is necessary for the periods to be multiples of each other. See https://docs.victoriametrics.com/#downsampling for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
   -dryRun
      Whether to check config files without running VictoriaMetrics. The following config files are checked: -promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag
   -enableTCP6
-     Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP is used
+     Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable
-     Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
+     Whether to enable reading flags from environment variables in addition to the command line. Command line flag values have priority over values from environment vars. Flags are read only from the command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
@@ -2271,13 +2323,13 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -http.shutdownDelay duration
      Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-     Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+     Password for HTTP server's Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-     Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+     Username for HTTP server's Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
      TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8428")
   -httpListenAddr.useProxyProtocol
-     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
+     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing
   -import.maxLineLen size
      The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with 'max_rows_per_line' query arg passed to /api/v1/export
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 104857600)
@@ -2302,7 +2354,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -influxTrimTimestamp duration
      Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -inmemoryDataFlushInterval duration
-     The interval for guaranteed saving of in-memory data to disk. The saved data survives unclean shutdown such as OOM crash, hardware reset, SIGKILL, etc. Bigger intervals may help increasing lifetime of flash storage with limited write cycles (e.g. Raspberry PI). Smaller intervals increase disk IO load. Minimum supported value is 1s (default 5s)
+     The interval for guaranteed saving of in-memory data to disk. The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). Smaller intervals increase disk IO load. Minimum supported value is 1s (default 5s)
   -insert.maxQueueDuration duration
      The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
   -internStringCacheExpireDuration duration
@@ -2310,7 +2362,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -internStringDisableCache
      Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
+     The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
   -logNewSeries
      Whether to log new series. This option is for debug purposes only. It can lead to performance issues when big number of new series are ingested into VictoriaMetrics
   -loggerDisableTimestamps
@@ -2330,7 +2382,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -maxConcurrentInserts int
-     The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
+     The maximum number of concurrent insert requests. The default value should work for most cases, since it minimizes memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
   -maxInsertRequestSize size
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
@@ -2339,10 +2391,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -maxLabelsPerTimeseries int
      The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented (default 30)
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
-     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
@@ -2388,6 +2440,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Wait time used by Consul service discovery. Default value is used if not set
   -promscrape.consulSDCheckInterval duration
      Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs for details (default 30s)
+  -promscrape.consulagentSDCheckInterval duration
+     Interval for checking for changes in Consul Agent. This works only if consulagent_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#consulagent_sd_configs for details (default 30s)
   -promscrape.digitaloceanSDCheckInterval duration
      Interval for checking for changes in digital ocean. This works only if digitalocean_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#digitalocean_sd_configs for details (default 1m0s)
   -promscrape.disableCompression
@@ -2467,7 +2521,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
   -retentionPeriod value
-     Data with timestamps outside the retentionPeriod is automatically deleted. See also -retentionFilter
+     Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter
      The following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 1)
   -retentionTimezoneOffset duration
      The offset for performing indexdb rotation. If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)
@@ -2498,6 +2552,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
   -search.maxGraphiteSeries int
      The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
+  -search.maxGraphiteTagKeys int
+     The maximum number of tag keys returned from Graphite API, which returns tags. See https://docs.victoriametrics.com/#graphite-tags-api-usage (default 100000)
+  -search.maxGraphiteTagValues int
+     The maximum number of tag values returned from Graphite API, which returns tag values. See https://docs.victoriametrics.com/#graphite-tags-api-usage (default 100000)
   -search.maxLookback duration
      Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
   -search.maxMemoryPerQuery size
@@ -2591,11 +2649,13 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -storageDataPath string
      Path to storage data (default "victoria-metrics-data")
   -streamAggr.config string
-     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html . See also -remoteWrite.streamAggr.keepInput and -streamAggr.dedupInterval
+     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html . See also -streamAggr.keepInput, -streamAggr.dropInput and -streamAggr.dedupInterval
   -streamAggr.dedupInterval duration
      Input samples are de-duplicated with this interval before being aggregated. Only the last sample per each time series per each interval is aggregated if the interval is greater than zero
+  -streamAggr.dropInput
+     Whether to drop all the input samples after the aggregation with -streamAggr.config. By default, only aggregated samples are dropped, while the remaining samples are stored in the database. See also -streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation.html
   -streamAggr.keepInput
-     Whether to keep input samples after the aggregation with -streamAggr.config. By default, the input is dropped after the aggregation, so only the aggregate data is stored. See https://docs.victoriametrics.com/stream-aggregation.html
+     Whether to keep all the input samples after the aggregation with -streamAggr.config. By default, only aggregated samples are dropped, while the remaining samples are stored in the database. See also -streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation.html
   -tls
      Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string

app/victoria-logs/Makefile

@@ -0,0 +1,103 @@
+# All these commands must run from repository root.
+
+victoria-logs:
+	APP_NAME=victoria-logs $(MAKE) app-local
+
+victoria-logs-race:
+	APP_NAME=victoria-logs RACE=-race $(MAKE) app-local
+
+victoria-logs-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker
+
+victoria-logs-pure-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-pure
+
+victoria-logs-linux-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-amd64
+
+victoria-logs-linux-arm-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm
+
+victoria-logs-linux-arm64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm64
+
+victoria-logs-linux-ppc64le-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-ppc64le
+
+victoria-logs-linux-386-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-386
+
+victoria-logs-darwin-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-amd64
+
+victoria-logs-darwin-arm64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-arm64
+
+victoria-logs-freebsd-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-freebsd-amd64
+
+victoria-logs-openbsd-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-openbsd-amd64
+
+victoria-logs-windows-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-windows-amd64
+
+package-victoria-logs:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker
+
+package-victoria-logs-pure:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-pure
+
+package-victoria-logs-amd64:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-amd64
+
+package-victoria-logs-arm:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm
+
+package-victoria-logs-arm64:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm64
+
+package-victoria-logs-ppc64le:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-ppc64le
+
+package-victoria-logs-386:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-386
+
+publish-victoria-logs:
+	APP_NAME=victoria-logs $(MAKE) publish-via-docker
+
+victoria-logs-linux-amd64:
+	APP_NAME=victoria-logs CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-arm:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-arm64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-ppc64le:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-s390x:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-386:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+
+victoria-logs-darwin-amd64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-darwin-arm64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-freebsd-amd64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-openbsd-amd64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-windows-amd64:
+	GOARCH=amd64 APP_NAME=victoria-logs $(MAKE) app-local-windows-goarch
+
+victoria-logs-pure:
+	APP_NAME=victoria-logs $(MAKE) app-local-pure

app/victoria-logs/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 8428
+
+ENTRYPOINT ["/victoria-logs-prod"]
+ARG src_binary
+COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+)
+
+var (
+	httpListenAddr   = flag.String("httpListenAddr", ":9428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+	gogc = flag.Int("gogc", 100, "GOGC to use. See https://tip.golang.org/doc/gc-guide")
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
+	envflag.Parse()
+	cgroup.SetGOGC(*gogc)
+	buildinfo.Init()
+	logger.Init()
+	pushmetrics.Init()
+
+	logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
+	startTime := time.Now()
+
+	vlstorage.Init()
+	vlselect.Init()
+	vlinsert.Init()
+
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
+
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+	startTime = time.Now()
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+
+	vlinsert.Stop()
+	vlselect.Stop()
+	vlstorage.Stop()
+
+	fs.MustStopDirRemover()
+
+	logger.Infof("the VictoriaLogs has been stopped in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.URL.Path == "/" {
+		if r.Method != http.MethodGet {
+			return false
+		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/VictoriaLogs/'>https://docs.victoriametrics.com/VictoriaLogs/</a></br>")
+		fmt.Fprintf(w, "Useful endpoints:</br>")
+		httpserver.WriteAPIHelp(w, [][2]string{
+			{"select/vmui", "Web UI for VictoriaLogs"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+		})
+		return true
+	}
+	if vlinsert.RequestHandler(w, r) {
+		return true
+	}
+	if vlselect.RequestHandler(w, r) {
+		return true
+	}
+	return false
+}
+
+func usage() {
+	const s = `
+victoria-logs is a log management and analytics service.
+
+See the docs at https://docs.victoriametrics.com/VictoriaLogs/
+`
+	flagutil.Usage(s)
+}

app/victoria-logs/multiarch/Dockerfile

@@ -0,0 +1,12 @@
+# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
+ARG certs_image
+ARG root_image
+FROM $certs_image as certs
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
+
+FROM $root_image
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+EXPOSE 8428
+ENTRYPOINT ["/victoria-logs-prod"]
+ARG TARGETARCH
+COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/victoria-metrics/main.go

@@ -36,8 +36,8 @@ var (
 		"-promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. "+
 		"This can be changed with -promscrape.config.strictParse=false command-line flag")
 	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
-		"The saved data survives unclean shutdown such as OOM crash, hardware reset, SIGKILL, etc. "+
-		"Bigger intervals may help increasing lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
+		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
+		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
 		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
 )
 

app/vlinsert/elasticsearch/bulk_response.qtpl

@@ -0,0 +1,20 @@
+{% stripspace %}
+
+{% func BulkResponse(n int, tookMs int64) %}
+{
+	"took":{%dl tookMs %},
+	"errors":false,
+	"items":[
+		{% for i := 0; i < n; i++ %}
+		{
+			"create":{
+				"status":201
+			}
+		}
+		{% if i+1 < n %},{% endif %}
+		{% endfor %}
+	]
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vlinsert/elasticsearch/bulk_response.qtpl.go

@@ -0,0 +1,69 @@
+// Code generated by qtc from "bulk_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+package elasticsearch
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+func StreamBulkResponse(qw422016 *qt422016.Writer, n int, tookMs int64) {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+	qw422016.N().S(`{"took":`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
+	qw422016.N().DL(tookMs)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
+	qw422016.N().S(`,"errors":false,"items":[`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
+	for i := 0; i < n; i++ {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
+		qw422016.N().S(`{"create":{"status":201}}`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
+		if i+1 < n {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
+			qw422016.N().S(`,`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
+		}
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
+	}
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
+	qw422016.N().S(`]}`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+}
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+func WriteBulkResponse(qq422016 qtio422016.Writer, n int, tookMs int64) {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	StreamBulkResponse(qw422016, n, tookMs)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+}
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+func BulkResponse(n int, tookMs int64) string {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	WriteBulkResponse(qb422016, n, tookMs)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qs422016 := string(qb422016.B)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	return qs422016
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -0,0 +1,264 @@
+package elasticsearch
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// RequestHandler processes Elasticsearch insert requests
+func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
+	w.Header().Add("Content-Type", "application/json")
+	// This header is needed for Logstash
+	w.Header().Set("X-Elastic-Product", "Elasticsearch")
+
+	if strings.HasPrefix(path, "/_ilm/policy") {
+		// Return fake response for Elasticsearch ilm request.
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
+	if strings.HasPrefix(path, "/_index_template") {
+		// Return fake response for Elasticsearch index template request.
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
+	if strings.HasPrefix(path, "/_ingest") {
+		// Return fake response for Elasticsearch ingest pipeline request.
+		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
+	if strings.HasPrefix(path, "/_nodes") {
+		// Return fake response for Elasticsearch nodes discovery request.
+		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
+	switch path {
+	case "/":
+		switch r.Method {
+		case http.MethodGet:
+			// Return fake response for Elasticsearch ping request.
+			// See the latest available version for Elasticsearch at https://github.com/elastic/elasticsearch/releases
+			fmt.Fprintf(w, `{
+			"version": {
+				"number": "8.8.0"
+			}
+		}`)
+		case http.MethodHead:
+			// Return empty response for Logstash ping request.
+		}
+
+		return true
+	case "/_license":
+		// Return fake response for Elasticsearch license request.
+		fmt.Fprintf(w, `{
+			"license": {
+				"uid": "cbff45e7-c553-41f7-ae4f-9205eabd80xx",
+				"type": "oss",
+				"status": "active",
+				"expiry_date_in_millis" : 4000000000000
+			}
+		}`)
+		return true
+	case "/_bulk":
+		startTime := time.Now()
+		bulkRequestsTotal.Inc()
+
+		cp, err := insertutils.GetCommonParams(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+		processLogMessage := cp.GetProcessLogMessageFunc(lr)
+		isGzip := r.Header.Get("Content-Encoding") == "gzip"
+		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, processLogMessage)
+		if err != nil {
+			logger.Warnf("cannot decode log message #%d in /_bulk request: %s", n, err)
+			return true
+		}
+		vlstorage.MustAddRows(lr)
+		logstorage.PutLogRows(lr)
+
+		tookMs := time.Since(startTime).Milliseconds()
+		bw := bufferedwriter.Get(w)
+		defer bufferedwriter.Put(bw)
+		WriteBulkResponse(bw, n, tookMs)
+		_ = bw.Flush()
+		return true
+	default:
+		return false
+	}
+}
+
+var (
+	bulkRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
+)
+
+func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
+	processLogMessage func(timestamp int64, fields []logstorage.Field),
+) (int, error) {
+	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
+
+	if isGzip {
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return 0, fmt.Errorf("cannot read gzipped _bulk request: %w", err)
+		}
+		defer common.PutGzipReader(zr)
+		r = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(r)
+	defer writeconcurrencylimiter.PutReader(wcr)
+
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
+
+	n := 0
+	nCheckpoint := 0
+	for {
+		ok, err := readBulkLine(sc, timeField, msgField, processLogMessage)
+		wcr.DecConcurrency()
+		if err != nil || !ok {
+			rowsIngestedTotal.Add(n - nCheckpoint)
+			return n, err
+		}
+		n++
+		if batchSize := n - nCheckpoint; n >= 1000 {
+			rowsIngestedTotal.Add(batchSize)
+			nCheckpoint = n
+		}
+	}
+}
+
+var lineBufferPool bytesutil.ByteBufferPool
+
+var rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
+
+func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
+	processLogMessage func(timestamp int64, fields []logstorage.Field),
+) (bool, error) {
+	var line []byte
+
+	// Read the command, must be "create" or "index"
+	for len(line) == 0 {
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`cannot read "create" or "index" command, since its size exceeds -insert.maxLineSizeBytes=%d`,
+						insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
+		}
+		line = sc.Bytes()
+	}
+	lineStr := bytesutil.ToUnsafeString(line)
+	if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
+		return false, fmt.Errorf(`unexpected command %q; expecting "create" or "index"`, line)
+	}
+
+	// Decode log message
+	if !sc.Scan() {
+		if err := sc.Err(); err != nil {
+			if errors.Is(err, bufio.ErrTooLong) {
+				return false, fmt.Errorf("cannot read log message, since its size exceeds -insert.maxLineSizeBytes=%d", insertutils.MaxLineSizeBytes.IntN())
+			}
+			return false, err
+		}
+		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
+	}
+	line = sc.Bytes()
+	p := logjson.GetParser()
+	if err := p.ParseLogMessage(line); err != nil {
+		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
+	}
+
+	ts, err := extractTimestampFromFields(timeField, p.Fields)
+	if err != nil {
+		return false, fmt.Errorf("cannot parse timestamp: %w", err)
+	}
+	if ts == 0 {
+		ts = time.Now().UnixNano()
+	}
+	p.RenameField(msgField, "_msg")
+	processLogMessage(ts, p.Fields)
+	logjson.PutParser(p)
+	return true, nil
+}
+
+func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
+		}
+		timestamp, err := parseElasticsearchTimestamp(f.Value)
+		if err != nil {
+			return 0, err
+		}
+		f.Value = ""
+		return timestamp, nil
+	}
+	return 0, nil
+}
+
+func parseElasticsearchTimestamp(s string) (int64, error) {
+	if s == "0" || s == "" {
+		// Special case - zero or empty timestamp must be substituted
+		// with the current time by the caller.
+		return 0, nil
+	}
+	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
+		// Try parsing timestamp in milliseconds
+		n, err := strconv.ParseInt(s, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
+		}
+		if n > int64(math.MaxInt64)/1e6 {
+			return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
+		}
+		if n < int64(math.MinInt64)/1e6 {
+			return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
+		}
+		n *= 1e6
+		return n, nil
+	}
+	if len(s) == len("YYYY-MM-DD") {
+		t, err := time.Parse("2006-01-02", s)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse date %q: %w", s, err)
+		}
+		return t.UnixNano(), nil
+	}
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
+	}
+	return t.UnixNano(), nil
+}

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -0,0 +1,129 @@
+package elasticsearch
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestReadBulkRequestFailure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			t.Fatalf("unexpected call to processLogMessage with timestamp=%d, fields=%s", timestamp, fields)
+		}
+
+		r := bytes.NewBufferString(data)
+		rows, err := readBulkRequest(r, false, "_time", "_msg", processLogMessage)
+		if err == nil {
+			t.Fatalf("expecting non-empty error")
+		}
+		if rows != 0 {
+			t.Fatalf("unexpected non-zero rows=%d", rows)
+		}
+	}
+	f("foobar")
+	f(`{}`)
+	f(`{"create":{}}`)
+	f(`{"creat":{}}
+{}`)
+	f(`{"create":{}}
+foobar`)
+}
+
+func TestReadBulkRequestSuccess(t *testing.T) {
+	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+
+		var timestamps []int64
+		var result string
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			timestamps = append(timestamps, timestamp)
+
+			a := make([]string, len(fields))
+			for i, f := range fields {
+				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
+			}
+			s := "{" + strings.Join(a, ",") + "}\n"
+			result += s
+		}
+
+		// Read the request without compression
+		r := bytes.NewBufferString(data)
+		rows, err := readBulkRequest(r, false, timeField, msgField, processLogMessage)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+
+		// Read the request with compression
+		timestamps = nil
+		result = ""
+		compressedData := compressData(data)
+		r = bytes.NewBufferString(compressedData)
+		rows, err = readBulkRequest(r, true, timeField, msgField, processLogMessage)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	// Verify an empty data
+	f("", "_time", "_msg", 0, nil, "")
+	f("\n", "_time", "_msg", 0, nil, "")
+	f("\n\n", "_time", "_msg", 0, nil, "")
+
+	// Verify non-empty data
+	data := `{"create":{"_index":"filebeat-8.8.0"}}
+{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
+{"create":{"_index":"filebeat-8.8.0"}}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
+{"index":{"_index":"filebeat-8.8.0"}}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
+`
+	timeField := "@timestamp"
+	msgField := "message"
+	rowsExpected := 3
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}
+`
+	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
+}
+
+func compressData(s string) string {
+	var bb bytes.Buffer
+	zw := gzip.NewWriter(&bb)
+	if _, err := zw.Write([]byte(s)); err != nil {
+		panic(fmt.Errorf("unexpected error when compressing data: %s", err))
+	}
+	if err := zw.Close(); err != nil {
+		panic(fmt.Errorf("unexpected error when closing gzip writer: %s", err))
+	}
+	return bb.String()
+}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -0,0 +1,50 @@
+package elasticsearch
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func BenchmarkReadBulkRequest(b *testing.B) {
+	b.Run("gzip:off", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, false)
+	})
+	b.Run("gzip:on", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, true)
+	})
+}
+
+func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
+	data := `{"create":{"_index":"filebeat-8.8.0"}}
+{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
+{"create":{"_index":"filebeat-8.8.0"}}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
+{"create":{"_index":"filebeat-8.8.0"}}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
+`
+	if isGzip {
+		data = compressData(data)
+	}
+	dataBytes := bytesutil.ToUnsafeBytes(data)
+
+	timeField := "@timestamp"
+	msgField := "message"
+	processLogMessage := func(timestmap int64, fields []logstorage.Field) {}
+
+	b.ReportAllocs()
+	b.SetBytes(int64(len(data)))
+	b.RunParallel(func(pb *testing.PB) {
+		r := &bytes.Reader{}
+		for pb.Next() {
+			r.Reset(dataBytes)
+			_, err := readBulkRequest(r, isGzip, timeField, msgField, processLogMessage)
+			if err != nil {
+				panic(fmt.Errorf("unexpected error: %s", err))
+			}
+		}
+	})
+}

app/vlinsert/insertutils/common_params.go

@@ -0,0 +1,91 @@
+package insertutils
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// CommonParams contains common HTTP parameters used by log ingestion APIs.
+//
+// See https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/#http-parameters
+type CommonParams struct {
+	TenantID     logstorage.TenantID
+	TimeField    string
+	MsgField     string
+	StreamFields []string
+	IgnoreFields []string
+
+	Debug           bool
+	DebugRequestURI string
+	DebugRemoteAddr string
+}
+
+// GetCommonParams returns CommonParams from r.
+func GetCommonParams(r *http.Request) (*CommonParams, error) {
+	// Extract tenantID
+	tenantID, err := logstorage.GetTenantIDFromRequest(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// Extract time field name from _time_field query arg
+	var timeField = "_time"
+	if tf := r.FormValue("_time_field"); tf != "" {
+		timeField = tf
+	}
+
+	// Extract message field name from _msg_field query arg
+	var msgField = ""
+	if msgf := r.FormValue("_msg_field"); msgf != "" {
+		msgField = msgf
+	}
+
+	streamFields := httputils.GetArray(r, "_stream_fields")
+	ignoreFields := httputils.GetArray(r, "ignore_fields")
+
+	debug := httputils.GetBool(r, "debug")
+	debugRequestURI := ""
+	debugRemoteAddr := ""
+	if debug {
+		debugRequestURI = httpserver.GetRequestURI(r)
+		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
+	}
+
+	cp := &CommonParams{
+		TenantID:        tenantID,
+		TimeField:       timeField,
+		MsgField:        msgField,
+		StreamFields:    streamFields,
+		IgnoreFields:    ignoreFields,
+		Debug:           debug,
+		DebugRequestURI: debugRequestURI,
+		DebugRemoteAddr: debugRemoteAddr,
+	}
+	return cp, nil
+}
+
+// GetProcessLogMessageFunc returns a function, which adds parsed log messages to lr.
+func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(timestamp int64, fields []logstorage.Field) {
+	return func(timestamp int64, fields []logstorage.Field) {
+		lr.MustAdd(cp.TenantID, timestamp, fields)
+		if cp.Debug {
+			s := lr.GetRowString(0)
+			lr.ResetKeepSettings()
+			logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", cp.DebugRemoteAddr, cp.DebugRequestURI, s)
+			rowsDroppedTotal.Inc()
+			return
+		}
+		if lr.NeedFlush() {
+			vlstorage.MustAddRows(lr)
+			lr.ResetKeepSettings()
+		}
+	}
+}
+
+var rowsDroppedTotal = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)

app/vlinsert/insertutils/flags.go

@@ -0,0 +1,10 @@
+package insertutils
+
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+)
+
+var (
+	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
+	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
+)

app/vlinsert/jsonline/jsonline.go

@@ -0,0 +1,149 @@
+package jsonline
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// RequestHandler processes jsonline insert requests
+func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+	w.Header().Add("Content-Type", "application/json")
+
+	if r.Method != "POST" {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return true
+	}
+
+	requestsTotal.Inc()
+
+	cp, err := insertutils.GetCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return true
+	}
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			logger.Errorf("cannot read gzipped _bulk request: %s", err)
+			return true
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	defer writeconcurrencylimiter.PutReader(wcr)
+
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
+
+	n := 0
+	for {
+		ok, err := readLine(sc, cp.TimeField, cp.MsgField, processLogMessage)
+		wcr.DecConcurrency()
+		if err != nil {
+			logger.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
+			break
+		}
+		if !ok {
+			break
+		}
+		n++
+		rowsIngestedTotal.Inc()
+	}
+
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
+
+	return true
+}
+
+func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage func(timestamp int64, fields []logstorage.Field)) (bool, error) {
+	var line []byte
+	for len(line) == 0 {
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`cannot read json line, since its size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
+		}
+		line = sc.Bytes()
+	}
+
+	p := logjson.GetParser()
+	if err := p.ParseLogMessage(line); err != nil {
+		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
+	}
+	ts, err := extractTimestampFromFields(timeField, p.Fields)
+	if err != nil {
+		return false, fmt.Errorf("cannot parse timestamp: %w", err)
+	}
+	if ts == 0 {
+		ts = time.Now().UnixNano()
+	}
+	p.RenameField(msgField, "_msg")
+	processLogMessage(ts, p.Fields)
+	logjson.PutParser(p)
+	return true, nil
+}
+
+func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
+		}
+		timestamp, err := parseISO8601Timestamp(f.Value)
+		if err != nil {
+			return 0, err
+		}
+		f.Value = ""
+		return timestamp, nil
+	}
+	return 0, nil
+}
+
+func parseISO8601Timestamp(s string) (int64, error) {
+	if s == "0" || s == "" {
+		// Special case for returning the current timestamp.
+		// It must be automatically converted to the current timestamp by the caller.
+		return 0, nil
+	}
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
+	}
+	return t.UnixNano(), nil
+}
+
+var lineBufferPool bytesutil.ByteBufferPool
+
+var (
+	requestsTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
+)

app/vlinsert/jsonline/jsonline_test.go

@@ -0,0 +1,70 @@
+package jsonline
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestReadBulkRequestSuccess(t *testing.T) {
+	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+
+		var timestamps []int64
+		var result string
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			timestamps = append(timestamps, timestamp)
+
+			a := make([]string, len(fields))
+			for i, f := range fields {
+				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
+			}
+			s := "{" + strings.Join(a, ",") + "}\n"
+			result += s
+		}
+
+		// Read the request without compression
+		r := bytes.NewBufferString(data)
+		sc := bufio.NewScanner(r)
+		rows := 0
+		for {
+			ok, err := readLine(sc, timeField, msgField, processLogMessage)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			if !ok {
+				break
+			}
+			rows++
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	// Verify non-empty data
+	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
+`
+	timeField := "@timestamp"
+	msgField := "message"
+	rowsExpected := 3
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}
+`
+	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
+}

app/vlinsert/loki/loki.go

@@ -0,0 +1,56 @@
+package loki
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	lokiRequestsJSONTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
+	lokiRequestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
+)
+
+// RequestHandler processes Loki insert requests
+//
+// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
+func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
+	if path != "/api/v1/push" {
+		return false
+	}
+	contentType := r.Header.Get("Content-Type")
+	switch contentType {
+	case "application/json":
+		lokiRequestsJSONTotal.Inc()
+		return handleJSON(r, w)
+	default:
+		// Protobuf request body should be handled by default accoring to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
+		lokiRequestsProtobufTotal.Inc()
+		return handleProtobuf(r, w)
+	}
+}
+
+func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
+	cp, err := insertutils.GetCommonParams(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// If parsed tenant is (0,0) it is likely to be default tenant
+	// Try parsing tenant from Loki headers
+	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
+		org := r.Header.Get("X-Scope-OrgID")
+		if org != "" {
+			tenantID, err := logstorage.GetTenantIDFromString(org)
+			if err != nil {
+				return nil, err
+			}
+			cp.TenantID = tenantID
+		}
+
+	}
+
+	return cp, nil
+}

app/vlinsert/loki/loki_json.go

@@ -0,0 +1,190 @@
+package loki
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson"
+)
+
+var (
+	rowsIngestedJSONTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
+	parserPool            fastjson.ParserPool
+)
+
+func handleJSON(r *http.Request, w http.ResponseWriter) bool {
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
+			return true
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return true
+	}
+
+	cp, err := getCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
+		return true
+	}
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+	n, err := parseJSONRequest(data, processLogMessage)
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse Loki request: %s", err)
+		return true
+	}
+	rowsIngestedJSONTotal.Add(n)
+	return true
+}
+
+func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
+	p := parserPool.Get()
+	defer parserPool.Put(p)
+	v, err := p.ParseBytes(data)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse JSON request body: %w", err)
+	}
+
+	streamsV := v.Get("streams")
+	if streamsV == nil {
+		return 0, fmt.Errorf("missing `streams` item in the parsed JSON: %q", v)
+	}
+	streams, err := streamsV.Array()
+	if err != nil {
+		return 0, fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
+	}
+
+	currentTimestamp := time.Now().UnixNano()
+	var commonFields []logstorage.Field
+	rowsIngested := 0
+	for _, stream := range streams {
+		// populate common labels from `stream` dict
+		commonFields = commonFields[:0]
+		labelsV := stream.Get("stream")
+		var labels *fastjson.Object
+		if labelsV != nil {
+			o, err := labelsV.Object()
+			if err != nil {
+				return rowsIngested, fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
+			}
+			labels = o
+		}
+		labels.Visit(func(k []byte, v *fastjson.Value) {
+			if err != nil {
+				return
+			}
+			vStr, errLocal := v.StringBytes()
+			if errLocal != nil {
+				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
+				return
+			}
+			commonFields = append(commonFields, logstorage.Field{
+				Name:  bytesutil.ToUnsafeString(k),
+				Value: bytesutil.ToUnsafeString(vStr),
+			})
+		})
+		if err != nil {
+			return rowsIngested, fmt.Errorf("error when parsing `stream` object: %w", err)
+		}
+
+		// populate messages from `values` array
+		linesV := stream.Get("values")
+		if linesV == nil {
+			return rowsIngested, fmt.Errorf("missing `values` item in the parsed JSON %q", stream)
+		}
+		lines, err := linesV.Array()
+		if err != nil {
+			return rowsIngested, fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
+		}
+
+		fields := commonFields
+		for _, line := range lines {
+			lineA, err := line.Array()
+			if err != nil {
+				return rowsIngested, fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
+			}
+			if len(lineA) != 2 {
+				return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2", line, len(lineA))
+			}
+
+			// parse timestamp
+			timestamp, err := lineA[0].StringBytes()
+			if err != nil {
+				return rowsIngested, fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
+			}
+			ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
+			if err != nil {
+				return rowsIngested, fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
+			}
+			if ts == 0 {
+				ts = currentTimestamp
+			}
+
+			// parse log message
+			msg, err := lineA[1].StringBytes()
+			if err != nil {
+				return rowsIngested, fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
+			}
+
+			fields = append(fields[:len(commonFields)], logstorage.Field{
+				Name:  "_msg",
+				Value: bytesutil.ToUnsafeString(msg),
+			})
+			processLogMessage(ts, fields)
+
+		}
+		rowsIngested += len(lines)
+	}
+
+	return rowsIngested, nil
+}
+
+func parseLokiTimestamp(s string) (int64, error) {
+	if s == "" {
+		// Special case - an empty timestamp must be substituted with the current time by the caller.
+		return 0, nil
+	}
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		// Fall back to parsing floating-point value
+		f, err := strconv.ParseFloat(s, 64)
+		if err != nil {
+			return 0, err
+		}
+		if f > math.MaxInt64 {
+			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
+		}
+		if f < math.MinInt64 {
+			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
+		}
+		n = int64(f)
+	}
+	if n < 0 {
+		return 0, fmt.Errorf("too small timestamp in nanoseconds: %d; must be bigger than 0", n)
+	}
+	return n, nil
+}

app/vlinsert/loki/loki_json_test.go

@@ -0,0 +1,130 @@
+package loki
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestParseJSONRequestFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
+			t.Fatalf("unexpected call to parseJSONRequest callback!")
+		})
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if n != 0 {
+			t.Fatalf("unexpected number of parsed lines: %d; want 0", n)
+		}
+	}
+	f(``)
+
+	// Invalid json
+	f(`{}`)
+	f(`[]`)
+	f(`"foo"`)
+	f(`123`)
+
+	// invalid type for `streams` item
+	f(`{"streams":123}`)
+
+	// Missing `values` item
+	f(`{"streams":[{}]}`)
+
+	// Invalid type for `values` item
+	f(`{"streams":[{"values":"foobar"}]}`)
+
+	// Invalid type for `stream` item
+	f(`{"streams":[{"stream":[],"values":[]}]}`)
+
+	// Invalid type for `values` individual item
+	f(`{"streams":[{"values":[123]}]}`)
+
+	// Invalid length of `values` individual item
+	f(`{"streams":[{"values":[[]]}]}`)
+	f(`{"streams":[{"values":[["123"]]}]}`)
+	f(`{"streams":[{"values":[["123","456","789"]]}]}`)
+
+	// Invalid type for timestamp inside `values` individual item
+	f(`{"streams":[{"values":[[123,"456"]}]}`)
+
+	// Invalid type for log message
+	f(`{"streams":[{"values":[["123",1234]]}]}`)
+}
+
+func TestParseJSONRequestSuccess(t *testing.T) {
+	f := func(s string, resultExpected string) {
+		t.Helper()
+		var lines []string
+		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
+			var a []string
+			for _, f := range fields {
+				a = append(a, f.String())
+			}
+			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
+			lines = append(lines, line)
+		})
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if n != len(lines) {
+			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
+		}
+		result := strings.Join(lines, "\n")
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	// Empty streams
+	f(`{"streams":[]}`, ``)
+	f(`{"streams":[{"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
+
+	// Empty stream labels
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+
+	// Non-empty stream labels
+	f(`{"streams":[{"stream":{
+	"label1": "value1",
+	"label2": "value2"
+},"values":[
+	["1577836800000000001", "foo bar"],
+	["1477836900005000002", "abc"],
+	["147.78369e9", "foobar"]
+]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
+_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
+_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
+
+	// Multiple streams
+	f(`{
+	"streams": [
+		{
+			"stream": {
+				"foo": "bar",
+				"a": "b"
+			},
+			"values": [
+				["1577836800000000001", "foo bar"],
+				["1577836900005000002", "abc"]
+			]
+		},
+		{
+			"stream": {
+				"x": "y"
+			},
+			"values": [
+				["1877836900005000002", "yx"]
+			]
+		}
+	]
+}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
+_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
+_time:1877836900005000002 "x":"y" "_msg":"yx"`)
+}

app/vlinsert/loki/loki_json_timing_test.go

@@ -0,0 +1,78 @@
+package loki
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func BenchmarkParseJSONRequest(b *testing.B) {
+	for _, streams := range []int{5, 10} {
+		for _, rows := range []int{100, 1000} {
+			for _, labels := range []int{10, 50} {
+				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
+					benchmarkParseJSONRequest(b, streams, rows, labels)
+				})
+			}
+		}
+	}
+}
+
+func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
+	b.ReportAllocs()
+	b.SetBytes(int64(streams * rows))
+	b.RunParallel(func(pb *testing.PB) {
+		data := getJSONBody(streams, rows, labels)
+		for pb.Next() {
+			_, err := parseJSONRequest(data, func(timestamp int64, fields []logstorage.Field) {})
+			if err != nil {
+				panic(fmt.Errorf("unexpected error: %s", err))
+			}
+		}
+	})
+}
+
+func getJSONBody(streams, rows, labels int) []byte {
+	body := append([]byte{}, `{"streams":[`...)
+	now := time.Now().UnixNano()
+	valuePrefix := fmt.Sprintf(`["%d","value_`, now)
+
+	for i := 0; i < streams; i++ {
+		body = append(body, `{"stream":{`...)
+
+		for j := 0; j < labels; j++ {
+			body = append(body, `"label_`...)
+			body = strconv.AppendInt(body, int64(j), 10)
+			body = append(body, `":"value_`...)
+			body = strconv.AppendInt(body, int64(j), 10)
+			body = append(body, '"')
+			if j < labels-1 {
+				body = append(body, ',')
+			}
+
+		}
+		body = append(body, `}, "values":[`...)
+
+		for j := 0; j < rows; j++ {
+			body = append(body, valuePrefix...)
+			body = strconv.AppendInt(body, int64(j), 10)
+			body = append(body, `"]`...)
+			if j < rows-1 {
+				body = append(body, ',')
+			}
+		}
+
+		body = append(body, `]}`...)
+		if i < streams-1 {
+			body = append(body, ',')
+		}
+
+	}
+
+	body = append(body, `]}`...)
+
+	return body
+}

app/vlinsert/loki/loki_protobuf.go

@@ -0,0 +1,171 @@
+package loki
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
+)
+
+var (
+	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
+	bytesBufPool              bytesutil.ByteBufferPool
+	pushReqsPool              sync.Pool
+)
+
+func handleProtobuf(r *http.Request, w http.ResponseWriter) bool {
+	wcr := writeconcurrencylimiter.GetReader(r.Body)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return true
+	}
+
+	cp, err := getCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
+		return true
+	}
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+	n, err := parseProtobufRequest(data, processLogMessage)
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse loki request: %s", err)
+		return true
+	}
+	rowsIngestedProtobufTotal.Add(n)
+	return true
+}
+
+func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
+	bb := bytesBufPool.Get()
+	defer bytesBufPool.Put(bb)
+
+	buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
+	if err != nil {
+		return 0, fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
+	}
+	bb.B = buf
+
+	req := getPushRequest()
+	defer putPushRequest(req)
+
+	err = req.Unmarshal(bb.B)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse request body: %s", err)
+	}
+
+	var commonFields []logstorage.Field
+	rowsIngested := 0
+	streams := req.Streams
+	currentTimestamp := time.Now().UnixNano()
+	for i := range streams {
+		stream := &streams[i]
+		// st.Labels contains labels for the stream.
+		// Labels are same for all entries in the stream.
+		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
+		if err != nil {
+			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %s", stream.Labels, err)
+		}
+		fields := commonFields
+
+		entries := stream.Entries
+		for j := range entries {
+			entry := &entries[j]
+			fields = append(fields[:len(commonFields)], logstorage.Field{
+				Name:  "_msg",
+				Value: entry.Line,
+			})
+			ts := entry.Timestamp.UnixNano()
+			if ts == 0 {
+				ts = currentTimestamp
+			}
+			processLogMessage(ts, fields)
+		}
+		rowsIngested += len(stream.Entries)
+	}
+	return rowsIngested, nil
+}
+
+// parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
+//
+// See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
+func parsePromLabels(dst []logstorage.Field, s string) ([]logstorage.Field, error) {
+	// Make sure s is wrapped into `{...}`
+	s = strings.TrimSpace(s)
+	if len(s) < 2 {
+		return nil, fmt.Errorf("too short string to parse: %q", s)
+	}
+	if s[0] != '{' {
+		return nil, fmt.Errorf("missing `{` at the beginning of %q", s)
+	}
+	if s[len(s)-1] != '}' {
+		return nil, fmt.Errorf("missing `}` at the end of %q", s)
+	}
+	s = s[1 : len(s)-1]
+
+	for len(s) > 0 {
+		// Parse label name
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("cannot find `=` char for label value at %s", s)
+		}
+		name := s[:n]
+		s = s[n+1:]
+
+		// Parse label value
+		qs, err := strconv.QuotedPrefix(s)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse value for label %q at %s: %w", name, s, err)
+		}
+		s = s[len(qs):]
+		value, err := strconv.Unquote(qs)
+		if err != nil {
+			return nil, fmt.Errorf("cannot unquote value %q for label %q: %w", qs, name, err)
+		}
+
+		// Append the found field to dst.
+		dst = append(dst, logstorage.Field{
+			Name:  name,
+			Value: value,
+		})
+
+		// Check whether there are other labels remaining
+		if len(s) == 0 {
+			break
+		}
+		if !strings.HasPrefix(s, ",") {
+			return nil, fmt.Errorf("missing `,` char at %s", s)
+		}
+		s = s[1:]
+		s = strings.TrimPrefix(s, " ")
+	}
+	return dst, nil
+}
+
+func getPushRequest() *PushRequest {
+	v := pushReqsPool.Get()
+	if v == nil {
+		return &PushRequest{}
+	}
+	return v.(*PushRequest)
+}
+
+func putPushRequest(req *PushRequest) {
+	req.Reset()
+	pushReqsPool.Put(req)
+}

app/vlinsert/loki/loki_protobuf_test.go

@@ -0,0 +1,171 @@
+package loki
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/golang/snappy"
+)
+
+func TestParseProtobufRequestSuccess(t *testing.T) {
+	f := func(s string, resultExpected string) {
+		t.Helper()
+		var pr PushRequest
+		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
+			msg := ""
+			for _, f := range fields {
+				if f.Name == "_msg" {
+					msg = f.Value
+				}
+			}
+			var a []string
+			for _, f := range fields {
+				if f.Name == "_msg" {
+					continue
+				}
+				item := fmt.Sprintf("%s=%q", f.Name, f.Value)
+				a = append(a, item)
+			}
+			labels := "{" + strings.Join(a, ", ") + "}"
+			pr.Streams = append(pr.Streams, Stream{
+				Labels: labels,
+				Entries: []Entry{
+					{
+						Timestamp: time.Unix(0, timestamp),
+						Line:      msg,
+					},
+				},
+			})
+		})
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if n != len(pr.Streams) {
+			t.Fatalf("unexpected number of streams; got %d; want %d", len(pr.Streams), n)
+		}
+
+		data, err := pr.Marshal()
+		if err != nil {
+			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
+		}
+		encodedData := snappy.Encode(nil, data)
+
+		var lines []string
+		n, err = parseProtobufRequest(encodedData, func(timestamp int64, fields []logstorage.Field) {
+			var a []string
+			for _, f := range fields {
+				a = append(a, f.String())
+			}
+			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
+			lines = append(lines, line)
+		})
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if n != len(lines) {
+			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
+		}
+		result := strings.Join(lines, "\n")
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	// Empty streams
+	f(`{"streams":[]}`, ``)
+	f(`{"streams":[{"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
+
+	// Empty stream labels
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+
+	// Non-empty stream labels
+	f(`{"streams":[{"stream":{
+	"label1": "value1",
+	"label2": "value2"
+},"values":[
+	["1577836800000000001", "foo bar"],
+	["1477836900005000002", "abc"],
+	["147.78369e9", "foobar"]
+]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
+_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
+_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
+
+	// Multiple streams
+	f(`{
+	"streams": [
+		{
+			"stream": {
+				"foo": "bar",
+				"a": "b"
+			},
+			"values": [
+				["1577836800000000001", "foo bar"],
+				["1577836900005000002", "abc"]
+			]
+		},
+		{
+			"stream": {
+				"x": "y"
+			},
+			"values": [
+				["1877836900005000002", "yx"]
+			]
+		}
+	]
+}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
+_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
+_time:1877836900005000002 "x":"y" "_msg":"yx"`)
+}
+
+func TestParsePromLabelsSuccess(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		fields, err := parsePromLabels(nil, s)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		var a []string
+		for _, f := range fields {
+			a = append(a, fmt.Sprintf("%s=%q", f.Name, f.Value))
+		}
+		result := "{" + strings.Join(a, ", ") + "}"
+		if result != s {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, s)
+		}
+	}
+
+	f("{}")
+	f(`{foo="bar"}`)
+	f(`{foo="bar", baz="x", y="z"}`)
+	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
+}
+
+func TestParsePromLabelsFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		fields, err := parsePromLabels(nil, s)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if len(fields) > 0 {
+			t.Fatalf("unexpected non-empty fields: %s", fields)
+		}
+	}
+
+	f("")
+	f("{")
+	f(`{foo}`)
+	f(`{foo=bar}`)
+	f(`{foo="bar}`)
+	f(`{foo="ba\",r}`)
+	f(`{foo="bar" baz="aa"}`)
+	f(`foobar`)
+	f(`foo{bar="baz"}`)
+}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -0,0 +1,65 @@
+package loki
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/golang/snappy"
+)
+
+func BenchmarkParseProtobufRequest(b *testing.B) {
+	for _, streams := range []int{5, 10} {
+		for _, rows := range []int{100, 1000} {
+			for _, labels := range []int{10, 50} {
+				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
+					benchmarkParseProtobufRequest(b, streams, rows, labels)
+				})
+			}
+		}
+	}
+}
+
+func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
+	b.ReportAllocs()
+	b.SetBytes(int64(streams * rows))
+	b.RunParallel(func(pb *testing.PB) {
+		body := getProtobufBody(streams, rows, labels)
+		for pb.Next() {
+			_, err := parseProtobufRequest(body, func(timestamp int64, fields []logstorage.Field) {})
+			if err != nil {
+				panic(fmt.Errorf("unexpected error: %s", err))
+			}
+		}
+	})
+}
+
+func getProtobufBody(streams, rows, labels int) []byte {
+	var pr PushRequest
+
+	for i := 0; i < streams; i++ {
+		var st Stream
+
+		st.Labels = `{`
+		for j := 0; j < labels; j++ {
+			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
+			if j < labels-1 {
+				st.Labels += `,`
+			}
+		}
+		st.Labels += `}`
+
+		for j := 0; j < rows; j++ {
+			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
+		}
+
+		pr.Streams = append(pr.Streams, st)
+	}
+
+	body, _ := pr.Marshal()
+	encodedBody := snappy.Encode(nil, body)
+
+	return encodedBody
+}

app/vlinsert/loki/push_request.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+package logproto;
+
+import "gogoproto/gogo.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
+
+message PushRequest {
+  repeated StreamAdapter streams = 1 [
+    (gogoproto.jsontag) = "streams",
+    (gogoproto.customtype) = "Stream"
+  ];
+}
+
+message StreamAdapter {
+  string labels = 1 [(gogoproto.jsontag) = "labels"];
+  repeated EntryAdapter entries = 2 [
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "entries"
+  ];
+  // hash contains the original hash of the stream.
+  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
+}
+
+message EntryAdapter {
+  google.protobuf.Timestamp timestamp = 1 [
+    (gogoproto.stdtime) = true,
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "ts"
+  ];
+  string line = 2 [(gogoproto.jsontag) = "line"];
+}

app/vlinsert/loki/timestamp.go

@@ -0,0 +1,110 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/gogo/protobuf/types"
+)
+
+const (
+	// Seconds field of the earliest valid Timestamp.
+	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	minValidSeconds = -62135596800
+	// Seconds field just after the latest valid Timestamp.
+	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	maxValidSeconds = 253402300800
+)
+
+// validateTimestamp determines whether a Timestamp is valid.
+// A valid timestamp represents a time in the range
+// [0001-01-01, 10000-01-01) and has a Nanos field
+// in the range [0, 1e9).
+//
+// If the Timestamp is valid, validateTimestamp returns nil.
+// Otherwise, it returns an error that describes
+// the problem.
+//
+// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
+func validateTimestamp(ts *types.Timestamp) error {
+	if ts == nil {
+		return errors.New("timestamp: nil Timestamp")
+	}
+	if ts.Seconds < minValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
+	}
+	if ts.Seconds >= maxValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
+	}
+	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
+		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
+	}
+	return nil
+}
+
+// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
+// but avoids the escape incurred by using fmt.Sprintf, eliminating
+// unnecessary heap allocations.
+func formatTimestamp(ts *types.Timestamp) string {
+	if ts == nil {
+		return "nil"
+	}
+
+	seconds := strconv.FormatInt(ts.Seconds, 10)
+	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
+	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
+}
+
+func sizeOfStdTime(t time.Time) int {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0
+	}
+	return ts.Size()
+}
+
+func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0, err
+	}
+	return ts.MarshalTo(data)
+}
+
+func stdTimeUnmarshal(t *time.Time, data []byte) error {
+	ts := &types.Timestamp{}
+	if err := ts.Unmarshal(data); err != nil {
+		return err
+	}
+	tt, err := timestampFromProto(ts)
+	if err != nil {
+		return err
+	}
+	*t = tt
+	return nil
+}
+
+func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
+	// Don't return the zero value on error, because corresponds to a valid
+	// timestamp. Instead return whatever time.Unix gives us.
+	var t time.Time
+	if ts == nil {
+		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
+	} else {
+		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
+	}
+	return t, validateTimestamp(ts)
+}
+
+func timestampProto(t time.Time) (types.Timestamp, error) {
+	ts := types.Timestamp{
+		Seconds: t.Unix(),
+		Nanos:   int32(t.Nanosecond()),
+	}
+	return ts, validateTimestamp(&ts)
+}

app/vlinsert/loki/types.go

@@ -0,0 +1,481 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"fmt"
+	"io"
+	"time"
+)
+
+// Stream contains a unique labels set as a string and a set of entries for it.
+// We are not using the proto generated version but this custom one so that we
+// can improve serialization see benchmark.
+type Stream struct {
+	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
+	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
+	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
+}
+
+// Entry is a log entry with a timestamp.
+type Entry struct {
+	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
+	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Stream) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Hash != 0 {
+		i = encodeVarintPush(dAtA, i, m.Hash)
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Entries) > 0 {
+		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintPush(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Labels) > 0 {
+		i -= len(m.Labels)
+		copy(dAtA[i:], m.Labels)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Entry) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Line) > 0 {
+		i -= len(m.Line)
+		copy(dAtA[i:], m.Line)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
+		i--
+		dAtA[i] = 0x12
+	}
+	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
+	if err7 != nil {
+		return 0, err7
+	}
+	i -= n7
+	i = encodeVarintPush(dAtA, i, uint64(n7))
+	i--
+	dAtA[i] = 0xa
+	return len(dAtA) - i, nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Stream) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Labels = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Entries = append(m.Entries, Entry{})
+			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
+			}
+			m.Hash = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Hash |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Entry) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Line = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Size returns the size of the serialized Stream.
+func (m *Stream) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Labels)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	if len(m.Entries) > 0 {
+		for _, e := range m.Entries {
+			l = e.Size()
+			n += 1 + l + sovPush(uint64(l))
+		}
+	}
+	if m.Hash != 0 {
+		n += 1 + sovPush(m.Hash)
+	}
+	return n
+}
+
+// Size returns the size of the serialized Entry
+func (m *Entry) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = sizeOfStdTime(m.Timestamp)
+	n += 1 + l + sovPush(uint64(l))
+	l = len(m.Line)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	return n
+}
+
+// Equal returns true if the two Streams are equal.
+func (m *Stream) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Stream)
+	if !ok {
+		that2, ok := that.(Stream)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if m.Labels != that1.Labels {
+		return false
+	}
+	if len(m.Entries) != len(that1.Entries) {
+		return false
+	}
+	for i := range m.Entries {
+		if !m.Entries[i].Equal(that1.Entries[i]) {
+			return false
+		}
+	}
+	return m.Hash == that1.Hash
+}
+
+// Equal returns true if the two Entries are equal.
+func (m *Entry) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Entry)
+	if !ok {
+		that2, ok := that.(Entry)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if !m.Timestamp.Equal(that1.Timestamp) {
+		return false
+	}
+	if m.Line != that1.Line {
+		return false
+	}
+	return true
+}

app/vlinsert/main.go

@@ -0,0 +1,43 @@
+package vlinsert
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
+)
+
+// Init initializes vlinsert
+func Init() {
+}
+
+// Stop stops vlinsert
+func Stop() {
+}
+
+// RequestHandler handles insert requests for VictoriaLogs
+func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := r.URL.Path
+	if !strings.HasPrefix(path, "/insert/") {
+		// Skip requests, which do not start with /insert/, since these aren't our requests.
+		return false
+	}
+	path = strings.TrimPrefix(path, "/insert")
+	path = strings.ReplaceAll(path, "//", "/")
+
+	if path == "/jsonline" {
+		return jsonline.RequestHandler(w, r)
+	}
+	switch {
+	case strings.HasPrefix(path, "/elasticsearch/"):
+		path = strings.TrimPrefix(path, "/elasticsearch")
+		return elasticsearch.RequestHandler(path, w, r)
+	case strings.HasPrefix(path, "/loki/"):
+		path = strings.TrimPrefix(path, "/loki")
+		return loki.RequestHandler(path, w, r)
+	default:
+		return false
+	}
+}

app/vlselect/logsql/logsql.go

@@ -0,0 +1,56 @@
+package logsql
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+var (
+	maxSortBufferSize = flagutil.NewBytes("select.maxSortBufferSize", 1024*1024, "Query results from /select/logsql/query are automatically sorted by _time "+
+		"if their summary size doesn't exceed this value; otherwise, query results are streamed in the response without sorting; "+
+		"too big value for this flag may result in high memory usage since the sorting is performed in memory")
+)
+
+// ProcessQueryRequest handles /select/logsql/query request
+func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}) {
+	// Extract tenantID
+	tenantID, err := logstorage.GetTenantIDFromRequest(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	qStr := r.FormValue("query")
+	q, err := logstorage.ParseQuery(qStr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse query [%s]: %s", qStr, err)
+		return
+	}
+	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
+
+	sw := getSortWriter()
+	sw.Init(w, maxSortBufferSize.IntN())
+	tenantIDs := []logstorage.TenantID{tenantID}
+	vlstorage.RunQuery(tenantIDs, q, stopCh, func(columns []logstorage.BlockColumn) {
+		if len(columns) == 0 {
+			return
+		}
+		rowsCount := len(columns[0].Values)
+
+		bb := blockResultPool.Get()
+		for rowIdx := 0; rowIdx < rowsCount; rowIdx++ {
+			WriteJSONRow(bb, columns, rowIdx)
+		}
+		sw.MustWrite(bb.B)
+		blockResultPool.Put(bb)
+	})
+	sw.FinalFlush()
+	putSortWriter(sw)
+}
+
+var blockResultPool bytesutil.ByteBufferPool

app/vlselect/logsql/query_response.qtpl

@@ -0,0 +1,41 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+) %}
+
+{% stripspace %}
+
+// JSONRow creates JSON row from the given fields.
+{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
+{
+	{% code c := &columns[0] %}
+	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
+	{% code columns = columns[1:] %}
+	{% for colIdx := range columns %}
+		{% code c := &columns[colIdx] %}
+		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
+	{% endfor %}
+}{% newline %}
+{% endfunc %}
+
+// JSONRows prints formatted rows
+{% func JSONRows(rows [][]logstorage.Field) %}
+	{% if len(rows) == 0 %}
+		{% return %}
+	{% endif %}
+	{% for _, fields := range rows %}
+	{
+		{% if len(fields) > 0 %}
+			{% code
+				f := fields[0]
+				fields = fields[1:]
+			%}
+			{%q= f.Name %}:{%q= f.Value %}
+			{% for _, f := range fields %}
+				,{%q= f.Name %}:{%q= f.Value %}
+			{% endfor %}
+		{% endif %}
+	}{% newline %}
+	{% endfor %}
+{% endfunc %}
+
+{% endstripspace %}

app/vlselect/logsql/query_response.qtpl.go

@@ -0,0 +1,166 @@
+// Code generated by qtc from "query_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vlselect/logsql/query_response.qtpl:1
+package logsql
+
+//line app/vlselect/logsql/query_response.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// JSONRow creates JSON row from the given fields.
+
+//line app/vlselect/logsql/query_response.qtpl:8
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlselect/logsql/query_response.qtpl:8
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlselect/logsql/query_response.qtpl:8
+func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+//line app/vlselect/logsql/query_response.qtpl:8
+	qw422016.N().S(`{`)
+//line app/vlselect/logsql/query_response.qtpl:10
+	c := &columns[0]
+
+//line app/vlselect/logsql/query_response.qtpl:11
+	qw422016.N().Q(c.Name)
+//line app/vlselect/logsql/query_response.qtpl:11
+	qw422016.N().S(`:`)
+//line app/vlselect/logsql/query_response.qtpl:11
+	qw422016.N().Q(c.Values[rowIdx])
+//line app/vlselect/logsql/query_response.qtpl:12
+	columns = columns[1:]
+
+//line app/vlselect/logsql/query_response.qtpl:13
+	for colIdx := range columns {
+//line app/vlselect/logsql/query_response.qtpl:14
+		c := &columns[colIdx]
+
+//line app/vlselect/logsql/query_response.qtpl:14
+		qw422016.N().S(`,`)
+//line app/vlselect/logsql/query_response.qtpl:15
+		qw422016.N().Q(c.Name)
+//line app/vlselect/logsql/query_response.qtpl:15
+		qw422016.N().S(`:`)
+//line app/vlselect/logsql/query_response.qtpl:15
+		qw422016.N().Q(c.Values[rowIdx])
+//line app/vlselect/logsql/query_response.qtpl:16
+	}
+//line app/vlselect/logsql/query_response.qtpl:16
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/query_response.qtpl:17
+	qw422016.N().S(`
+`)
+//line app/vlselect/logsql/query_response.qtpl:18
+}
+
+//line app/vlselect/logsql/query_response.qtpl:18
+func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+//line app/vlselect/logsql/query_response.qtpl:18
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/query_response.qtpl:18
+	StreamJSONRow(qw422016, columns, rowIdx)
+//line app/vlselect/logsql/query_response.qtpl:18
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/query_response.qtpl:18
+}
+
+//line app/vlselect/logsql/query_response.qtpl:18
+func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
+//line app/vlselect/logsql/query_response.qtpl:18
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/query_response.qtpl:18
+	WriteJSONRow(qb422016, columns, rowIdx)
+//line app/vlselect/logsql/query_response.qtpl:18
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/query_response.qtpl:18
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/query_response.qtpl:18
+	return qs422016
+//line app/vlselect/logsql/query_response.qtpl:18
+}
+
+// JSONRows prints formatted rows
+
+//line app/vlselect/logsql/query_response.qtpl:21
+func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
+//line app/vlselect/logsql/query_response.qtpl:22
+	if len(rows) == 0 {
+//line app/vlselect/logsql/query_response.qtpl:23
+		return
+//line app/vlselect/logsql/query_response.qtpl:24
+	}
+//line app/vlselect/logsql/query_response.qtpl:25
+	for _, fields := range rows {
+//line app/vlselect/logsql/query_response.qtpl:25
+		qw422016.N().S(`{`)
+//line app/vlselect/logsql/query_response.qtpl:27
+		if len(fields) > 0 {
+//line app/vlselect/logsql/query_response.qtpl:29
+			f := fields[0]
+			fields = fields[1:]
+
+//line app/vlselect/logsql/query_response.qtpl:32
+			qw422016.N().Q(f.Name)
+//line app/vlselect/logsql/query_response.qtpl:32
+			qw422016.N().S(`:`)
+//line app/vlselect/logsql/query_response.qtpl:32
+			qw422016.N().Q(f.Value)
+//line app/vlselect/logsql/query_response.qtpl:33
+			for _, f := range fields {
+//line app/vlselect/logsql/query_response.qtpl:33
+				qw422016.N().S(`,`)
+//line app/vlselect/logsql/query_response.qtpl:34
+				qw422016.N().Q(f.Name)
+//line app/vlselect/logsql/query_response.qtpl:34
+				qw422016.N().S(`:`)
+//line app/vlselect/logsql/query_response.qtpl:34
+				qw422016.N().Q(f.Value)
+//line app/vlselect/logsql/query_response.qtpl:35
+			}
+//line app/vlselect/logsql/query_response.qtpl:36
+		}
+//line app/vlselect/logsql/query_response.qtpl:36
+		qw422016.N().S(`}`)
+//line app/vlselect/logsql/query_response.qtpl:37
+		qw422016.N().S(`
+`)
+//line app/vlselect/logsql/query_response.qtpl:38
+	}
+//line app/vlselect/logsql/query_response.qtpl:39
+}
+
+//line app/vlselect/logsql/query_response.qtpl:39
+func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
+//line app/vlselect/logsql/query_response.qtpl:39
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/query_response.qtpl:39
+	StreamJSONRows(qw422016, rows)
+//line app/vlselect/logsql/query_response.qtpl:39
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/query_response.qtpl:39
+}
+
+//line app/vlselect/logsql/query_response.qtpl:39
+func JSONRows(rows [][]logstorage.Field) string {
+//line app/vlselect/logsql/query_response.qtpl:39
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/query_response.qtpl:39
+	WriteJSONRows(qb422016, rows)
+//line app/vlselect/logsql/query_response.qtpl:39
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/query_response.qtpl:39
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/query_response.qtpl:39
+	return qs422016
+//line app/vlselect/logsql/query_response.qtpl:39
+}

app/vlselect/logsql/sort_writer.go

@@ -0,0 +1,225 @@
+package logsql
+
+import (
+	"bytes"
+	"io"
+	"sort"
+	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func getSortWriter() *sortWriter {
+	v := sortWriterPool.Get()
+	if v == nil {
+		return &sortWriter{}
+	}
+	return v.(*sortWriter)
+}
+
+func putSortWriter(sw *sortWriter) {
+	sw.reset()
+	sortWriterPool.Put(sw)
+}
+
+var sortWriterPool sync.Pool
+
+// sortWriter expects JSON line stream to be written to it.
+//
+// It buffers the incoming data until its size reaches maxBufLen.
+// Then it streams the buffered data and all the incoming data to w.
+//
+// The FinalFlush() must be called when all the data is written.
+// If the buf isn't empty at FinalFlush() call, then the buffered data
+// is sorted by _time field.
+type sortWriter struct {
+	mu         sync.Mutex
+	w          io.Writer
+	maxBufLen  int
+	buf        []byte
+	bufFlushed bool
+
+	hasErr bool
+}
+
+func (sw *sortWriter) reset() {
+	sw.w = nil
+	sw.maxBufLen = 0
+	sw.buf = sw.buf[:0]
+	sw.bufFlushed = false
+	sw.hasErr = false
+}
+
+func (sw *sortWriter) Init(w io.Writer, maxBufLen int) {
+	sw.reset()
+
+	sw.w = w
+	sw.maxBufLen = maxBufLen
+}
+
+func (sw *sortWriter) MustWrite(p []byte) {
+	sw.mu.Lock()
+	defer sw.mu.Unlock()
+
+	if sw.hasErr {
+		return
+	}
+
+	if sw.bufFlushed {
+		if _, err := sw.w.Write(p); err != nil {
+			sw.hasErr = true
+		}
+		return
+	}
+	if len(sw.buf)+len(p) < sw.maxBufLen {
+		sw.buf = append(sw.buf, p...)
+		return
+	}
+	sw.bufFlushed = true
+	if len(sw.buf) > 0 {
+		if _, err := sw.w.Write(sw.buf); err != nil {
+			sw.hasErr = true
+			return
+		}
+		sw.buf = sw.buf[:0]
+	}
+	if _, err := sw.w.Write(p); err != nil {
+		sw.hasErr = true
+	}
+}
+
+func (sw *sortWriter) FinalFlush() {
+	if sw.hasErr || sw.bufFlushed {
+		return
+	}
+	rs := getRowsSorter()
+	rs.parseRows(sw.buf)
+	rs.sort()
+	WriteJSONRows(sw.w, rs.rows)
+	putRowsSorter(rs)
+}
+
+func getRowsSorter() *rowsSorter {
+	v := rowsSorterPool.Get()
+	if v == nil {
+		return &rowsSorter{}
+	}
+	return v.(*rowsSorter)
+}
+
+func putRowsSorter(rs *rowsSorter) {
+	rs.reset()
+	rowsSorterPool.Put(rs)
+}
+
+var rowsSorterPool sync.Pool
+
+type rowsSorter struct {
+	buf       []byte
+	fieldsBuf []logstorage.Field
+	rows      [][]logstorage.Field
+	times     []string
+}
+
+func (rs *rowsSorter) reset() {
+	rs.buf = rs.buf[:0]
+
+	fieldsBuf := rs.fieldsBuf
+	for i := range fieldsBuf {
+		fieldsBuf[i].Reset()
+	}
+	rs.fieldsBuf = fieldsBuf[:0]
+
+	rows := rs.rows
+	for i := range rows {
+		rows[i] = nil
+	}
+	rs.rows = rows[:0]
+
+	times := rs.times
+	for i := range times {
+		times[i] = ""
+	}
+	rs.times = times[:0]
+}
+
+func (rs *rowsSorter) parseRows(src []byte) {
+	rs.reset()
+
+	buf := rs.buf
+	fieldsBuf := rs.fieldsBuf
+	rows := rs.rows
+	times := rs.times
+
+	p := logjson.GetParser()
+	for len(src) > 0 {
+		var line []byte
+		n := bytes.IndexByte(src, '\n')
+		if n < 0 {
+			line = src
+			src = nil
+		} else {
+			line = src[:n]
+			src = src[n+1:]
+		}
+		if len(line) == 0 {
+			continue
+		}
+
+		if err := p.ParseLogMessage(line); err != nil {
+			logger.Panicf("BUG: unexpected invalid JSON line: %s", err)
+		}
+
+		timeValue := ""
+		fieldsBufLen := len(fieldsBuf)
+		for _, f := range p.Fields {
+			bufLen := len(buf)
+			buf = append(buf, f.Name...)
+			name := bytesutil.ToUnsafeString(buf[bufLen:])
+
+			bufLen = len(buf)
+			buf = append(buf, f.Value...)
+			value := bytesutil.ToUnsafeString(buf[bufLen:])
+
+			fieldsBuf = append(fieldsBuf, logstorage.Field{
+				Name:  name,
+				Value: value,
+			})
+
+			if name == "_time" {
+				timeValue = value
+			}
+		}
+		rows = append(rows, fieldsBuf[fieldsBufLen:])
+		times = append(times, timeValue)
+	}
+	logjson.PutParser(p)
+
+	rs.buf = buf
+	rs.fieldsBuf = fieldsBuf
+	rs.rows = rows
+	rs.times = times
+}
+
+func (rs *rowsSorter) Len() int {
+	return len(rs.rows)
+}
+
+func (rs *rowsSorter) Less(i, j int) bool {
+	times := rs.times
+	return times[i] < times[j]
+}
+
+func (rs *rowsSorter) Swap(i, j int) {
+	times := rs.times
+	rows := rs.rows
+	times[i], times[j] = times[j], times[i]
+	rows[i], rows[j] = rows[j], rows[i]
+}
+
+func (rs *rowsSorter) sort() {
+	sort.Sort(rs)
+}

app/vlselect/logsql/sort_writer_test.go

@@ -0,0 +1,39 @@
+package logsql
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+func TestSortWriter(t *testing.T) {
+	f := func(maxBufLen int, data string, expectedResult string) {
+		t.Helper()
+
+		var bb bytes.Buffer
+		sw := getSortWriter()
+		sw.Init(&bb, maxBufLen)
+
+		for _, s := range strings.Split(data, "\n") {
+			sw.MustWrite([]byte(s + "\n"))
+		}
+		sw.FinalFlush()
+		putSortWriter(sw)
+
+		result := bb.String()
+		if result != expectedResult {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, expectedResult)
+		}
+	}
+
+	f(100, "", "")
+	f(100, "{}", "{}\n")
+
+	data := `{"_time":"def","_msg":"xxx"}
+{"_time":"abc","_msg":"foo"}`
+	resultExpected := `{"_time":"abc","_msg":"foo"}
+{"_time":"def","_msg":"xxx"}
+`
+	f(100, data, resultExpected)
+	f(10, data, data+"\n")
+}

app/vlselect/main.go

@@ -0,0 +1,162 @@
+package vlselect
+
+import (
+	"embed"
+	"flag"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
+		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
+		"See also -search.maxQueueDuration")
+	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
+		"limit is reached; see also -search.maxQueryDuration")
+	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution")
+)
+
+func getDefaultMaxConcurrentRequests() int {
+	n := cgroup.AvailableCPUs()
+	if n <= 4 {
+		n *= 2
+	}
+	if n > 16 {
+		// A single request can saturate all the CPU cores, so there is no sense
+		// in allowing higher number of concurrent requests - they will just contend
+		// for unavailable CPU time.
+		n = 16
+	}
+	return n
+}
+
+// Init initializes vlselect
+func Init() {
+	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
+}
+
+// Stop stops vlselect
+func Stop() {
+}
+
+var concurrencyLimitCh chan struct{}
+
+var (
+	concurrencyLimitReached = metrics.NewCounter(`vl_concurrent_select_limit_reached_total`)
+	concurrencyLimitTimeout = metrics.NewCounter(`vl_concurrent_select_limit_timeout_total`)
+
+	_ = metrics.NewGauge(`vl_concurrent_select_capacity`, func() float64 {
+		return float64(cap(concurrencyLimitCh))
+	})
+	_ = metrics.NewGauge(`vl_concurrent_select_current`, func() float64 {
+		return float64(len(concurrencyLimitCh))
+	})
+)
+
+//go:embed vmui
+var vmuiFiles embed.FS
+
+var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
+
+// RequestHandler handles select requests for VictoriaLogs
+func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := r.URL.Path
+	if !strings.HasPrefix(path, "/select/") {
+		// Skip requests, which do not start with /select/, since these aren't our requests.
+		return false
+	}
+	path = strings.TrimPrefix(path, "/select")
+	path = strings.ReplaceAll(path, "//", "/")
+
+	if path == "/vmui" {
+		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
+		// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
+		// is hidden behind vmauth or similar proxy.
+		_ = r.ParseForm()
+		newURL := "vmui/?" + r.Form.Encode()
+		httpserver.Redirect(w, newURL)
+		return true
+	}
+	if strings.HasPrefix(path, "/vmui/") {
+		r.URL.Path = path
+		vmuiFileServer.ServeHTTP(w, r)
+		return true
+	}
+
+	// Limit the number of concurrent queries, which can consume big amounts of CPU.
+	startTime := time.Now()
+	stopCh := r.Context().Done()
+	select {
+	case concurrencyLimitCh <- struct{}{}:
+		defer func() { <-concurrencyLimitCh }()
+	default:
+		// Sleep for a while until giving up. This should resolve short bursts in requests.
+		concurrencyLimitReached.Inc()
+		d := getMaxQueryDuration(r)
+		if d > *maxQueueDuration {
+			d = *maxQueueDuration
+		}
+		t := timerpool.Get(d)
+		select {
+		case concurrencyLimitCh <- struct{}{}:
+			timerpool.Put(t)
+			defer func() { <-concurrencyLimitCh }()
+		case <-stopCh:
+			timerpool.Put(t)
+			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+			requestURI := httpserver.GetRequestURI(r)
+			logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+				time.Since(startTime).Seconds(), remoteAddr, requestURI)
+			return true
+		case <-t.C:
+			timerpool.Put(t)
+			concurrencyLimitTimeout.Inc()
+			err := &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
+					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
+					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
+					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+	}
+
+	switch {
+	case path == "/logsql/query":
+		logsqlQueryRequests.Inc()
+		httpserver.EnableCORS(w, r)
+		logsql.ProcessQueryRequest(w, r, stopCh)
+		return true
+	default:
+		return false
+	}
+}
+
+// getMaxQueryDuration returns the maximum duration for query from r.
+func getMaxQueryDuration(r *http.Request) time.Duration {
+	dms, err := httputils.GetDuration(r, "timeout", 0)
+	if err != nil {
+		dms = 0
+	}
+	d := time.Duration(dms) * time.Millisecond
+	if d <= 0 || d > *maxQueryDuration {
+		d = *maxQueryDuration
+	}
+	return d
+}
+
+var (
+	logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
+)

app/vlselect/vmui/asset-manifest.json

@@ -0,0 +1,14 @@
+{
+  "files": {
+    "main.css": "./static/css/main.5f461a27.css",
+    "main.js": "./static/js/main.7566144a.js",
+    "static/js/522.b5ae4365.chunk.js": "./static/js/522.b5ae4365.chunk.js",
+    "static/media/Lato-Regular.ttf": "./static/media/Lato-Regular.d714fec1633b69a9c2e9.ttf",
+    "static/media/Lato-Bold.ttf": "./static/media/Lato-Bold.32360ba4b57802daa4d6.ttf",
+    "index.html": "./index.html"
+  },
+  "entrypoints": [
+    "static/css/main.5f461a27.css",
+    "static/js/main.7566144a.js"
+  ]
+}

app/vlselect/vmui/index.html

@@ -0,0 +1 @@
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.7566144a.js"></script><link href="./static/css/main.5f461a27.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/manifest.json

@@ -0,0 +1,20 @@
+{
+  "short_name": "Victoria Metrics UI",
+  "name": "Victoria Metrics UI is a metric explorer for Victoria Metrics",
+  "icons": [
+    {
+      "src": "favicon-32x32.png",
+      "sizes": "32x32",
+      "type": "image/png"
+    },
+    {
+      "src": "apple-touch-icon.png",
+      "type": "image/png",
+      "sizes": "192x192"
+    }
+  ],
+  "start_url": ".",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}

app/vlselect/vmui/robots.txt

@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:

app/vlselect/vmui/static/js/main.7566144a.js.LICENSE.txt

@@ -7,7 +7,7 @@
 /*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
 
 /**
- * @remix-run/router v1.5.0
+ * @remix-run/router v1.7.2
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -18,7 +18,7 @@
  */
 
 /**
- * React Router DOM v6.10.0
+ * React Router DOM v6.14.2
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -29,7 +29,7 @@
  */
 
 /**
- * React Router v6.10.0
+ * React Router v6.14.2
  *
  * Copyright (c) Remix Software Inc.
  *

app/vlstorage/main.go

@@ -0,0 +1,189 @@
+package vlstorage
+
+import (
+	"flag"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
+		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
+		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
+	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
+		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
+	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory with the VictoriaLogs data; "+
+		"see https://docs.victoriametrics.com/VictoriaLogs/#storage")
+	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
+		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
+		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
+		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
+	logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
+		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields ; see also -logIngestedRows")
+	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
+		"see https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/ ; see also -logNewStreams")
+)
+
+// Init initializes vlstorage.
+//
+// Stop must be called when vlstorage is no longer needed
+func Init() {
+	if strg != nil {
+		logger.Panicf("BUG: Init() has been already called")
+	}
+
+	if retentionPeriod.Msecs < 24*3600*1000 {
+		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
+	}
+	cfg := &logstorage.StorageConfig{
+		Retention:       time.Millisecond * time.Duration(retentionPeriod.Msecs),
+		FlushInterval:   *inmemoryDataFlushInterval,
+		FutureRetention: time.Millisecond * time.Duration(futureRetention.Msecs),
+		LogNewStreams:   *logNewStreams,
+		LogIngestedRows: *logIngestedRows,
+	}
+	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
+	startTime := time.Now()
+	strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
+
+	var ss logstorage.StorageStats
+	strg.UpdateStats(&ss)
+	logger.Infof("successfully opened storage in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
+		time.Since(startTime).Seconds(), ss.FileParts, ss.FileBlocks, ss.FileRowsCount, ss.CompressedFileSize)
+	storageMetrics = initStorageMetrics(strg)
+
+	metrics.RegisterSet(storageMetrics)
+}
+
+// Stop stops vlstorage.
+func Stop() {
+	metrics.UnregisterSet(storageMetrics)
+	storageMetrics = nil
+
+	strg.MustClose()
+	strg = nil
+}
+
+var strg *logstorage.Storage
+var storageMetrics *metrics.Set
+
+// MustAddRows adds lr to vlstorage
+func MustAddRows(lr *logstorage.LogRows) {
+	strg.MustAddRows(lr)
+}
+
+// RunQuery runs the given q and calls processBlock for the returned data blocks
+func RunQuery(tenantIDs []logstorage.TenantID, q *logstorage.Query, stopCh <-chan struct{}, processBlock func(columns []logstorage.BlockColumn)) {
+	strg.RunQuery(tenantIDs, q, stopCh, processBlock)
+}
+
+func initStorageMetrics(strg *logstorage.Storage) *metrics.Set {
+	ssCache := &logstorage.StorageStats{}
+	var ssCacheLock sync.Mutex
+	var lastUpdateTime time.Time
+
+	m := func() *logstorage.StorageStats {
+		ssCacheLock.Lock()
+		defer ssCacheLock.Unlock()
+		if time.Since(lastUpdateTime) < time.Second {
+			return ssCache
+		}
+		var ss logstorage.StorageStats
+		strg.UpdateStats(&ss)
+		ssCache = &ss
+		lastUpdateTime = time.Now()
+		return ssCache
+	}
+
+	ms := metrics.NewSet()
+
+	ms.NewGauge(fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), func() float64 {
+		return float64(fs.MustGetFreeSpace(*storageDataPath))
+	})
+
+	ms.NewGauge(`vl_active_merges{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryActiveMerges)
+	})
+	ms.NewGauge(`vl_merges_total{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryMergesTotal)
+	})
+	ms.NewGauge(`vl_active_merges{type="file"}`, func() float64 {
+		return float64(m().FileActiveMerges)
+	})
+	ms.NewGauge(`vl_merges_total{type="file"}`, func() float64 {
+		return float64(m().FileMergesTotal)
+	})
+
+	ms.NewGauge(`vl_storage_rows{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryRowsCount)
+	})
+	ms.NewGauge(`vl_storage_rows{type="file"}`, func() float64 {
+		return float64(m().FileRowsCount)
+	})
+	ms.NewGauge(`vl_storage_parts{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryParts)
+	})
+	ms.NewGauge(`vl_storage_parts{type="file"}`, func() float64 {
+		return float64(m().FileParts)
+	})
+	ms.NewGauge(`vl_storage_blocks{type="inmemory"}`, func() float64 {
+		return float64(m().InmemoryBlocks)
+	})
+	ms.NewGauge(`vl_storage_blocks{type="file"}`, func() float64 {
+		return float64(m().FileBlocks)
+	})
+
+	ms.NewGauge(`vl_partitions`, func() float64 {
+		return float64(m().PartitionsCount)
+	})
+	ms.NewGauge(`vl_streams_created_total`, func() float64 {
+		return float64(m().StreamsCreatedTotal)
+	})
+
+	ms.NewGauge(`vl_indexdb_rows`, func() float64 {
+		return float64(m().IndexdbItemsCount)
+	})
+	ms.NewGauge(`vl_indexdb_parts`, func() float64 {
+		return float64(m().IndexdbPartsCount)
+	})
+	ms.NewGauge(`vl_indexdb_blocks`, func() float64 {
+		return float64(m().IndexdbBlocksCount)
+	})
+
+	ms.NewGauge(`vl_data_size_bytes{type="indexdb"}`, func() float64 {
+		return float64(m().IndexdbSizeBytes)
+	})
+	ms.NewGauge(`vl_data_size_bytes{type="storage"}`, func() float64 {
+		dm := m()
+		return float64(dm.CompressedInmemorySize + dm.CompressedFileSize)
+	})
+
+	ms.NewGauge(`vl_compressed_data_size_bytes{type="inmemory"}`, func() float64 {
+		return float64(m().CompressedInmemorySize)
+	})
+	ms.NewGauge(`vl_compressed_data_size_bytes{type="file"}`, func() float64 {
+		return float64(m().CompressedFileSize)
+	})
+	ms.NewGauge(`vl_uncompressed_data_size_bytes{type="inmemory"}`, func() float64 {
+		return float64(m().UncompressedInmemorySize)
+	})
+	ms.NewGauge(`vl_uncompressed_data_size_bytes{type="file"}`, func() float64 {
+		return float64(m().UncompressedFileSize)
+	})
+
+	ms.NewGauge(`vl_rows_dropped_total{reason="too_big_timestamp"}`, func() float64 {
+		return float64(m().RowsDroppedTooBigTimestamp)
+	})
+	ms.NewGauge(`vl_rows_dropped_total{reason="too_small_timestamp"}`, func() float64 {
+		return float64(m().RowsDroppedTooSmallTimestamp)
+	})
+
+	return ms
+}

app/vmagent/README.md

@@ -54,7 +54,7 @@ and sending the data to the Prometheus-compatible remote storage:
   The path can point either to local file or to http url. `vmagent` doesn't support some sections of Prometheus config file,
   so you may need either to delete these sections or to run `vmagent` with `-promscrape.config.strictParse=false` command-line flag.
   In this case `vmagent` ignores unsupported sections. See [the list of unsupported sections](#unsupported-prometheus-config-sections).
-* `-remoteWrite.url` with Prometheus-compatible remote storage endpoint such as VictoriaMetrics.
+* `-remoteWrite.url` with Prometheus-compatible remote storage endpoint such as VictoriaMetrics, where to send the data to.
 
 Example command for writing the data received via [supported push-based protocols](#how-to-push-data-to-vmagent)
 to [single-node VictoriaMetrics](https://docs.victoriametrics.com/) located at `victoria-metrics-host:8428`:
@@ -93,6 +93,7 @@ additionally to pull-based Prometheus-compatible targets' scraping:
 * DataDog "submit metrics" API. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-datadog-agent).
 * InfluxDB line protocol via `http://<vmagent>:8429/write`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf).
 * Graphite plaintext protocol if `-graphiteListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd).
+* OpenTelemetry http API. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#sending-data-via-opentelemetry).
 * OpenTSDB telnet and http protocols if `-opentsdbListenAddr` command-line flag is set. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-opentsdb-compatible-agents).
 * Prometheus remote write protocol via `http://<vmagent>:8429/api/v1/write`.
 * JSON lines import protocol via `http://<vmagent>:8429/api/v1/import`. See [these docs](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-import-data-in-json-line-format).
@@ -158,6 +159,20 @@ and then it sends the buffered data to the remote storage in order to prevent da
 so there is no need in specifying multiple `-remoteWrite.url` flags when writing data to the same cluster.
 See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#replication-and-data-safety).
 
+### Sharding among remote storages
+
+By default `vmagent` replicates data among remote storage systems enumerated via `-remoteWrite.url` command-line flag.
+If the `-remoteWrite.shardByURL` command-line flag is set, then `vmagent` spreads evenly
+the outgoing [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series)
+among all the remote storage systems enumerated via `-remoteWrite.url`. Note that samples for the same
+time series are routed to the same remote storage system if `-remoteWrite.shardByURL` flag is specified.
+This allows building scalable data processing pipelines when a single remote storage cannot keep up with the data ingestion workload.
+For example, this allows building horizontally scalable [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html)
+by routing outgoing samples for the same time series of [counter](https://docs.victoriametrics.com/keyConcepts.html#counter)
+and [histogram](https://docs.victoriametrics.com/keyConcepts.html#histogram) types from top-level `vmagent` instances
+to the same second-level `vmagent` instance, so they are aggregated properly.
+
+See also [how to scrape big number of targets](#scraping-big-number-of-targets).
 
 ### Relabeling and filtering
 
@@ -465,6 +480,7 @@ with [additional enhancements](#relabeling-enhancements). The relabeling can be
 
   This relabeling can be debugged by clicking the `debug` link at the corresponding target on the `http://vmagent:8429/targets` page
   or on the `http://vmagent:8429/service-discovery` page. See [these docs](#relabel-debug) for details.
+  The link is unavailable if `vmagent` runs with `-promscrape.dropOriginalLabels` command-line flag.
 
 * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file.
   This relabeling is used for modifying labels in scraped metrics and for dropping unneeded metrics.
@@ -512,8 +528,9 @@ The following articles contain useful information about Prometheus relabeling:
   {% endraw %}
 
 * An optional `if` filter can be used for conditional relabeling. The `if` filter may contain
-  arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
-  For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics:
+  arbitrary [time series selector](https://docs.victoriametrics.com/keyConcepts.html#filtering).
+  The `action` is performed only for [samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples), which match the provided `if` filter.
+  For example, the following relabeling rule keeps metrics matching `foo{bar="baz"}` series selector, while dropping the rest of metrics:
 
   ```yaml
   - if: 'foo{bar="baz"}'
@@ -528,6 +545,18 @@ The following articles contain useful information about Prometheus relabeling:
     regex: 'foo;baz'
   ```
 
+  The `if` option may contain more than one filter. In this case the `action` is performed if at least a single filter
+  matches the given [sample](https://docs.victoriametrics.com/keyConcepts.html#raw-samples).
+  For example, the following relabeling rule adds `foo="bar"` label to samples with `job="foo"` or `instance="bar"` labels:
+
+  ```yaml
+  - target_label: foo
+    replacement: bar
+    if:
+    - '{job="foo"}'
+    - '{instance="bar"}'
+  ```
+
 * The `regex` value can be split into multiple lines for improved readability and maintainability.
   These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent:
 
@@ -645,18 +674,20 @@ provide the following tools for debugging target-level and metric-level relabeli
 - Target-level debugging (e.g. `relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs))
   can be performed by navigating to `http://vmagent:8429/targets` page (`http://victoriametrics:8428/targets` page for single-node VictoriaMetrics)
   and clicking the `debug target relabeling` link at the target, which must be debugged.
+  The link is unavailable if `vmagent` runs with `-promscrape.dropOriginalLabels` command-line flag.
   The opened page shows step-by-step results for the actual target relabeling rules applied to the discovered target labels.
   The page shows also the target URL generated after applying all the relabeling rules.
 
   The `http://vmagent:8429/targets` page shows only active targets. If you need to understand why some target
   is dropped during the relabeling, then navigate to `http://vmagent:8428/service-discovery` page
   (`http://victoriametrics:8428/service-discovery` for single-node VictoriaMetrics), find the dropped target
-  and click the `debug` link there. The opened page shows step-by-step results for the actual relabeling rules,
-  which result to target drop.
+  and click the `debug` link there. The link is unavailable if `vmagent` runs with `-promscrape.dropOriginalLabels` command-line flag.
+  The opened page shows step-by-step results for the actual relabeling rules, which result to target drop.
 
 - Metric-level debugging (e.g. `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs)
   can be performed by navigating to `http://vmagent:8429/targets` page (`http://victoriametrics:8428/targets` page for single-node VictoriaMetrics)
   and clicking the `debug metrics relabeling` link at the target, which must be debugged.
+  The link is unavailable if `vmagent` runs with `-promscrape.dropOriginalLabels` command-line flag.
   The opened page shows step-by-step results for the actual metric relabeling rules applied to the given target labels.
 
 ## Prometheus staleness markers
@@ -750,6 +781,9 @@ If each target is scraped by multiple `vmagent` instances, then data deduplicati
 The `-dedup.minScrapeInterval` must be set to the `scrape_interval` configured at `-promscrape.config`.
 See [these docs](https://docs.victoriametrics.com/#deduplication) for details.
 
+See also [how to shard data among multiple remote storage systems](#sharding-among-remote-storages).
+
+
 ## High availability
 
 It is possible to run multiple **identically configured** `vmagent` instances or `vmagent` 
@@ -1097,13 +1131,13 @@ It may be needed to build `vmagent` from source code when developing or testing
 ### Development build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
-2. Run `make vmagent` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make vmagent` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds the `vmagent` binary and puts it into the `bin` folder.
 
 ### Production build
 
 1. [Install docker](https://docs.docker.com/install/).
-2. Run `make vmagent-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make vmagent-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmagent-prod` binary and puts it into the `bin` folder.
 
 ### Building docker images
@@ -1126,13 +1160,13 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 ### Development ARM build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
-2. Run `make vmagent-linux-arm` or `make vmagent-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics)
+1. Run `make vmagent-linux-arm` or `make vmagent-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics)
    It builds `vmagent-linux-arm` or `vmagent-linux-arm64` binary respectively and puts it into the `bin` folder.
 
 ### Production ARM build
 
 1. [Install docker](https://docs.docker.com/install/).
-2. Run `make vmagent-linux-arm-prod` or `make vmagent-linux-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make vmagent-linux-arm-prod` or `make vmagent-linux-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmagent-linux-arm-prod` or `vmagent-linux-arm64-prod` binary respectively and puts it into the `bin` folder.
 
 ## Profiling
@@ -1178,6 +1212,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
 
   -cacheExpireDuration duration
      Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s)
+  -clients.docker
+     Decides whether a docker container be brought up automatically
+  -clients.semaphore
+     Tells if the job is running on Semaphore
   -configAuthKey string
      Authorization key for accessing /config page. It must be passed via authKey query arg
   -csvTrimTimestamp duration
@@ -1192,9 +1230,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -dryRun
      Whether to check config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
   -enableTCP6
-     Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP is used
+     Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable
-     Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
+     Whether to enable reading flags from environment variables in addition to the command line. Command line flag values have priority over values from environment vars. Flags are read only from the command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
@@ -1222,13 +1260,13 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -http.shutdownDelay duration
      Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-     Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+     Password for HTTP server's Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-     Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+     Username for HTTP server's Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
      TCP address to listen for http connections. Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol (default ":8429")
   -httpListenAddr.useProxyProtocol
-     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
+     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing
   -import.maxLineLen size
      The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with 'max_rows_per_line' query arg passed to /api/v1/export
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 104857600)
@@ -1259,7 +1297,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -internStringDisableCache
      Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
+     The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
   -kafka.consumer.topic array
      Kafka topic names for data consumption. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
@@ -1272,6 +1310,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -kafka.consumer.topic.brokers array
      List of brokers to connect for given topic, e.g. -kafka.consumer.topic.broker=host-1:9092;host-2:9092 . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
+  -kafka.consumer.topic.concurrency array
+     Configures consumer concurrency for topic specified via -kafka.consumer.topic flag.This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Supports array of values separated by comma or specified via multiple flags.
   -kafka.consumer.topic.defaultFormat string
      Expected data format in the topic if -kafka.consumer.topic.format is skipped. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html (default "promremotewrite")
   -kafka.consumer.topic.format array
@@ -1303,15 +1344,15 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -maxConcurrentInserts int
-     The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
+     The maximum number of concurrent insert requests. The default value should work for most cases, since it minimizes memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
   -maxInsertRequestSize size
      The maximum size in bytes of a single Prometheus remote_write API request
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
-     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
@@ -1480,6 +1521,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -remoteWrite.headers array
      Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'
      Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.keepDanglingQueues
+     Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.
   -remoteWrite.label array
      Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage
      Supports an array of values separated by comma or specified via multiple flags.
@@ -1524,27 +1567,30 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.relabelConfig string
      Optional path to file with relabeling configs, which are applied to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling
-  -remoteWrite.keepDanglingQueues
-     Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.
   -remoteWrite.roundDigits array
      Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.sendTimeout array
-     Timeout for sending a single block of data to the corresponding -remoteWrite.url
+     Timeout for sending a single block of data to the corresponding -remoteWrite.url (default 1m)
      Supports array of values separated by comma or specified via multiple flags.
+  -remoteWrite.shardByURL
+     Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages
   -remoteWrite.showURL
      Whether to show -remoteWrite.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -remoteWrite.significantFigures array
      The number of significant figures to leave in metric values before writing them to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. This option may be used for improving data compression for the stored metrics. See also -remoteWrite.roundDigits
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.streamAggr.config array
-     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html . See also -remoteWrite.streamAggr.keepInput and -remoteWrite.streamAggr.dedupInterval
+     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html . See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval
      Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.streamAggr.dedupInterval array
      Input samples are de-duplicated with this interval before being aggregated. Only the last sample per each time series per each interval is aggregated if the interval is greater than zero
      Supports array of values separated by comma or specified via multiple flags.
+  -remoteWrite.streamAggr.dropInput array
+     Whether to drop all the input samples after the aggregation with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation.html
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.streamAggr.keepInput array
-     Whether to keep input samples after the aggregation with -remoteWrite.streamAggr.config. By default, the input is dropped after the aggregation, so only the aggregate data is sent to the -remoteWrite.url. See https://docs.victoriametrics.com/stream-aggregation.html
+     Whether to keep all the input samples after the aggregation with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation.html
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.tlsCAFile array
      Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. By default, system CA is used
@@ -1564,7 +1610,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -remoteWrite.tmpDataPath string
      Path to directory where temporary data for remote write component is stored. See also -remoteWrite.maxDiskUsagePerURL (default "vmagent-remotewrite-data")
   -remoteWrite.url array
-     Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. See also -remoteWrite.multitenantURL
+     Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set. See also -remoteWrite.multitenantURL
      Supports an array of values separated by comma or specified via multiple flags.
   -remoteWrite.urlRelabelConfig array
      Optional path to relabel configs for the corresponding -remoteWrite.url. See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling

app/vmagent/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
@@ -308,6 +309,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/opentelemetry/api/v1/push":
+		opentelemetryPushRequests.Inc()
+		if err := opentelemetry.InsertHandler(nil, r); err != nil {
+			opentelemetryPushErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "/datadog/api/v1/series":
 		datadogWriteRequests.Inc()
 		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
@@ -499,6 +509,15 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "opentelemetry/api/v1/push":
+		opentelemetryPushRequests.Inc()
+		if err := opentelemetry.InsertHandler(at, r); err != nil {
+			opentelemetryPushErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "datadog/api/v1/series":
 		datadogWriteRequests.Inc()
 		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
@@ -568,6 +587,9 @@ var (
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
 	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
+	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
+	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/api/v1/push", protocol="opentelemetry"}`)
+
 	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
 

app/vmagent/opentelemetry/request_handler.go

@@ -0,0 +1,65 @@
+package opentelemetry
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
+)
+
+// InsertHandler processes opentelemetry metrics.
+func InsertHandler(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return stream.ParseStream(req.Body, isGzipped, func(tss []prompbmarshal.TimeSeries) error {
+		return insertRows(at, tss, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range tss {
+		ts := &tss[i]
+		rowsTotal += len(ts.Samples)
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		samples = append(samples, ts.Samples...)
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(at, &ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/remotewrite/client.go

@@ -29,7 +29,7 @@ var (
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage")
-	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url (default 1m)")
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
@@ -257,7 +257,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	}
 	authCfg, err := opts.NewConfig()
 	if err != nil {
-		return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
+		return nil, fmt.Errorf("cannot populate auth config for remoteWrite idx: %d, err: %w", argIdx, err)
 	}
 	return authCfg, nil
 }

app/vmagent/remotewrite/remotewrite.go

@@ -33,10 +33,14 @@ import (
 var (
 	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol "+
 		"or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
-		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. See also -remoteWrite.multitenantURL")
+		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. "+
+		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set. "+
+		"See also -remoteWrite.multitenantURL")
 	remoteWriteMultitenantURLs = flagutil.NewArrayString("remoteWrite.multitenantURL", "Base path for multitenant remote storage URL to write data to. "+
 		"See https://docs.victoriametrics.com/vmagent.html#multitenancy for details. Example url: http://<vminsert>:8480 . "+
 		"Pass multiple -remoteWrite.multitenantURL flags in order to replicate data to multiple remote storage systems. See also -remoteWrite.url")
+	shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . "+
+		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#sharding-among-remote-storages")
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory where temporary data for remote write component is stored. "+
 		"See also -remoteWrite.maxDiskUsagePerURL")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
@@ -67,10 +71,13 @@ var (
 
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
 		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
-		"See also -remoteWrite.streamAggr.keepInput and -remoteWrite.streamAggr.dedupInterval")
-	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep input samples after the aggregation with -remoteWrite.streamAggr.config. "+
-		"By default, the input is dropped after the aggregation, so only the aggregate data is sent to the -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/stream-aggregation.html")
+		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
+	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
+		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation.html")
+	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
+		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation.html")
 	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", "Input samples are de-duplicated with this interval before being aggregated. "+
 		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
 )
@@ -93,7 +100,7 @@ func MultitenancyEnabled() bool {
 }
 
 // Contains the current relabelConfigs.
-var allRelabelConfigs atomic.Value
+var allRelabelConfigs atomic.Pointer[relabelConfigs]
 
 // maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
 // since it may lead to high memory usage due to big number of buffers.
@@ -319,7 +326,7 @@ func Stop() {
 // If at is nil, then the data is pushed to the configured `-remoteWrite.url`.
 // If at isn't nil, the data is pushed to the configured `-remoteWrite.multitenantURL`.
 //
-// Note that wr may be modified by Push due to relabeling and rounding.
+// Note that wr may be modified by Push because of relabeling and rounding.
 func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	if at == nil && len(*remoteWriteMultitenantURLs) > 0 {
 		// Write data to default tenant if at isn't set while -remoteWrite.multitenantURL is set.
@@ -346,7 +353,7 @@ func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	}
 
 	var rctx *relabelCtx
-	rcs := allRelabelConfigs.Load().(*relabelConfigs)
+	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
 	if pcsGlobal.Len() > 0 || len(labelsGlobal) > 0 {
 		rctx = getRelabelCtx()
@@ -400,10 +407,46 @@ func pushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarsha
 		// Nothing to push
 		return
 	}
-	// Push block to remote storages in parallel in order to reduce the time needed for sending the data to multiple remote storage systems.
+	if len(rwctxs) == 1 {
+		// Fast path - just push data to the configured single remote storage
+		rwctxs[0].Push(tssBlock)
+		return
+	}
+
+	// We need to push tssBlock to multiple remote storages.
+	// This is either sharding or replication depending on -remoteWrite.shardByURL command-line flag value.
+	if *shardByURL {
+		// Shard the data among rwctxs
+		tssByURL := make([][]prompbmarshal.TimeSeries, len(rwctxs))
+		for _, ts := range tssBlock {
+			h := getLabelsHash(ts.Labels)
+			idx := h % uint64(len(tssByURL))
+			tssByURL[idx] = append(tssByURL[idx], ts)
+		}
+		// Push sharded data to remote storages in parallel in order to reduce
+		// the time needed for sending the data to multiple remote storage systems.
+		var wg sync.WaitGroup
+		wg.Add(len(rwctxs))
+		for i, rwctx := range rwctxs {
+			tssShard := tssByURL[i]
+			if len(tssShard) == 0 {
+				continue
+			}
+			go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
+				defer wg.Done()
+				rwctx.Push(tss)
+			}(rwctx, tssShard)
+		}
+		wg.Wait()
+		return
+	}
+
+	// Replicate data among rwctxs.
+	// Push block to remote storages in parallel in order to reduce
+	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
+	wg.Add(len(rwctxs))
 	for _, rwctx := range rwctxs {
-		wg.Add(1)
 		go func(rwctx *remoteWriteCtx) {
 			defer wg.Done()
 			rwctx.Push(tssBlock)
@@ -507,6 +550,7 @@ type remoteWriteCtx struct {
 
 	sas                 atomic.Pointer[streamaggr.Aggregators]
 	streamAggrKeepInput bool
+	streamAggrDropInput bool
 
 	pss        []*pendingSeries
 	pssNextIdx uint64
@@ -579,6 +623,7 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 		}
 		rwctx.sas.Store(sas)
 		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
+		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
 	}
@@ -587,6 +632,11 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
+	// sas must be stopped before rwctx is closed
+	// because sas can write pending series to rwctx.pss if there are any
+	sas := rwctx.sas.Swap(nil)
+	sas.MustStop()
+
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
 	}
@@ -596,9 +646,6 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.c.MustStop()
 	rwctx.c = nil
 
-	sas := rwctx.sas.Swap(nil)
-	sas.MustStop()
-
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
 
@@ -610,7 +657,7 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 	// Apply relabeling
 	var rctx *relabelCtx
 	var v *[]prompbmarshal.TimeSeries
-	rcs := allRelabelConfigs.Load().(*relabelConfigs)
+	rcs := allRelabelConfigs.Load()
 	pcs := rcs.perURL[rwctx.idx]
 	if pcs.Len() > 0 {
 		rctx = getRelabelCtx()
@@ -618,7 +665,7 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		// from affecting time series for other remoteWrite.url configs.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
-		v = tssRelabelPool.Get().(*[]prompbmarshal.TimeSeries)
+		v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
 		rowsCountBeforeRelabel := getRowsCount(tss)
 		tss = rctx.applyRelabeling(tss, nil, pcs)
@@ -630,20 +677,45 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 
 	// Apply stream aggregation if any
 	sas := rwctx.sas.Load()
-	sas.Push(tss)
-	if sas == nil || rwctx.streamAggrKeepInput {
-		// Push samples to the remote storage
-		rwctx.pushInternal(tss)
+	if sas != nil {
+		matchIdxs := matchIdxsPool.Get()
+		matchIdxs.B = sas.Push(tss, matchIdxs.B)
+		if !rwctx.streamAggrKeepInput {
+			if rctx == nil {
+				rctx = getRelabelCtx()
+				// Make a copy of tss before dropping aggregated series
+				v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
+				tss = append(*v, tss...)
+			}
+			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
+		}
+		matchIdxsPool.Put(matchIdxs)
 	}
+	rwctx.pushInternal(tss)
 
 	// Return back relabeling contexts to the pool
 	if rctx != nil {
 		*v = prompbmarshal.ResetTimeSeries(tss)
-		tssRelabelPool.Put(v)
+		tssPool.Put(v)
 		putRelabelCtx(rctx)
 	}
 }
 
+var matchIdxsPool bytesutil.ByteBufferPool
+
+func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, dropInput bool) []prompbmarshal.TimeSeries {
+	dst := src[:0]
+	for i, match := range matchIdxs {
+		if match == 0 {
+			continue
+		}
+		dst = append(dst, src[i])
+	}
+	tail := src[len(dst):]
+	_ = prompbmarshal.ResetTimeSeries(tail)
+	return dst
+}
+
 func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
@@ -680,7 +752,7 @@ func (rwctx *remoteWriteCtx) reinitStreamAggr() {
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
 }
 
-var tssRelabelPool = &sync.Pool{
+var tssPool = &sync.Pool{
 	New: func() interface{} {
 		a := []prompbmarshal.TimeSeries{}
 		return &a

app/vmalert/README.md

@@ -203,16 +203,22 @@ expr: <string>
 # as firing once they return.
 [ for: <duration> | default = 0s ]
 
+# Alert will continue firing for this long even when the alerting expression no longer has results.
+# This allows you to delay alert resolution.
+[ keep_firing_for: <duration> | default = 0s ]
+
 # Whether to print debug information into logs.
 # Information includes alerts state changes and requests sent to the datasource.
 # Please note, that if rule's query params contain sensitive
 # information - it will be printed to logs.
 # Is applicable to alerting rules only.
+# Available starting from https://docs.victoriametrics.com/CHANGELOG.html#v1820
 [ debug: <bool> | default = false ]
 
 # Defines the number of rule's updates entries stored in memory
 # and available for view on rule's Details page.
 # Overrides `rule.updateEntriesLimit` value for this specific rule.
+# Available starting from https://docs.victoriametrics.com/CHANGELOG.html#v1860
 [ update_entries_limit: <integer> | default 0 ]
 
 # Labels to add or overwrite for each alert.
@@ -355,19 +361,24 @@ For recording rules to work `-remoteWrite.url` must be specified.
 
 ### Alerts state on restarts
 
-`vmalert` has no local storage, so alerts state is stored in the process memory. Hence, after restart of `vmalert`
-the process alerts state will be lost. To avoid this situation, `vmalert` should be configured via the following flags:
+`vmalert` holds alerts state in the memory. Restart of the `vmalert` process will reset the state of all active alerts 
+in the memory. To prevent `vmalert` from losing the state on restarts configure it to persist the state 
+to the remote database via the following flags:
 
 * `-remoteWrite.url` - URL to VictoriaMetrics (Single) or vminsert (Cluster). `vmalert` will persist alerts state
-  into the configured address in the form of time series named `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
-  These are regular time series and maybe queried from VM just as any other time series.
-  The state is stored to the configured address on every rule evaluation.
+  to the configured address in the form of [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series)
+  `ALERTS` and `ALERTS_FOR_STATE` via remote-write protocol.
+  These time series can be queried from VictoriaMetrics just as any other time series.
+  The state will be persisted to the configured address on each evaluation.
 * `-remoteRead.url` - URL to VictoriaMetrics (Single) or vmselect (Cluster). `vmalert` will try to restore alerts state
-  from configured address by querying time series with name `ALERTS_FOR_STATE`.
+  from the configured address by querying time series with name `ALERTS_FOR_STATE`. The restore happens only once when
+  `vmalert` process starts, and only for the configured rules. Config [hot reload](#hot-config-reload) doesn't trigger 
+  state restore.
 
 Both flags are required for proper state restoration. Restore process may fail if time series are missing
 in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by `-remoteRead.lookback`)
-or received state doesn't match current `vmalert` rules configuration.
+or received state doesn't match current `vmalert` rules configuration. `vmalert` marks successfully restored rules
+with `restored` label in [web UI](#web).
 
 ### Multitenancy
 
@@ -508,7 +519,7 @@ Alertmanagers.
 
 To avoid recording rules results and alerts state duplication in VictoriaMetrics server
 don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
-The recommended value for `-dedup.minScrapeInterval` must be greater or equal to vmalert `evaluation_interval`.
+The recommended value for `-dedup.minScrapeInterval` must be multiple of vmalert's `evaluation_interval`.
 If you observe inconsistent or "jumping" values in series produced by vmalert, try disabling `-datasource.queryTimeAlignment`
 command line flag. Because of alignment, two or more vmalert HA pairs will produce results with the same timestamps.
 But due of backfilling (data delivered to the datasource with some delay) values of such results may differ,
@@ -526,6 +537,9 @@ Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics)
 
 #### Downsampling and aggregation via vmalert
 
+_Please note, [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html) might be more efficient
+for cases when downsampling or aggregation need to be applied **before data gets into the TSDB.**_
+
 `vmalert` can't modify existing data. But it can run arbitrary PromQL/MetricsQL queries
 via [recording rules](#recording-rules) and backfill results to the configured `-remoteWrite.url`.
 This ability allows to aggregate data. For example, the following rule will calculate the average value for
@@ -726,6 +740,7 @@ See full description for these flags in `./vmalert -help`.
 * Graphite engine isn't supported yet;
 * `query` template function is disabled for performance reasons (might be changed in future);
 * `limit` group's param has no effect during replay (might be changed in future);
+* `keep_firing_for` alerting rule param has no effect during replay (might be changed in future).
 
 ## Monitoring
 
@@ -789,7 +804,8 @@ If `-remoteWrite.url` command-line flag is configured, vmalert will persist aler
 changed in time.
 
 vmalert stores last `-rule.updateEntriesLimit` (or `update_entries_limit` [per-rule config](https://docs.victoriametrics.com/vmalert.html#alerting-rules)) 
-state updates for each rule. To check updates, click on `Details` link next to rule's name on `/vmalert/groups` page 
+state updates for each rule starting from [v1.86](https://docs.victoriametrics.com/CHANGELOG.html#v1860).
+To check updates, click on `Details` link next to rule's name on `/vmalert/groups` page 
 and check the `Last updates` section:
 
 <img alt="vmalert state" src="vmalert_state.png">
@@ -801,8 +817,8 @@ moment when rule was evaluated.
 
 ### Debug mode
 
-vmalert allows configuring more detailed logging for specific alerting rule. Just set `debug: true` in rule's configuration
-and vmalert will start printing additional log messages:
+vmalert allows configuring more detailed logging for specific alerting rule starting from [v1.82](https://docs.victoriametrics.com/CHANGELOG.html#v1820).
+Just set `debug: true` in rule's configuration and vmalert will start printing additional log messages:
 ```terminal
 2022-09-15T13:35:41.155Z  DEBUG rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:41+02:00: query returned 0 samples (elapsed: 5.896041ms)
 2022-09-15T13:35:56.149Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29+by%28instance%29+%3E+0&step=15s&time=1663248945"
@@ -815,7 +831,8 @@ and vmalert will start printing additional log messages:
 
 ### Never-firing alerts
 
-vmalert can detect if alert's expression doesn't match any time series in runtime. This problem usually happens
+vmalert can detect if alert's expression doesn't match any time series in runtime 
+starting from [v1.91](https://docs.victoriametrics.com/CHANGELOG.html#v1910). This problem usually happens
 when alerting expression selects time series which aren't present in the datasource (i.e. wrong `job` label)
 or there is a typo in the series selector (i.e. `env=rpod`). Such alerting rules will be marked with special icon in 
 vmalerts UI and exposed via `vmalert_alerting_rules_last_evaluation_series_fetched` metric. The metric value will
@@ -829,6 +846,32 @@ max(vmalert_alerting_rules_last_evaluation_series_fetched) by(group, alertname)
 See more details [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4039).
 This feature is available only if vmalert is using VictoriaMetrics v1.90 or higher as a datasource.
 
+### Series with the same labelset
+
+vmalert can produce the following error message during rules evaluation:
+```
+result contains metrics with the same labelset after applying rule labels
+```
+
+The error means there is a collision between [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series)
+after applying extra labels to result.
+
+For example, a rule with `expr: foo > 0` returns two distinct time series in response:
+```
+foo{bar="baz"} 1
+foo{bar="qux"} 2
+```
+
+If user configures `-external.label=bar=baz` cmd-line flag to enforce
+adding `bar="baz"` label-value pair, then time series won't be distinct anymore:
+```
+foo{bar="baz"} 1
+foo{bar="baz"} 2 # 'bar' label was overriden by `-external.label=bar=baz
+```
+
+The same issue can be caused by collision of configured `labels` on [Group](#groups) or [Rule](#rules) levels.
+To fix it one should avoid collisions by carefully picking label overrides in configuration.
+
 
 ## Profiling
 
@@ -868,7 +911,7 @@ command-line flags with their descriptions.
 
 The shortlist of configuration flags is the following:
 {% raw  %}
-```
+```console
   -clusterMode
      If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/vmalert.html#multitenancy . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -configCheckInterval duration
@@ -887,6 +930,8 @@ The shortlist of configuration flags is the following:
      Optional path to bearer token file to use for -datasource.url.
   -datasource.disableKeepAlive
      Whether to disable long-lived connections to the datasource. If true, disables HTTP keep-alives and will only use the connection to the server for a single HTTP request.
+  -datasource.disableStepParam
+     Whether to disable adding 'step' param to the issued instant queries. This might be useful when using vmalert with datasources that do not support 'step' param for instant queries, like Google Managed Prometheus. It is not recommended to enable this flag if you use vmalert with VictoriaMetrics.
   -datasource.headers string
      Optional HTTP extraHeaders to send with each request to the corresponding -datasource.url. For example, -datasource.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -datasource.url. Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'
   -datasource.lookback duration
@@ -932,9 +977,9 @@ The shortlist of configuration flags is the following:
   -dryRun
      Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.
   -enableTCP6
-     Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP is used
+     Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable
-     Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
+     Whether to enable reading flags from environment variables in addition to the command line. Command line flag values have priority over values from environment vars. Flags are read only from the command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
@@ -965,17 +1010,19 @@ The shortlist of configuration flags is the following:
   -http.shutdownDelay duration
      Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-     Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+     Password for HTTP server's Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-     Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+     Username for HTTP server's Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
      Address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8880")
   -httpListenAddr.useProxyProtocol
-     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
-  -insert.maxQueueDuration duration
-     The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
+     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing
+  -internStringCacheExpireDuration duration
+     The expiry duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
+  -internStringDisableCache
+     Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 500)
+     The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -992,13 +1039,11 @@ The shortlist of configuration flags is the following:
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
-  -maxConcurrentInserts int
-     The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
-     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
@@ -1053,6 +1098,8 @@ The shortlist of configuration flags is the following:
   -notifier.url array
      Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.
      Supports an array of values separated by comma or specified via multiple flags.
+  -notifier.blackhole bool
+     Whether to blackhole alerting notifications. Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). `-notifier.url`, `-notifier.config` and `-notifier.blackhole` are mutually exclusive.
   -pprofAuthKey string
      Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -promscrape.consul.waitTime duration
@@ -1147,6 +1194,10 @@ The shortlist of configuration flags is the following:
      Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.
   -remoteWrite.oauth2.tokenUrl string
      Optional OAuth2 tokenURL to use for -notifier.url.
+  -remoteWrite.retryMaxTime duration
+     The max time spent on retry attempts for the failed remote-write request. Change this value if it is expected for remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval (default 30s)
+  -remoteWrite.retryMinInterval duration
+     The minimum delay between retry attempts. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxInterval (default 1s)
   -remoteWrite.sendTimeout duration
      Timeout for sending data to the configured -remoteWrite.url. (default 30s)
   -remoteWrite.showURL
@@ -1176,12 +1227,12 @@ The shortlist of configuration flags is the following:
   -replay.timeTo string
      The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
   -rule array
-     Path to the files with alerting and/or recording rules.
+     Path to the files or http url with alerting and/or recording rules.
      Supports hierarchical patterns and regexpes.
      Examples:
       -rule="/path/to/file". Path to a single file with alerting rules.
       -rule="http://<some-server-addr>/path/to/rules". HTTP URL to a page with alerting rules.
-      -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod".
+      -rule="dir/*.yaml" -rule="/*.yaml" -rule="gcs://vmalert-rules/tenant_%{TENANT_ID}/prod". 
       -rule="dir/**/*.yaml". Includes all the .yaml files in "dir" subfolders recursively.
      Rule files may contain %{ENV_VAR} placeholders, which are substituted by the corresponding env vars.
      
@@ -1192,20 +1243,19 @@ The shortlist of configuration flags is the following:
      See https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage
      
      Supports an array of values separated by comma or specified via multiple flags.
-  -rule.configCheckInterval duration
-     Interval for checking for changes in '-rule' files. By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead
   -rule.maxResolveDuration duration
      Limits the maximum duration for automatic alert expiration, which by default is 4 times evaluationInterval of the parent group.
   -rule.resendDelay duration
      Minimum amount of time to wait before resending an alert to notifier
   -rule.templates array
      Path or glob pattern to location with go template definitions
-      for rules annotations templating. Flag can be specified multiple times.
+     	for rules annotations templating. Flag can be specified multiple times.
      Examples:
       -rule.templates="/path/to/file". Path to a single file with go templates
       -rule.templates="dir/*.tpl" -rule.templates="/*.tpl". Relative path to all .tpl files in "dir" folder,
      absolute path to all .tpl files in root.
       -rule.templates="dir/**/*.tpl". Includes all the .tpl files in "dir" subfolders recursively.
+     
      Supports an array of values separated by comma or specified via multiple flags.
   -rule.updateEntriesLimit int
      Defines the max number of rule's state updates stored in-memory. Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param. (default 20)
@@ -1213,6 +1263,10 @@ The shortlist of configuration flags is the following:
      Whether to validate rules expressions via MetricsQL engine (default true)
   -rule.validateTemplates
      Whether to validate annotation and label templates (default true)
+  -s2a_enable_appengine_dialer
+     If true, opportunistically use AppEngine-specific dialer to call S2A.
+  -s2a_timeout duration
+     Timeout enforced on the connection to the S2A service for handshake. (default 3s)
   -s3.configFilePath string
      Path to file with S3 configs. Configs are loaded from default location if not set.
      See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
@@ -1285,7 +1339,7 @@ The configuration file allows to configure static notifiers, discover notifiers
 and [DNS](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config):
 For example:
 
-```
+```yaml
 static_configs:
   - targets:
       - localhost:9093
@@ -1303,14 +1357,14 @@ dns_sd_configs:
     port: 9093
 ```
 
-The list of configured or discovered Notifiers can be explored via [UI](#Web).
+The list of configured or discovered Notifiers can be explored via [UI](#web).
 If Alertmanager runs in cluster mode then all its URLs needs to be available during discovery
 to ensure [high availability](https://github.com/prometheus/alertmanager#high-availability).
 
 The configuration file [specification](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/notifier/config.go)
 is the following:
 
-```
+```yaml
 # Per-target Notifier timeout when pushing alerts.
 [ timeout: <duration> | default = 10s ]
 
@@ -1429,6 +1483,7 @@ docker push my-repo:my-version-name
 ```
 
 To run the built image in `victoria-metrics-k8s-stack` or `VMAlert` CR object apply the following config change:
+
 ```yaml
 kind: VMAlert
 spec:
@@ -1440,13 +1495,13 @@ spec:
 ### Development build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
-2. Run `make vmalert` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make vmalert` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert` binary and puts it into the `bin` folder.
 
 ### Production build
 
 1. [Install docker](https://docs.docker.com/install/).
-2. Run `make vmalert-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make vmalert-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert-prod` binary and puts it into the `bin` folder.
 
 ### ARM build
@@ -1456,11 +1511,11 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 ### Development ARM build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
-2. Run `make vmalert-linux-arm` or `make vmalert-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make vmalert-linux-arm` or `make vmalert-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert-linux-arm` or `vmalert-linux-arm64` binary respectively and puts it into the `bin` folder.
 
 ### Production ARM build
 
 1. [Install docker](https://docs.docker.com/install/).
-2. Run `make vmalert-linux-arm-prod` or `make vmalert-linux-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make vmalert-linux-arm-prod` or `make vmalert-linux-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert-linux-arm-prod` or `vmalert-linux-arm64-prod` binary respectively and puts it into the `bin` folder.

app/vmalert/alerting.go

@@ -21,17 +21,18 @@ import (
 
 // AlertingRule is basic alert entity
 type AlertingRule struct {
-	Type         config.Type
-	RuleID       uint64
-	Name         string
-	Expr         string
-	For          time.Duration
-	Labels       map[string]string
-	Annotations  map[string]string
-	GroupID      uint64
-	GroupName    string
-	EvalInterval time.Duration
-	Debug        bool
+	Type          config.Type
+	RuleID        uint64
+	Name          string
+	Expr          string
+	For           time.Duration
+	KeepFiringFor time.Duration
+	Labels        map[string]string
+	Annotations   map[string]string
+	GroupID       uint64
+	GroupName     string
+	EvalInterval  time.Duration
+	Debug         bool
 
 	q datasource.Querier
 
@@ -56,17 +57,18 @@ type alertingRuleMetrics struct {
 
 func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
 	ar := &AlertingRule{
-		Type:         group.Type,
-		RuleID:       cfg.ID,
-		Name:         cfg.Alert,
-		Expr:         cfg.Expr,
-		For:          cfg.For.Duration(),
-		Labels:       cfg.Labels,
-		Annotations:  cfg.Annotations,
-		GroupID:      group.ID(),
-		GroupName:    group.Name,
-		EvalInterval: group.Interval,
-		Debug:        cfg.Debug,
+		Type:          group.Type,
+		RuleID:        cfg.ID,
+		Name:          cfg.Alert,
+		Expr:          cfg.Expr,
+		For:           cfg.For.Duration(),
+		KeepFiringFor: cfg.KeepFiringFor.Duration(),
+		Labels:        cfg.Labels,
+		Annotations:   cfg.Annotations,
+		GroupID:       group.ID(),
+		GroupName:     group.Name,
+		EvalInterval:  group.Interval,
+		Debug:         cfg.Debug,
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
@@ -366,6 +368,7 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			if err != nil {
 				return nil, err
 			}
+			a.KeepFiringSince = time.Time{}
 			continue
 		}
 		a, err := ar.newAlert(m, ls, start, qFn)
@@ -391,12 +394,24 @@ func (ar *AlertingRule) Exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				ar.logDebugf(ts, a, "PENDING => DELETED: is absent in current evaluation round")
 				continue
 			}
+			// check if alert should keep StateFiring if rule has
+			// `keep_firing_for` field
 			if a.State == notifier.StateFiring {
-				a.State = notifier.StateInactive
-				a.ResolvedAt = ts
-				ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
+				if ar.KeepFiringFor > 0 {
+					if a.KeepFiringSince.IsZero() {
+						a.KeepFiringSince = ts
+					}
+				}
+				// alerts with ar.KeepFiringFor>0 may remain FIRING
+				// even if their expression isn't true anymore
+				if ts.Sub(a.KeepFiringSince) > ar.KeepFiringFor {
+					a.State = notifier.StateInactive
+					a.ResolvedAt = ts
+					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
+					continue
+				}
+				ar.logDebugf(ts, a, "KEEP_FIRING: will keep firing for %fs since %v", ar.KeepFiringFor.Seconds(), a.KeepFiringSince)
 			}
-			continue
 		}
 		numActivePending++
 		if a.State == notifier.StatePending && ts.Sub(a.ActiveAt) >= ar.For {
@@ -436,6 +451,7 @@ func (ar *AlertingRule) UpdateWith(r Rule) error {
 	}
 	ar.Expr = nr.Expr
 	ar.For = nr.For
+	ar.KeepFiringFor = nr.KeepFiringFor
 	ar.Labels = nr.Labels
 	ar.Annotations = nr.Annotations
 	ar.EvalInterval = nr.EvalInterval
@@ -508,6 +524,7 @@ func (ar *AlertingRule) ToAPI() APIRule {
 		Name:              ar.Name,
 		Query:             ar.Expr,
 		Duration:          ar.For.Seconds(),
+		KeepFiringFor:     ar.KeepFiringFor.Seconds(),
 		Labels:            ar.Labels,
 		Annotations:       ar.Annotations,
 		LastEvaluation:    lastState.time,
@@ -576,6 +593,9 @@ func (ar *AlertingRule) newAlertAPI(a notifier.Alert) *APIAlert {
 	if alertURLGeneratorFn != nil {
 		aa.SourceLink = alertURLGeneratorFn(a)
 	}
+	if a.State == notifier.StateFiring && !a.KeepFiringSince.IsZero() {
+		aa.Stabilizing = true
+	}
 	return aa
 }
 

app/vmalert/alerting_test.go

@@ -113,7 +113,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 	testCases := []struct {
 		rule      *AlertingRule
 		steps     [][]datasource.Metric
-		expAlerts []testAlert
+		expAlerts map[int][]testAlert
 	}{
 		{
 			newTestAlertingRule("empty", 0),
@@ -125,50 +125,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
 			},
-			[]testAlert{
-				{alert: &notifier.Alert{State: notifier.StateFiring}},
-			},
-		},
-		{
-			newTestAlertingRule("single-firing", 0),
-			[][]datasource.Metric{
-				{metricWithLabels(t, "name", "foo")},
-			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-			},
-		},
-		{
-			newTestAlertingRule("single-firing=>inactive", 0),
-			[][]datasource.Metric{
-				{metricWithLabels(t, "name", "foo")},
-				{},
-			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-			},
-		},
-		{
-			newTestAlertingRule("single-firing=>inactive=>firing", 0),
-			[][]datasource.Metric{
-				{metricWithLabels(t, "name", "foo")},
-				{},
-				{metricWithLabels(t, "name", "foo")},
-			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-			},
-		},
-		{
-			newTestAlertingRule("single-firing=>inactive=>firing=>inactive", 0),
-			[][]datasource.Metric{
-				{metricWithLabels(t, "name", "foo")},
-				{},
-				{metricWithLabels(t, "name", "foo")},
-				{},
-			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			map[int][]testAlert{
+				0: {{alert: &notifier.Alert{State: notifier.StateFiring}}},
 			},
 		},
 		{
@@ -180,12 +138,16 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{},
 			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+			map[int][]testAlert{
+				0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+				2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+				4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 			},
 		},
 		{
-			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>empty=>firing", 0),
+			newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive=>firing", 0),
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{},
@@ -194,8 +156,13 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			map[int][]testAlert{
+				0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+				2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+				4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+				5: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 			},
 		},
 		{
@@ -207,10 +174,12 @@ func TestAlertingRule_Exec(t *testing.T) {
 					metricWithLabels(t, "name", "foo2"),
 				},
 			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			map[int][]testAlert{
+				0: {
+					{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+					{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+					{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				},
 			},
 		},
 		{
@@ -223,10 +192,19 @@ func TestAlertingRule_Exec(t *testing.T) {
 			// 1: fire first alert
 			// 2: fire second alert, set first inactive
 			// 3: fire third alert, set second inactive
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			map[int][]testAlert{
+				0: {
+					{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				},
+				1: {
+					{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+					{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				},
+				2: {
+					{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+					{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+					{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+				},
 			},
 		},
 		{
@@ -234,8 +212,8 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
+			map[int][]testAlert{
+				0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 			},
 		},
 		{
@@ -244,8 +222,9 @@ func TestAlertingRule_Exec(t *testing.T) {
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			map[int][]testAlert{
+				0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+				1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 			},
 		},
 		{
@@ -253,34 +232,13 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
-				// empty step to reset and delete pending alerts
+				// empty step to delete pending alerts
 				{},
 			},
-			nil,
-		},
-		{
-			newTestAlertingRule("for-pending=>firing=>inactive", defaultStep),
-			[][]datasource.Metric{
-				{metricWithLabels(t, "name", "foo")},
-				{metricWithLabels(t, "name", "foo")},
-				// empty step to reset pending alerts
-				{},
-			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-			},
-		},
-		{
-			newTestAlertingRule("for-pending=>firing=>inactive=>pending", defaultStep),
-			[][]datasource.Metric{
-				{metricWithLabels(t, "name", "foo")},
-				{metricWithLabels(t, "name", "foo")},
-				// empty step to reset pending alerts
-				{},
-				{metricWithLabels(t, "name", "foo")},
-			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}},
+			map[int][]testAlert{
+				0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+				1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+				2: {},
 			},
 		},
 		{
@@ -288,13 +246,57 @@ func TestAlertingRule_Exec(t *testing.T) {
 			[][]datasource.Metric{
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
-				// empty step to reset pending alerts
+				// empty step to set alert inactive
 				{},
 				{metricWithLabels(t, "name", "foo")},
 				{metricWithLabels(t, "name", "foo")},
 			},
-			[]testAlert{
-				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			map[int][]testAlert{
+				0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+				1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+				3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+				4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+			},
+		},
+		{
+			newTestAlertingRuleWithKeepFiring("for-pending=>firing=>keepfiring=>firing", defaultStep, defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to keep firing
+				{},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[int][]testAlert{
+				0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+				1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+			},
+		},
+		{
+			newTestAlertingRuleWithKeepFiring("for-pending=>firing=>keepfiring=>keepfiring=>inactive=>pending=>firing", defaultStep, 2*defaultStep),
+			[][]datasource.Metric{
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+				// empty step to keep firing
+				{},
+				// another empty step to keep firing
+				{},
+				// empty step to set alert inactive
+				{},
+				{metricWithLabels(t, "name", "foo")},
+				{metricWithLabels(t, "name", "foo")},
+			},
+			map[int][]testAlert{
+				0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+				1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
+				4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
+				5: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
+				6: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 			},
 		},
 	}
@@ -304,7 +306,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			fq := &fakeQuerier{}
 			tc.rule.q = fq
 			tc.rule.GroupID = fakeGroup.ID()
-			for _, step := range tc.steps {
+			for i, step := range tc.steps {
 				fq.reset()
 				fq.add(step...)
 				if _, err := tc.rule.Exec(context.TODO(), time.Now(), 0); err != nil {
@@ -312,28 +314,31 @@ func TestAlertingRule_Exec(t *testing.T) {
 				}
 				// artificial delay between applying steps
 				time.Sleep(defaultStep)
-			}
-			if len(tc.rule.alerts) != len(tc.expAlerts) {
-				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
-			}
-			expAlerts := make(map[uint64]*notifier.Alert)
-			for _, ta := range tc.expAlerts {
-				labels := make(map[string]string)
-				for i := 0; i < len(ta.labels); i += 2 {
-					k, v := ta.labels[i], ta.labels[i+1]
-					labels[k] = v
+				if _, ok := tc.expAlerts[i]; !ok {
+					continue
 				}
-				labels[alertNameLabel] = tc.rule.Name
-				h := hash(labels)
-				expAlerts[h] = ta.alert
-			}
-			for key, exp := range expAlerts {
-				got, ok := tc.rule.alerts[key]
-				if !ok {
-					t.Fatalf("expected to have key %d", key)
+				if len(tc.rule.alerts) != len(tc.expAlerts[i]) {
+					t.Fatalf("evalIndex %d: expected %d alerts; got %d", i, len(tc.expAlerts[i]), len(tc.rule.alerts))
 				}
-				if got.State != exp.State {
-					t.Fatalf("expected state %d; got %d", exp.State, got.State)
+				expAlerts := make(map[uint64]*notifier.Alert)
+				for _, ta := range tc.expAlerts[i] {
+					labels := make(map[string]string)
+					for i := 0; i < len(ta.labels); i += 2 {
+						k, v := ta.labels[i], ta.labels[i+1]
+						labels[k] = v
+					}
+					labels[alertNameLabel] = tc.rule.Name
+					h := hash(labels)
+					expAlerts[h] = ta.alert
+				}
+				for key, exp := range expAlerts {
+					got, ok := tc.rule.alerts[key]
+					if !ok {
+						t.Fatalf("evalIndex %d: expected to have key %d", i, key)
+					}
+					if got.State != exp.State {
+						t.Fatalf("evalIndex %d: expected state %d; got %d", i, exp.State, got.State)
+					}
 				}
 			}
 		})
@@ -866,8 +871,8 @@ func TestAlertingRule_Template(t *testing.T) {
 			for hash, expAlert := range tc.expAlerts {
 				gotAlert := tc.rule.alerts[hash]
 				if gotAlert == nil {
-					t.Fatalf("alert %d is missing; labels: %v; annotations: %v",
-						hash, expAlert.Labels, expAlert.Annotations)
+					t.Fatalf("alert %d is missing; labels: %v; annotations: %v", hash, expAlert.Labels, expAlert.Annotations)
+					break
 				}
 				if !reflect.DeepEqual(expAlert.Annotations, gotAlert.Annotations) {
 					t.Fatalf("expected to have annotations %#v; got %#v", expAlert.Annotations, gotAlert.Annotations)
@@ -970,11 +975,18 @@ func newTestRuleWithLabels(name string, labels ...string) *AlertingRule {
 }
 
 func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
-	return &AlertingRule{
+	rule := AlertingRule{
 		Name:         name,
 		For:          waitFor,
 		EvalInterval: waitFor,
 		alerts:       make(map[uint64]*notifier.Alert),
 		state:        newRuleState(10),
 	}
+	return &rule
+}
+
+func newTestAlertingRuleWithKeepFiring(name string, waitFor, keepFiringFor time.Duration) *AlertingRule {
+	rule := newTestAlertingRule(name, waitFor)
+	rule.KeepFiringFor = keepFiringFor
+	return rule
 }

app/vmalert/config/config.go

@@ -105,14 +105,16 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 // Rule describes entity that represent either
 // recording rule or alerting rule.
 type Rule struct {
-	ID          uint64
-	Record      string              `yaml:"record,omitempty"`
-	Alert       string              `yaml:"alert,omitempty"`
-	Expr        string              `yaml:"expr"`
-	For         *promutils.Duration `yaml:"for,omitempty"`
-	Labels      map[string]string   `yaml:"labels,omitempty"`
-	Annotations map[string]string   `yaml:"annotations,omitempty"`
-	Debug       bool                `yaml:"debug,omitempty"`
+	ID     uint64
+	Record string              `yaml:"record,omitempty"`
+	Alert  string              `yaml:"alert,omitempty"`
+	Expr   string              `yaml:"expr"`
+	For    *promutils.Duration `yaml:"for,omitempty"`
+	// Alert will continue firing for this long even when the alerting expression no longer has results.
+	KeepFiringFor *promutils.Duration `yaml:"keep_firing_for,omitempty"`
+	Labels        map[string]string   `yaml:"labels,omitempty"`
+	Annotations   map[string]string   `yaml:"annotations,omitempty"`
+	Debug         bool                `yaml:"debug,omitempty"`
 	// UpdateEntriesLimit defines max number of rule's state updates stored in memory.
 	// Overrides `-rule.updateEntriesLimit`.
 	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`

app/vmalert/config/config_test.go

@@ -404,7 +404,7 @@ func TestHashRule(t *testing.T) {
 			true,
 		},
 		{
-			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute)},
 			Rule{Alert: "alert", Expr: "up == 1"},
 			true,
 		},

app/vmalert/datasource/datasource.go

@@ -1,9 +1,12 @@
 package datasource
 
 import (
+	"bytes"
 	"context"
 	"net/http"
 	"net/url"
+	"sort"
+	"strconv"
 	"time"
 )
 
@@ -108,3 +111,69 @@ type Label struct {
 	Name  string
 	Value string
 }
+
+// Labels is collection of Label
+type Labels []Label
+
+func (ls Labels) Len() int           { return len(ls) }
+func (ls Labels) Swap(i, j int)      { ls[i], ls[j] = ls[j], ls[i] }
+func (ls Labels) Less(i, j int) bool { return ls[i].Name < ls[j].Name }
+
+func (ls Labels) String() string {
+	var b bytes.Buffer
+
+	b.WriteByte('{')
+	for i, l := range ls {
+		if i > 0 {
+			b.WriteByte(',')
+			b.WriteByte(' ')
+		}
+		b.WriteString(l.Name)
+		b.WriteByte('=')
+		b.WriteString(strconv.Quote(l.Value))
+	}
+	b.WriteByte('}')
+	return b.String()
+}
+
+// LabelCompare return negative if a is less than b, return 0 if they are the same
+// eg.
+// a=[]Label{{Name: "a", Value: "1"}},b=[]Label{{Name: "b", Value: "1"}}, return -1
+// a=[]Label{{Name: "a", Value: "2"}},b=[]Label{{Name: "a", Value: "1"}}, return 1
+// a=[]Label{{Name: "a", Value: "1"}},b=[]Label{{Name: "a", Value: "1"}}, return 0
+func LabelCompare(a, b Labels) int {
+	l := len(a)
+	if len(b) < l {
+		l = len(b)
+	}
+
+	for i := 0; i < l; i++ {
+		if a[i].Name != b[i].Name {
+			if a[i].Name < b[i].Name {
+				return -1
+			}
+			return 1
+		}
+		if a[i].Value != b[i].Value {
+			if a[i].Value < b[i].Value {
+				return -1
+			}
+			return 1
+		}
+	}
+	// if all labels so far were in common, the set with fewer labels comes first.
+	return len(a) - len(b)
+}
+
+// ConvertToLabels convert map to Labels
+func ConvertToLabels(m map[string]string) (labelset Labels) {
+	for k, v := range m {
+		labelset = append(labelset, Label{
+			Name:  k,
+			Value: v,
+		})
+	}
+	// sort label
+	sort.Slice(labelset, func(i, j int) bool { return labelset[i].Name < labelset[j].Name })
+	return
+}

app/vmalert/datasource/vm_graphite_api.go

@@ -54,6 +54,16 @@ func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestam
 	}
 	r.URL.Path += graphitePath
 	q := r.URL.Query()
+	from := "-5min"
+	if s.lookBack > 0 {
+		lookBack := timestamp.Add(-s.lookBack)
+		from = strconv.FormatInt(lookBack.Unix(), 10)
+	}
+	q.Set("from", from)
+	q.Set("format", "json")
+	q.Set("target", query)
+	q.Set("until", "now")
+
 	for k, vs := range s.extraParams {
 		if q.Has(k) { // extraParams are prior to params in URL
 			q.Del(k)
@@ -62,14 +72,6 @@ func (s *VMStorage) setGraphiteReqParams(r *http.Request, query string, timestam
 			q.Add(k, v)
 		}
 	}
-	q.Set("format", "json")
-	q.Set("target", query)
-	from := "-5min"
-	if s.lookBack > 0 {
-		lookBack := timestamp.Add(-s.lookBack)
-		from = strconv.FormatInt(lookBack.Unix(), 10)
-	}
-	q.Set("from", from)
-	q.Set("until", "now")
+
 	r.URL.RawQuery = q.Encode()
 }

app/vmalert/datasource/vm_prom_api.go

@@ -12,6 +12,9 @@ import (
 var (
 	disablePathAppend = flag.Bool("remoteRead.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/query' path "+
 		"to the configured -datasource.url and -remoteRead.url")
+	disableStepParam = flag.Bool("datasource.disableStepParam", false, "Whether to disable adding 'step' param to the issued instant queries. "+
+		"This might be useful when using vmalert with datasources that do not support 'step' param for instant queries, like Google Managed Prometheus. "+
+		"It is not recommended to enable this flag if you use vmalert with VictoriaMetrics.")
 )
 
 type promResponse struct {
@@ -165,13 +168,13 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
 		timestamp = timestamp.Truncate(s.evaluationInterval)
 	}
-	q.Set("time", fmt.Sprintf("%d", timestamp.Unix()))
-	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
+	q.Set("time", timestamp.Format(time.RFC3339))
+	if !*disableStepParam && s.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
 		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
 	}
-	if s.queryStep > 0 { // override step with user-specified value
+	if !*disableStepParam && s.queryStep > 0 { // override step with user-specified value
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
 		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
@@ -188,8 +191,8 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s
 		r.URL.Path += "/api/v1/query_range"
 	}
 	q := r.URL.Query()
-	q.Add("start", fmt.Sprintf("%d", start.Unix()))
-	q.Add("end", fmt.Sprintf("%d", end.Unix()))
+	q.Add("start", start.Format(time.RFC3339))
+	q.Add("end", end.Format(time.RFC3339))
 	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943

app/vmalert/datasource/vm_test.go

@@ -8,7 +8,6 @@ import (
 	"net/url"
 	"reflect"
 	"sort"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -50,8 +49,8 @@ func TestVMInstantQuery(t *testing.T) {
 		if timeParam == "" {
 			t.Errorf("expected 'time' in query param, got nil instead")
 		}
-		if _, err := strconv.ParseInt(timeParam, 10, 64); err != nil {
-			t.Errorf("failed to parse 'time' query param: %s", err)
+		if _, err := time.Parse(time.RFC3339, timeParam); err != nil {
+			t.Errorf("failed to parse 'time' query param %q: %s", timeParam, err)
 		}
 		switch c {
 		case 0:
@@ -193,7 +192,6 @@ func TestVMInstantQuery(t *testing.T) {
 		},
 	}
 	metricsEqual(t, res.Data, exp)
-
 }
 
 func TestVMInstantQueryWithRetry(t *testing.T) {
@@ -309,14 +307,14 @@ func TestVMRangeQuery(t *testing.T) {
 		if startTS == "" {
 			t.Errorf("expected 'start' in query param, got nil instead")
 		}
-		if _, err := strconv.ParseInt(startTS, 10, 64); err != nil {
+		if _, err := time.Parse(time.RFC3339, startTS); err != nil {
 			t.Errorf("failed to parse 'start' query param: %s", err)
 		}
 		endTS := r.URL.Query().Get("end")
 		if endTS == "" {
 			t.Errorf("expected 'end' in query param, got nil instead")
 		}
-		if _, err := strconv.ParseInt(endTS, 10, 64); err != nil {
+		if _, err := time.Parse(time.RFC3339, endTS); err != nil {
 			t.Errorf("failed to parse 'end' query param: %s", err)
 		}
 		step := r.URL.Query().Get("step")
@@ -455,8 +453,8 @@ func TestRequestParams(t *testing.T) {
 			false,
 			&VMStorage{},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("query=%s&time=%d", query, timestamp.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -464,8 +462,9 @@ func TestRequestParams(t *testing.T) {
 			true,
 			&VMStorage{},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("end=%d&query=%s&start=%d", timestamp.Unix(), query, timestamp.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				ts := timestamp.Format(time.RFC3339)
+				exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -495,8 +494,8 @@ func TestRequestParams(t *testing.T) {
 				lookBack: time.Minute,
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("query=%s&time=%d", query, timestamp.Add(-time.Minute).Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				exp := url.Values{"query": {query}, "time": {timestamp.Add(-time.Minute).Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -508,8 +507,8 @@ func TestRequestParams(t *testing.T) {
 			func(t *testing.T, r *http.Request) {
 				evalInterval := 15 * time.Second
 				tt := timestamp.Truncate(evalInterval)
-				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, evalInterval, tt.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {tt.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -523,8 +522,8 @@ func TestRequestParams(t *testing.T) {
 				evalInterval := 15 * time.Second
 				tt := timestamp.Add(-time.Minute)
 				tt = tt.Truncate(evalInterval)
-				exp := fmt.Sprintf("query=%s&step=%v&time=%d", query, evalInterval, tt.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {tt.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -534,8 +533,12 @@ func TestRequestParams(t *testing.T) {
 				queryStep: time.Minute,
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(time.Minute.Seconds()), timestamp.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				exp := url.Values{
+					"query": {query},
+					"step":  {fmt.Sprintf("%ds", int(time.Minute.Seconds()))},
+					"time":  {timestamp.Format(time.RFC3339)},
+				}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -547,8 +550,8 @@ func TestRequestParams(t *testing.T) {
 			func(t *testing.T, r *http.Request) {
 				evalInterval := 3 * time.Hour
 				tt := timestamp.Truncate(evalInterval)
-				exp := fmt.Sprintf("query=%s&step=%ds&time=%d", query, int(evalInterval.Seconds()), tt.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {tt.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -558,8 +561,8 @@ func TestRequestParams(t *testing.T) {
 				extraParams: url.Values{"round_digits": {"10"}},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("query=%s&round_digits=10&time=%d", query, timestamp.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -572,9 +575,14 @@ func TestRequestParams(t *testing.T) {
 				},
 			},
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("end=%d&max_lookback=1h&nocache=1&query=%s&start=%d",
-					timestamp.Unix(), query, timestamp.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				exp := url.Values{
+					"query":        {query},
+					"end":          {timestamp.Format(time.RFC3339)},
+					"start":        {timestamp.Format(time.RFC3339)},
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -584,8 +592,8 @@ func TestRequestParams(t *testing.T) {
 				QueryParams: url.Values{"round_digits": {"2"}},
 			}),
 			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("query=%s&round_digits=2&time=%d", query, timestamp.Unix())
-				checkEqualString(t, exp, r.URL.RawQuery)
+				exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
 		{
@@ -603,6 +611,20 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
+		{
+			"graphite extra params allows to override from",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+				extraParams: url.Values{
+					"from": {"-10m"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
 	}
 
 	for _, tc := range testCases {
@@ -627,7 +649,7 @@ func TestRequestParams(t *testing.T) {
 }
 
 func TestHeaders(t *testing.T) {
-	var testCases = []struct {
+	testCases := []struct {
 		name    string
 		vmFn    func() *VMStorage
 		checkFn func(t *testing.T, r *http.Request)
@@ -692,7 +714,8 @@ func TestHeaders(t *testing.T) {
 					authCfg: cfg,
 					extraHeaders: []keyValue{
 						{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
-					}}
+					},
+				}
 			},
 			checkFn: func(t *testing.T, r *http.Request) {
 				u, p, _ := r.BasicAuth()

app/vmalert/group.go

@@ -384,11 +384,17 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 				g.Interval = ng.Interval
 				t.Stop()
 				t = time.NewTicker(g.Interval)
+				evalTS = time.Now()
 			}
 			g.mu.Unlock()
 			logger.Infof("group %q re-started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 		case <-t.C:
 			missed := (time.Since(evalTS) / g.Interval) - 1
+			if missed < 0 {
+				// missed can become < 0 due to irregular delays during evaluation
+				// which can result in time.Since(evalTS) < g.Interval
+				missed = 0
+			}
 			if missed > 0 {
 				g.metrics.iterationMissed.Inc()
 			}

app/vmalert/group_test.go

@@ -46,18 +46,36 @@ func TestUpdateWith(t *testing.T) {
 					"summary":     "{{ $value|humanize }}",
 					"description": "{{$labels}}",
 				},
-			}},
-			[]config.Rule{{
-				Alert: "foo",
-				Expr:  "up > 10",
-				For:   promutils.NewDuration(time.Second),
-				Labels: map[string]string{
-					"baz": "bar",
+			},
+				{
+					Alert: "bar",
+					Expr:  "up > 0",
+					For:   promutils.NewDuration(time.Second),
+					Labels: map[string]string{
+						"bar": "baz",
+					},
+				}},
+			[]config.Rule{
+				{
+					Alert: "foo",
+					Expr:  "up > 10",
+					For:   promutils.NewDuration(time.Second),
+					Labels: map[string]string{
+						"baz": "bar",
+					},
+					Annotations: map[string]string{
+						"summary": "none",
+					},
 				},
-				Annotations: map[string]string{
-					"summary": "none",
-				},
-			}},
+				{
+					Alert:         "bar",
+					Expr:          "up > 0",
+					For:           promutils.NewDuration(2 * time.Second),
+					KeepFiringFor: promutils.NewDuration(time.Minute),
+					Labels: map[string]string{
+						"bar": "baz",
+					},
+				}},
 		},
 		{
 			"update recording rule",

app/vmalert/helpers_test.go

@@ -272,6 +272,9 @@ func compareAlertingRules(t *testing.T, a, b *AlertingRule) error {
 	if a.For != b.For {
 		return fmt.Errorf("expected to have for %q; got %q", a.For, b.For)
 	}
+	if a.KeepFiringFor != b.KeepFiringFor {
+		return fmt.Errorf("expected to have KeepFiringFor %q; got %q", a.KeepFiringFor, b.KeepFiringFor)
+	}
 	if !reflect.DeepEqual(a.Annotations, b.Annotations) {
 		return fmt.Errorf("expected to have annotations %#v; got %#v", a.Annotations, b.Annotations)
 	}

app/vmalert/main.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -54,9 +55,6 @@ absolute path to all .tpl files in root.
  -rule.templates="dir/**/*.tpl". Includes all the .tpl files in "dir" subfolders recursively.
 `)
 
-	rulesCheckInterval = flag.Duration("rule.configCheckInterval", 0, "Interval for checking for changes in '-rule' files. "+
-		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes. DEPRECATED - see '-configCheckInterval' instead")
-
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
 		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
@@ -310,10 +308,6 @@ See the docs at https://docs.victoriametrics.com/vmalert.html .
 func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sighupCh <-chan os.Signal) {
 	var configCheckCh <-chan time.Time
 	checkInterval := *configCheckInterval
-	if checkInterval == 0 && *rulesCheckInterval > 0 {
-		logger.Warnf("flag `rule.configCheckInterval` is deprecated - use `configCheckInterval` instead")
-		checkInterval = *rulesCheckInterval
-	}
 	if checkInterval > 0 {
 		ticker := time.NewTicker(checkInterval)
 		configCheckCh = ticker.C
@@ -325,9 +319,9 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 		validateTplFn = notifier.ValidateTemplates
 	}
 
-	// init reload metrics with positive values to improve alerting conditions
-	configSuccess.Set(1)
-	configTimestamp.Set(fasttime.UnixTimestamp())
+	// init metrics for config state with positive values to improve alerting conditions
+	setConfigSuccessAt(fasttime.UnixTimestamp())
+
 	parseFn := config.Parse
 	for {
 		select {
@@ -347,43 +341,40 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			parseFn = config.ParseSilent
 		}
 		if err := notifier.Reload(); err != nil {
-			configReloadErrors.Inc()
-			configSuccess.Set(0)
+			setConfigError(err)
 			logger.Errorf("failed to reload notifier config: %s", err)
 			continue
 		}
 		err := templates.Load(*ruleTemplatesPath, false)
 		if err != nil {
-			configReloadErrors.Inc()
-			configSuccess.Set(0)
+			setConfigError(err)
 			logger.Errorf("failed to load new templates: %s", err)
 			continue
 		}
 		newGroupsCfg, err := parseFn(*rulePath, validateTplFn, *validateExpressions)
 		if err != nil {
-			configReloadErrors.Inc()
-			configSuccess.Set(0)
+			setConfigError(err)
 			logger.Errorf("cannot parse configuration file: %s", err)
 			continue
 		}
 		if configsEqual(newGroupsCfg, groupsCfg) {
 			templates.Reload()
-			// set success to 1 since previous reload
-			// could have been unsuccessful
+			// set success to 1 since previous reload could have been unsuccessful
+			// do not update configTimestamp as config version remains old.
 			configSuccess.Set(1)
-			// config didn't change - skip it
+			// reset the last config error since the config change was rolled back
+			setLastConfigErr(nil)
+			// config didn't change - skip iteration
 			continue
 		}
 		if err := m.update(ctx, newGroupsCfg, false); err != nil {
-			configReloadErrors.Inc()
-			configSuccess.Set(0)
+			setConfigError(err)
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
 		templates.Reload()
 		groupsCfg = newGroupsCfg
-		configSuccess.Set(1)
-		configTimestamp.Set(fasttime.UnixTimestamp())
+		setConfigSuccessAt(fasttime.UnixTimestamp())
 		logger.Infof("Rules reloaded successfully from %q", *rulePath)
 	}
 }
@@ -399,3 +390,37 @@ func configsEqual(a, b []config.Group) bool {
 	}
 	return true
 }
+
+// setConfigSuccessAt updates config related metrics as successful.
+func setConfigSuccessAt(at uint64) {
+	configSuccess.Set(1)
+	configTimestamp.Set(at)
+	// reset the lastConfigErr
+	setLastConfigErr(nil)
+}
+
+// setConfigError updates config related metrics according to the error.
+func setConfigError(err error) {
+	configReloadErrors.Inc()
+	configSuccess.Set(0)
+	setLastConfigErr(err)
+}
+
+var (
+	lastConfigErrMu sync.RWMutex
+	// lastConfigErr represent the error message from the last config reload.
+	// The message is used in web UI as notification
+	lastConfigErr error
+)
+
+func setLastConfigErr(err error) {
+	lastConfigErrMu.Lock()
+	lastConfigErr = err
+	lastConfigErrMu.Unlock()
+}
+
+func getLastConfigError() error {
+	lastConfigErrMu.RLock()
+	defer lastConfigErrMu.RUnlock()
+	return lastConfigErr
+}

app/vmalert/main_test.go

@@ -87,12 +87,13 @@ groups:
 	)
 
 	f, err := os.CreateTemp("", "")
+	defer os.Remove(f.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
 	writeToFile(t, f.Name(), rules1)
 
-	*rulesCheckInterval = 200 * time.Millisecond
+	*configCheckInterval = 200 * time.Millisecond
 	*rulePath = []string{f.Name()}
 	ctx, cancel := context.WithCancel(context.Background())
 
@@ -117,14 +118,37 @@ groups:
 		return len(m.groups)
 	}
 
-	time.Sleep(*rulesCheckInterval * 2)
+	checkCfg := func(err error) {
+		cErr := getLastConfigError()
+		cfgSuc := configSuccess.Get()
+		if err != nil {
+			if cErr == nil {
+				t.Fatalf("expected to have config error %s; got nil instead", cErr)
+			}
+			if cfgSuc != 0 {
+				t.Fatalf("expected to have metric configSuccess to be set to 0; got %d instead", cfgSuc)
+			}
+			return
+		}
+
+		if cErr != nil {
+			t.Fatalf("unexpected config error: %s", cErr)
+		}
+		if cfgSuc != 1 {
+			t.Fatalf("expected to have metric configSuccess to be set to 1; got %d instead", cfgSuc)
+		}
+	}
+
+	time.Sleep(*configCheckInterval * 2)
+	checkCfg(nil)
 	groupsLen := lenLocked(m)
 	if groupsLen != 1 {
 		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
 	}
 
 	writeToFile(t, f.Name(), rules2)
-	time.Sleep(*rulesCheckInterval * 2)
+	time.Sleep(*configCheckInterval * 2)
+	checkCfg(nil)
 	groupsLen = lenLocked(m)
 	if groupsLen != 2 {
 		fmt.Println(m.groups)
@@ -133,7 +157,8 @@ groups:
 
 	writeToFile(t, f.Name(), rules1)
 	procutil.SelfSIGHUP()
-	time.Sleep(*rulesCheckInterval / 2)
+	time.Sleep(*configCheckInterval / 2)
+	checkCfg(nil)
 	groupsLen = lenLocked(m)
 	if groupsLen != 1 {
 		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)
@@ -141,7 +166,8 @@ groups:
 
 	writeToFile(t, f.Name(), `corrupted`)
 	procutil.SelfSIGHUP()
-	time.Sleep(*rulesCheckInterval / 2)
+	time.Sleep(*configCheckInterval / 2)
+	checkCfg(fmt.Errorf("config error"))
 	groupsLen = lenLocked(m)
 	if groupsLen != 1 { // should remain unchanged
 		t.Fatalf("expected to have exactly 1 group loaded; got %d", groupsLen)

app/vmalert/manager.go

@@ -120,7 +120,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
 	}
 	if arPresent && m.notifiers == nil {
-		return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` aren't set")
+		return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` nor `-notifier.blackhole` aren't set")
 	}
 
 	type updateItem struct {

app/vmalert/manager_test.go

@@ -340,3 +340,21 @@ func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressio
 	}
 	return cfg
 }
+
+func TestUrlValuesToStrings(t *testing.T) {
+	mapQueryParams := map[string][]string{
+		"param1": {"param1"},
+		"param2": {"anotherparam"},
+	}
+	expectedRes := []string{"param1=param1", "param2=anotherparam"}
+	res := urlValuesToStrings(mapQueryParams)
+
+	if len(res) != len(expectedRes) {
+		t.Errorf("Expected length %d, but got %d", len(expectedRes), len(res))
+	}
+	for ind, val := range expectedRes {
+		if val != res[ind] {
+			t.Errorf("Expected %v; but got %v", val, res[ind])
+		}
+	}
+}

app/vmalert/notifier/alert.go

@@ -39,6 +39,8 @@ type Alert struct {
 	ResolvedAt time.Time
 	// LastSent defines the moment when Alert was sent last time
 	LastSent time.Time
+	// KeepFiringSince defines the moment when StateFiring was kept because of `keep_firing_for` instead of real alert
+	KeepFiringSince time.Time
 	// Value stores the value returned from evaluating expression from Expr field
 	Value float64
 	// ID is the unique identifier for the Alert

app/vmalert/notifier/config_watcher_test.go

@@ -14,6 +14,7 @@ import (
 
 func TestConfigWatcherReload(t *testing.T) {
 	f, err := os.CreateTemp("", "")
+	defer os.Remove(f.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -36,6 +37,7 @@ static_configs:
 	}
 
 	f2, err := os.CreateTemp("", "")
+	defer os.Remove(f2.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -63,6 +65,7 @@ func TestConfigWatcherStart(t *testing.T) {
 	defer consulSDServer.Close()
 
 	consulSDFile, err := os.CreateTemp("", "")
+	defer os.Remove(consulSDFile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -109,6 +112,7 @@ func TestConfigWatcherReloadConcurrent(t *testing.T) {
 	defer consulSDServer2.Close()
 
 	consulSDFile, err := os.CreateTemp("", "")
+	defer os.Remove(consulSDFile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -125,6 +129,7 @@ consul_sd_configs:
 `, consulSDServer1.URL, consulSDServer2.URL))
 
 	staticAndConsulSDFile, err := os.CreateTemp("", "")
+	defer os.Remove(staticAndConsulSDFile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}

app/vmalert/notifier/init.go

@@ -19,6 +19,9 @@ var (
 
 	addrs = flagutil.NewArrayString("notifier.url", "Prometheus Alertmanager URL, e.g. http://127.0.0.1:9093. "+
 		"List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.")
+	blackHole = flag.Bool("notifier.blackhole", false, "Whether to blackhole alerting notifications. "+
+		"Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). "+
+		"`-notifier.url`, `-notifier.config` and `-notifier.blackhole` are mutually exclusive.")
 
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
@@ -90,6 +93,17 @@ func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (fu
 
 	templates.UpdateWithFuncs(templates.FuncsWithExternalURL(eu))
 
+	if *blackHole {
+		if len(*addrs) > 0 || *configPath != "" {
+			return nil, fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
+		}
+
+		staticNotifiersFn = func() []Notifier {
+			return []Notifier{newBlackHoleNotifier()}
+		}
+		return staticNotifiersFn, nil
+	}
+
 	if *configPath == "" && len(*addrs) == 0 {
 		return nil, nil
 	}

app/vmalert/notifier/init_test.go

@@ -1,8 +1,9 @@
 package notifier
 
 import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 func TestInit(t *testing.T) {
@@ -35,3 +36,58 @@ func TestInit(t *testing.T) {
 		t.Fatalf("expected to get \"127.0.0.2/api/v2/alerts\"; got %q instead", nf2.Addr())
 	}
 }
+
+func TestInitNegative(t *testing.T) {
+	oldConfigPath := *configPath
+	oldAddrs := *addrs
+	oldBlackHole := *blackHole
+
+	defer func() {
+		*configPath = oldConfigPath
+		*addrs = oldAddrs
+		*blackHole = oldBlackHole
+	}()
+
+	f := func(path, addr string, bh bool) {
+		*configPath = path
+		*addrs = flagutil.ArrayString{addr}
+		*blackHole = bh
+		if _, err := Init(nil, nil, ""); err == nil {
+			t.Fatalf("expected to get error; got nil instead")
+		}
+	}
+
+	// *configPath, *addrs and *blackhole are mutually exclusive
+	f("/dummy/path", "127.0.0.1", false)
+	f("/dummy/path", "", true)
+	f("", "127.0.0.1", true)
+}
+
+func TestBlackHole(t *testing.T) {
+	oldBlackHole := *blackHole
+	defer func() { *blackHole = oldBlackHole }()
+
+	*blackHole = true
+
+	fn, err := Init(nil, nil, "")
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+
+	nfs := fn()
+	if len(nfs) != 1 {
+		t.Fatalf("expected to get 1 notifier; got %d", len(nfs))
+	}
+
+	targets := GetTargets()
+	if targets == nil || targets[TargetStatic] == nil {
+		t.Fatalf("expected to get static targets in response")
+	}
+	if len(targets[TargetStatic]) != 1 {
+		t.Fatalf("expected to get 1 static targets in response; but got %d", len(targets[TargetStatic]))
+	}
+	nf1 := targets[TargetStatic][0]
+	if nf1.Addr() != "blackhole" {
+		t.Fatalf("expected to get \"blackhole\"; got %q instead", nf1.Addr())
+	}
+}

app/vmalert/notifier/notifier_blackhole.go

@@ -0,0 +1,36 @@
+package notifier
+
+import "context"
+
+// blackHoleNotifier is a Notifier stub, used when no notifications need
+// to be sent.
+type blackHoleNotifier struct {
+	addr    string
+	metrics *metrics
+}
+
+// Send will send no notifications, but increase the metric.
+func (bh *blackHoleNotifier) Send(_ context.Context, alerts []Alert, _ map[string]string) error { //nolint:revive
+	bh.metrics.alertsSent.Add(len(alerts))
+	return nil
+}
+
+// Addr of black hole notifier
+func (bh blackHoleNotifier) Addr() string {
+	return bh.addr
+}
+
+// Close unregister the metrics
+func (bh *blackHoleNotifier) Close() {
+	bh.metrics.alertsSent.Unregister()
+	bh.metrics.alertsSendErrors.Unregister()
+}
+
+// newBlackHoleNotifier creates a new blackHoleNotifier
+func newBlackHoleNotifier() *blackHoleNotifier {
+	address := "blackhole"
+	return &blackHoleNotifier{
+		addr:    address,
+		metrics: newMetrics(address),
+	}
+}

app/vmalert/notifier/notifier_blackhole_test.go

@@ -0,0 +1,50 @@
+package notifier
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	metricset "github.com/VictoriaMetrics/metrics"
+)
+
+func TestBlackHoleNotifier_Send(t *testing.T) {
+	bh := newBlackHoleNotifier()
+	if err := bh.Send(context.Background(), []Alert{{
+		GroupID:     0,
+		Name:        "alert0",
+		Start:       time.Now().UTC(),
+		End:         time.Now().UTC(),
+		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
+	}}, nil); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+
+	alertCount := bh.metrics.alertsSent.Get()
+	if alertCount != 1 {
+		t.Errorf("expect value 1; instead got %d", alertCount)
+	}
+}
+
+func TestBlackHoleNotifier_Close(t *testing.T) {
+	bh := newBlackHoleNotifier()
+	if err := bh.Send(context.Background(), []Alert{{
+		GroupID:     0,
+		Name:        "alert0",
+		Start:       time.Now().UTC(),
+		End:         time.Now().UTC(),
+		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
+	}}, nil); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+
+	bh.Close()
+
+	defaultMetrics := metricset.GetDefaultSet()
+	alertMetricName := "vmalert_alerts_sent_total{addr=\"blackhole\"}"
+	for _, name := range defaultMetrics.ListMetricNames() {
+		if name == alertMetricName {
+			t.Errorf("Metric name should have unregistered.But still present")
+		}
+	}
+}

app/vmalert/remotewrite/remotewrite.go

@@ -23,6 +23,8 @@ import (
 var (
 	disablePathAppend = flag.Bool("remoteWrite.disablePathAppend", false, "Whether to disable automatic appending of '/api/v1/write' path to the configured -remoteWrite.url.")
 	sendTimeout       = flag.Duration("remoteWrite.sendTimeout", 30*time.Second, "Timeout for sending data to the configured -remoteWrite.url.")
+	retryMinInterval  = flag.Duration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxInterval")
+	retryMaxTime      = flag.Duration("remoteWrite.retryMaxTime", time.Second*30, "The max time spent on retry attempts for the failed remote-write request. Change this value if it is expected for remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
 )
 
 // Client is an asynchronous HTTP client for writing
@@ -147,6 +149,7 @@ func (c *Client) run(ctx context.Context) {
 			wr.Timeseries = append(wr.Timeseries, ts)
 		}
 		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
+		logger.Infof("shutting down remote write client and flushing remained %d series", len(wr.Timeseries))
 		c.flush(lastCtx, wr)
 		cancel()
 	}
@@ -180,9 +183,14 @@ func (c *Client) run(ctx context.Context) {
 var (
 	sentRows            = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
 	sentBytes           = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
+	sendDuration        = metrics.NewFloatCounter(`vmalert_remotewrite_send_duration_seconds_total`)
 	droppedRows         = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
 	droppedBytes        = metrics.NewCounter(`vmalert_remotewrite_dropped_bytes_total`)
 	bufferFlushDuration = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
+
+	_ = metrics.NewGauge(`vmalert_remotewrite_concurrency`, func() float64 {
+		return float64(*concurrency)
+	})
 )
 
 // flush is a blocking function that marshals WriteRequest and sends
@@ -203,12 +211,14 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 
 	b := snappy.Encode(nil, data)
 
-	const (
-		retryCount   = 5
-		retryBackoff = time.Second
-	)
-
-	for attempts := 0; attempts < retryCount; attempts++ {
+	retryInterval, maxRetryInterval := *retryMinInterval, *retryMaxTime
+	if retryInterval > maxRetryInterval {
+		retryInterval = maxRetryInterval
+	}
+	timeStart := time.Now()
+	defer sendDuration.Add(time.Since(timeStart).Seconds())
+L:
+	for attempts := 0; ; attempts++ {
 		err := c.send(ctx, b)
 		if err == nil {
 			sentRows.Add(len(wr.Timeseries))
@@ -216,10 +226,10 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 			return
 		}
 
-		_, isRetriable := err.(*retriableError)
-		logger.Warnf("attempt %d to send request failed: %s (retriable: %v)", attempts+1, err, isRetriable)
+		_, isNotRetriable := err.(*nonRetriableError)
+		logger.Warnf("attempt %d to send request failed: %s (retriable: %v)", attempts+1, err, !isNotRetriable)
 
-		if !isRetriable {
+		if isNotRetriable {
 			// exit fast if error isn't retriable
 			break
 		}
@@ -227,12 +237,24 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 		// check if request has been cancelled before backoff
 		select {
 		case <-ctx.Done():
-			break
+			logger.Errorf("interrupting retry attempt %d: context cancelled", attempts+1)
+			break L
 		default:
 		}
 
-		// sleeping to avoid remote db hammering
-		time.Sleep(retryBackoff)
+		timeLeftForRetries := maxRetryInterval - time.Since(timeStart)
+		if timeLeftForRetries < 0 {
+			// the max retry time has passed, so we give up
+			break
+		}
+
+		if retryInterval > timeLeftForRetries {
+			retryInterval = timeLeftForRetries
+		}
+		// sleeping to prevent remote db hammering
+		time.Sleep(retryInterval)
+		retryInterval *= 2
+
 	}
 
 	droppedRows.Add(len(wr.Timeseries))
@@ -276,22 +298,23 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	case 2:
 		// respond with a HTTP 2xx status code when the write is successful.
 		return nil
-	case 5:
-		// respond with HTTP status code 5xx when the write fails and SHOULD be retried.
-		return &retriableError{fmt.Errorf("unexpected response code %d for %s. Response body %q",
-			resp.StatusCode, req.URL.Redacted(), body)}
+	case 4:
+		if resp.StatusCode != http.StatusTooManyRequests {
+			// MUST NOT retry write requests on HTTP 4xx responses other than 429
+			return &nonRetriableError{fmt.Errorf("unexpected response code %d for %s. Response body %q",
+				resp.StatusCode, req.URL.Redacted(), body)}
+		}
+		fallthrough
 	default:
-		// respond with HTTP status code 4xx when the request is invalid, will never be able to succeed
-		// and should not be retried.
 		return fmt.Errorf("unexpected response code %d for %s. Response body %q",
 			resp.StatusCode, req.URL.Redacted(), body)
 	}
 }
 
-type retriableError struct {
+type nonRetriableError struct {
 	err error
 }
 
-func (e *retriableError) Error() string {
+func (e *nonRetriableError) Error() string {
 	return e.err.Error()
 }

app/vmalert/remotewrite/remotewrite_test.go

@@ -7,6 +7,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/http/httptest"
+	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -18,15 +19,30 @@ import (
 )
 
 func TestClient_Push(t *testing.T) {
+	oldMinInterval := *retryMinInterval
+	*retryMinInterval = time.Millisecond * 10
+	defer func() {
+		*retryMinInterval = oldMinInterval
+	}()
+
 	testSrv := newRWServer()
-	cfg := Config{
+	client, err := NewClient(context.Background(), Config{
 		Addr:         testSrv.URL,
 		MaxBatchSize: 100,
-	}
-	client, err := NewClient(context.Background(), cfg)
+	})
 	if err != nil {
 		t.Fatalf("failed to create client: %s", err)
 	}
+
+	faultySrv := newFaultyRWServer()
+	faultyClient, err := NewClient(context.Background(), Config{
+		Addr:         faultySrv.URL,
+		MaxBatchSize: 50,
+	})
+	if err != nil {
+		t.Fatalf("failed to create faulty client: %s", err)
+	}
+
 	r := rand.New(rand.NewSource(1))
 	const rowsN = 1e4
 	var sent int
@@ -38,9 +54,16 @@ func TestClient_Push(t *testing.T) {
 			}},
 		}
 		err := client.Push(s)
+		if err != nil {
+			t.Fatalf("unexpected err: %s", err)
+		}
 		if err == nil {
 			sent++
 		}
+		err = faultyClient.Push(s)
+		if err != nil {
+			t.Fatalf("unexpected err: %s", err)
+		}
 	}
 	if sent == 0 {
 		t.Fatalf("0 series sent")
@@ -48,10 +71,17 @@ func TestClient_Push(t *testing.T) {
 	if err := client.Close(); err != nil {
 		t.Fatalf("failed to close client: %s", err)
 	}
+	if err := faultyClient.Close(); err != nil {
+		t.Fatalf("failed to close faulty client: %s", err)
+	}
 	got := testSrv.accepted()
 	if got != sent {
 		t.Fatalf("expected to have %d series; got %d", sent, got)
 	}
+	got = faultySrv.accepted()
+	if got != sent {
+		t.Fatalf("expected to have %d series for faulty client; got %d", sent, got)
+	}
 }
 
 func newRWServer() *rwServer {
@@ -117,3 +147,42 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 	atomic.AddUint64(&rw.acceptedRows, uint64(len(wr.Timeseries)))
 	w.WriteHeader(http.StatusNoContent)
 }
+
+// faultyRWServer sometimes respond with 5XX status code
+// or just closes the connection. Is used for testing retries.
+type faultyRWServer struct {
+	*rwServer
+
+	reqsMu sync.Mutex
+	reqs   int
+}
+
+func newFaultyRWServer() *faultyRWServer {
+	rw := &faultyRWServer{
+		rwServer: &rwServer{},
+	}
+	rw.Server = httptest.NewServer(http.HandlerFunc(rw.handler))
+	return rw
+}
+
+func (frw *faultyRWServer) handler(w http.ResponseWriter, r *http.Request) {
+	frw.reqsMu.Lock()
+	reqs := frw.reqs
+	frw.reqs++
+	if frw.reqs > 5 {
+		frw.reqs = 0
+	}
+	frw.reqsMu.Unlock()
+
+	switch reqs {
+	case 0, 1, 2, 3:
+		frw.rwServer.handler(w, r)
+	case 4:
+		hj, _ := w.(http.Hijacker)
+		conn, _, _ := hj.Hijack()
+		conn.Close()
+	case 5:
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("server overloaded"))
+	}
+}

app/vmalert/rule.go

@@ -31,7 +31,7 @@ type Rule interface {
 	Close()
 }
 
-var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels")
+var errDuplicate = errors.New("result contains metrics with the same labelset after applying rule labels. See https://docs.victoriametrics.com/vmalert.html#series-with-the-same-labelset for details")
 
 type ruleState struct {
 	sync.RWMutex

app/vmalert/static/js/custom.js

@@ -7,9 +7,11 @@ function collapseAll() {
 }
 
 function toggleByID(id) {
-    let el = $("#" + id);
-    if (el.length > 0) {
-        el.click();
+    if (id) {
+        let el = $("#" + id);
+        if (el.length > 0) {
+            el.click();
+        }
     }
 }
 
@@ -36,4 +38,4 @@ $(document).ready(function () {
 
 $(document).ready(function () {
     $('[data-bs-toggle="tooltip"]').tooltip();
-});
+});

app/vmalert/tpl/header.qtpl

@@ -6,8 +6,8 @@
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 ) %}
 
-{% func Header(r *http.Request, navItems []NavItem, title string) %}
-    {%code prefix := utils.Prefix(r.URL.Path) %}
+{% func Header(r *http.Request, navItems []NavItem, title string, userErr error) %}
+{%code prefix := utils.Prefix(r.URL.Path) %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -70,8 +70,9 @@
     </style>
 </head>
 <body>
-    {%= printNavItems(r, title, navItems) %}
+    {%= printNavItems(r, title, navItems, userErr) %}
     <main class="px-2">
+    {%= errorBody(userErr) %}
 {% endfunc %}
 
 
@@ -82,7 +83,7 @@ type NavItem struct {
 }
 %}
 
-{% func printNavItems(r *http.Request, current string, items []NavItem) %}
+{% func printNavItems(r *http.Request, current string, items []NavItem, userErr error) %}
 {%code
     prefix := utils.Prefix(r.URL.Path)
 %}
@@ -103,5 +104,30 @@ type NavItem struct {
             {% endfor %}
         </ul>
   </div>
+  {%= errorIcon(userErr) %}
 </nav>
 {% endfunc %}
+
+{% func errorIcon(err error) %}
+{% if err != nil %}
+<div class="d-flex" data-bs-toggle="tooltip" data-bs-placement="left" title="Configuration file failed to reload! Click to see more details.">
+  <a type="button" data-bs-toggle="collapse" href="#reload-groups-error">
+      <span class="text-danger">
+          <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-exclamation-triangle-fill" viewBox="0 0 16 16">
+              <path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/>
+          </svg>
+      </span>
+  </a>
+</div>
+{% endif %}
+{% endfunc %}
+
+{% func errorBody(err error) %}
+{% if err != nil %}
+<div class="collapse mt-2 mb-2" id="reload-groups-error">
+  <div class="card card-body">
+    {%s err.Error() %}
+  </div>
+</div>
+{% endif %}
+{% endfunc %}

app/vmalert/tpl/header.qtpl.go

@@ -27,10 +27,10 @@ var (
 )
 
 //line app/vmalert/tpl/header.qtpl:9
-func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem, title string) {
+func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem, title string, userErr error) {
 //line app/vmalert/tpl/header.qtpl:9
 	qw422016.N().S(`
-    `)
+`)
 //line app/vmalert/tpl/header.qtpl:10
 	prefix := utils.Prefix(r.URL.Path)
 
@@ -114,135 +114,251 @@ func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem
 <body>
     `)
 //line app/vmalert/tpl/header.qtpl:73
-	streamprintNavItems(qw422016, r, title, navItems)
+	streamprintNavItems(qw422016, r, title, navItems, userErr)
 //line app/vmalert/tpl/header.qtpl:73
 	qw422016.N().S(`
     <main class="px-2">
+    `)
+//line app/vmalert/tpl/header.qtpl:75
+	streamerrorBody(qw422016, userErr)
+//line app/vmalert/tpl/header.qtpl:75
+	qw422016.N().S(`
 `)
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:76
 }
 
-//line app/vmalert/tpl/header.qtpl:75
-func WriteHeader(qq422016 qtio422016.Writer, r *http.Request, navItems []NavItem, title string) {
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:76
+func WriteHeader(qq422016 qtio422016.Writer, r *http.Request, navItems []NavItem, title string, userErr error) {
+//line app/vmalert/tpl/header.qtpl:76
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:75
-	StreamHeader(qw422016, r, navItems, title)
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:76
+	StreamHeader(qw422016, r, navItems, title, userErr)
+//line app/vmalert/tpl/header.qtpl:76
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:76
 }
 
-//line app/vmalert/tpl/header.qtpl:75
-func Header(r *http.Request, navItems []NavItem, title string) string {
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:76
+func Header(r *http.Request, navItems []NavItem, title string, userErr error) string {
+//line app/vmalert/tpl/header.qtpl:76
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:75
-	WriteHeader(qb422016, r, navItems, title)
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:76
+	WriteHeader(qb422016, r, navItems, title, userErr)
+//line app/vmalert/tpl/header.qtpl:76
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:76
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:76
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:76
 }
 
-//line app/vmalert/tpl/header.qtpl:79
+//line app/vmalert/tpl/header.qtpl:80
 type NavItem struct {
 	Name string
 	Url  string
 }
 
-//line app/vmalert/tpl/header.qtpl:85
-func streamprintNavItems(qw422016 *qt422016.Writer, r *http.Request, current string, items []NavItem) {
-//line app/vmalert/tpl/header.qtpl:85
+//line app/vmalert/tpl/header.qtpl:86
+func streamprintNavItems(qw422016 *qt422016.Writer, r *http.Request, current string, items []NavItem, userErr error) {
+//line app/vmalert/tpl/header.qtpl:86
 	qw422016.N().S(`
 `)
-//line app/vmalert/tpl/header.qtpl:87
+//line app/vmalert/tpl/header.qtpl:88
 	prefix := utils.Prefix(r.URL.Path)
 
-//line app/vmalert/tpl/header.qtpl:88
+//line app/vmalert/tpl/header.qtpl:89
 	qw422016.N().S(`
 <nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
   <div class="container-fluid">
     <div class="collapse navbar-collapse" id="navbarCollapse">
         <ul class="navbar-nav me-auto mb-2 mb-md-0">
             `)
-//line app/vmalert/tpl/header.qtpl:93
+//line app/vmalert/tpl/header.qtpl:94
 	for _, item := range items {
-//line app/vmalert/tpl/header.qtpl:93
+//line app/vmalert/tpl/header.qtpl:94
 		qw422016.N().S(`
                 <li class="nav-item">
                     `)
-//line app/vmalert/tpl/header.qtpl:96
+//line app/vmalert/tpl/header.qtpl:97
 		u, _ := url.Parse(item.Url)
 
-//line app/vmalert/tpl/header.qtpl:97
+//line app/vmalert/tpl/header.qtpl:98
 		qw422016.N().S(`
                     <a class="nav-link`)
-//line app/vmalert/tpl/header.qtpl:98
+//line app/vmalert/tpl/header.qtpl:99
 		if current == item.Name {
-//line app/vmalert/tpl/header.qtpl:98
+//line app/vmalert/tpl/header.qtpl:99
 			qw422016.N().S(` active`)
-//line app/vmalert/tpl/header.qtpl:98
+//line app/vmalert/tpl/header.qtpl:99
 		}
-//line app/vmalert/tpl/header.qtpl:98
+//line app/vmalert/tpl/header.qtpl:99
 		qw422016.N().S(`"
                        href="`)
-//line app/vmalert/tpl/header.qtpl:99
+//line app/vmalert/tpl/header.qtpl:100
 		if u.IsAbs() {
-//line app/vmalert/tpl/header.qtpl:99
+//line app/vmalert/tpl/header.qtpl:100
 			qw422016.E().S(item.Url)
-//line app/vmalert/tpl/header.qtpl:99
+//line app/vmalert/tpl/header.qtpl:100
 		} else {
-//line app/vmalert/tpl/header.qtpl:99
+//line app/vmalert/tpl/header.qtpl:100
 			qw422016.E().S(path.Join(prefix, item.Url))
-//line app/vmalert/tpl/header.qtpl:99
+//line app/vmalert/tpl/header.qtpl:100
 		}
-//line app/vmalert/tpl/header.qtpl:99
+//line app/vmalert/tpl/header.qtpl:100
 		qw422016.N().S(`">
                         `)
-//line app/vmalert/tpl/header.qtpl:100
+//line app/vmalert/tpl/header.qtpl:101
 		qw422016.E().S(item.Name)
-//line app/vmalert/tpl/header.qtpl:100
+//line app/vmalert/tpl/header.qtpl:101
 		qw422016.N().S(`
                     </a>
                 </li>
             `)
-//line app/vmalert/tpl/header.qtpl:103
+//line app/vmalert/tpl/header.qtpl:104
 	}
-//line app/vmalert/tpl/header.qtpl:103
+//line app/vmalert/tpl/header.qtpl:104
 	qw422016.N().S(`
         </ul>
   </div>
+  `)
+//line app/vmalert/tpl/header.qtpl:107
+	streamerrorIcon(qw422016, userErr)
+//line app/vmalert/tpl/header.qtpl:107
+	qw422016.N().S(`
 </nav>
 `)
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:109
 }
 
-//line app/vmalert/tpl/header.qtpl:107
-func writeprintNavItems(qq422016 qtio422016.Writer, r *http.Request, current string, items []NavItem) {
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:109
+func writeprintNavItems(qq422016 qtio422016.Writer, r *http.Request, current string, items []NavItem, userErr error) {
+//line app/vmalert/tpl/header.qtpl:109
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:107
-	streamprintNavItems(qw422016, r, current, items)
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:109
+	streamprintNavItems(qw422016, r, current, items, userErr)
+//line app/vmalert/tpl/header.qtpl:109
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:109
 }
 
-//line app/vmalert/tpl/header.qtpl:107
-func printNavItems(r *http.Request, current string, items []NavItem) string {
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:109
+func printNavItems(r *http.Request, current string, items []NavItem, userErr error) string {
+//line app/vmalert/tpl/header.qtpl:109
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:107
-	writeprintNavItems(qb422016, r, current, items)
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:109
+	writeprintNavItems(qb422016, r, current, items, userErr)
+//line app/vmalert/tpl/header.qtpl:109
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:109
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:109
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:109
+}
+
+//line app/vmalert/tpl/header.qtpl:111
+func streamerrorIcon(qw422016 *qt422016.Writer, err error) {
+//line app/vmalert/tpl/header.qtpl:111
+	qw422016.N().S(`
+`)
+//line app/vmalert/tpl/header.qtpl:112
+	if err != nil {
+//line app/vmalert/tpl/header.qtpl:112
+		qw422016.N().S(`
+<div class="d-flex" data-bs-toggle="tooltip" data-bs-placement="left" title="Configuration file failed to reload! Click to see more details.">
+  <a type="button" data-bs-toggle="collapse" href="#reload-groups-error">
+      <span class="text-danger">
+          <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-exclamation-triangle-fill" viewBox="0 0 16 16">
+              <path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/>
+          </svg>
+      </span>
+  </a>
+</div>
+`)
+//line app/vmalert/tpl/header.qtpl:122
+	}
+//line app/vmalert/tpl/header.qtpl:122
+	qw422016.N().S(`
+`)
+//line app/vmalert/tpl/header.qtpl:123
+}
+
+//line app/vmalert/tpl/header.qtpl:123
+func writeerrorIcon(qq422016 qtio422016.Writer, err error) {
+//line app/vmalert/tpl/header.qtpl:123
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/header.qtpl:123
+	streamerrorIcon(qw422016, err)
+//line app/vmalert/tpl/header.qtpl:123
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/header.qtpl:123
+}
+
+//line app/vmalert/tpl/header.qtpl:123
+func errorIcon(err error) string {
+//line app/vmalert/tpl/header.qtpl:123
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/header.qtpl:123
+	writeerrorIcon(qb422016, err)
+//line app/vmalert/tpl/header.qtpl:123
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/header.qtpl:123
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/header.qtpl:123
+	return qs422016
+//line app/vmalert/tpl/header.qtpl:123
+}
+
+//line app/vmalert/tpl/header.qtpl:125
+func streamerrorBody(qw422016 *qt422016.Writer, err error) {
+//line app/vmalert/tpl/header.qtpl:125
+	qw422016.N().S(`
+`)
+//line app/vmalert/tpl/header.qtpl:126
+	if err != nil {
+//line app/vmalert/tpl/header.qtpl:126
+		qw422016.N().S(`
+<div class="collapse mt-2 mb-2" id="reload-groups-error">
+  <div class="card card-body">
+    `)
+//line app/vmalert/tpl/header.qtpl:129
+		qw422016.E().S(err.Error())
+//line app/vmalert/tpl/header.qtpl:129
+		qw422016.N().S(`
+  </div>
+</div>
+`)
+//line app/vmalert/tpl/header.qtpl:132
+	}
+//line app/vmalert/tpl/header.qtpl:132
+	qw422016.N().S(`
+`)
+//line app/vmalert/tpl/header.qtpl:133
+}
+
+//line app/vmalert/tpl/header.qtpl:133
+func writeerrorBody(qq422016 qtio422016.Writer, err error) {
+//line app/vmalert/tpl/header.qtpl:133
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmalert/tpl/header.qtpl:133
+	streamerrorBody(qw422016, err)
+//line app/vmalert/tpl/header.qtpl:133
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmalert/tpl/header.qtpl:133
+}
+
+//line app/vmalert/tpl/header.qtpl:133
+func errorBody(err error) string {
+//line app/vmalert/tpl/header.qtpl:133
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmalert/tpl/header.qtpl:133
+	writeerrorBody(qb422016, err)
+//line app/vmalert/tpl/header.qtpl:133
+	qs422016 := string(qb422016.B)
+//line app/vmalert/tpl/header.qtpl:133
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmalert/tpl/header.qtpl:133
+	return qs422016
+//line app/vmalert/tpl/header.qtpl:133
 }

app/vmalert/web.go

@@ -138,27 +138,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 
 	default:
-		// Support of deprecated links:
-		// * /api/v1/<groupID>/<alertID>/status
-		// * <groupID>/<alertID>/status
-		// TODO: to remove in next versions
-
-		if !strings.HasSuffix(r.URL.Path, "/status") {
-			httpserver.Errorf(w, r, "unsupported path requested: %q ", r.URL.Path)
-			return false
-		}
-		alert, err := rh.alertByPath(strings.TrimPrefix(r.URL.Path, "/api/v1/"))
-		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-
-		redirectURL := alert.WebLink()
-		if strings.HasPrefix(r.URL.Path, "/api/v1/") {
-			redirectURL = alert.APILink()
-		}
-		httpserver.Redirect(w, "/"+redirectURL)
-		return true
+		httpserver.Errorf(w, r, "unsupported path requested: %q ", r.URL.Path)
+		return false
 	}
 }
 
@@ -302,41 +283,6 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 	return b, nil
 }
 
-func (rh *requestHandler) alertByPath(path string) (*APIAlert, error) {
-	if strings.HasPrefix(path, "/vmalert") {
-		path = strings.TrimLeft(path, "/vmalert")
-	}
-	parts := strings.SplitN(strings.TrimLeft(path, "/"), "/", -1)
-	if len(parts) != 3 {
-		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/groupID/alertID/status"`, path),
-			StatusCode: http.StatusBadRequest,
-		}
-	}
-	groupID, err := uint64FromPath(parts[0])
-	if err != nil {
-		return nil, badRequest(fmt.Errorf(`cannot parse groupID: %w`, err))
-	}
-	alertID, err := uint64FromPath(parts[1])
-	if err != nil {
-		return nil, badRequest(fmt.Errorf(`cannot parse alertID: %w`, err))
-	}
-	resp, err := rh.m.AlertAPI(groupID, alertID)
-	if err != nil {
-		return nil, errResponse(err, http.StatusNotFound)
-	}
-	return resp, nil
-}
-
-func uint64FromPath(path string) (uint64, error) {
-	s := strings.TrimRight(path, "/")
-	return strconv.ParseUint(s, 10, 0)
-}
-
-func badRequest(err error) *httpserver.ErrorWithStatusCode {
-	return errResponse(err, http.StatusBadRequest)
-}
-
 func errResponse(err error, sc int) *httpserver.ErrorWithStatusCode {
 	return &httpserver.ErrorWithStatusCode{
 		Err:        err,

app/vmalert/web.qtpl

@@ -12,7 +12,7 @@
 
 
 {% func Welcome(r *http.Request) %}
-    {%= tpl.Header(r, navItems, "vmalert") %}
+    {%= tpl.Header(r, navItems, "vmalert", getLastConfigError()) %}
     <p>
         API:<br>
         {% for _, p := range apiLinks  %}
@@ -40,7 +40,7 @@ btn-primary
 
 {% func ListGroups(r *http.Request, originGroups []APIGroup) %}
     {%code prefix := utils.Prefix(r.URL.Path) %}
-    {%= tpl.Header(r, navItems, "Groups") %}
+    {%= tpl.Header(r, navItems, "Groups", getLastConfigError()) %}
         {%code
             filter := r.URL.Query().Get("filter")
             rOk := make(map[string]int)
@@ -116,7 +116,11 @@ btn-primary
                                     <div class="row">
                                         <div class="col-12 mb-2">
                                             {% if r.Type == "alerting" %}
+                                            {% if r.KeepFiringFor > 0 %}
+                                            <b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds, keep_firing_for: {%v r.KeepFiringFor %} seconds)
+                                            {% else %}
                                             <b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds)
+                                            {% endif %}
                                             {% else %}
                                             <b>record:</b> {%s r.Name %}
                                             {% endif %}
@@ -164,7 +168,7 @@ btn-primary
 
 {% func ListAlerts(r *http.Request, groupAlerts []GroupAlerts) %}
     {%code prefix := utils.Prefix(r.URL.Path) %}
-    {%= tpl.Header(r, navItems, "Alerts") %}
+    {%= tpl.Header(r, navItems, "Alerts", getLastConfigError()) %}
     {% if len(groupAlerts) > 0 %}
          <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
          <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
@@ -225,6 +229,7 @@ btn-primary
                                 <td>
                                     {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
                                     {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
+                                    {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
                                 </td>
                                 <td>{%s ar.Value %}</td>
                                 <td>
@@ -250,7 +255,7 @@ btn-primary
 {% endfunc %}
 
 {% func ListTargets(r *http.Request, targets map[notifier.TargetType][]notifier.Target) %}
-    {%= tpl.Header(r, navItems, "Notifiers") %}
+    {%= tpl.Header(r, navItems, "Notifiers", getLastConfigError()) %}
     {% if len(targets) > 0 %}
          <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
          <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
@@ -307,7 +312,7 @@ btn-primary
 
 {% func Alert(r *http.Request, alert *APIAlert) %}
     {%code prefix := utils.Prefix(r.URL.Path) %}
-    {%= tpl.Header(r, navItems, "") %}
+    {%= tpl.Header(r, navItems, "", getLastConfigError()) %}
     {%code
         var labelKeys []string
         for k := range alert.Labels {
@@ -394,7 +399,7 @@ btn-primary
 
 {% func RuleDetails(r *http.Request, rule APIRule) %}
     {%code prefix := utils.Prefix(r.URL.Path) %}
-    {%= tpl.Header(r, navItems, "") %}
+    {%= tpl.Header(r, navItems, "", getLastConfigError()) %}
     {%code
         var labelKeys []string
         for k := range rule.Labels {
@@ -442,6 +447,18 @@ btn-primary
         </div>
       </div>
     </div>
+    {% if rule.KeepFiringFor > 0 %}
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Keep firing for
+        </div>
+        <div class="col">
+         {%v rule.KeepFiringFor %} seconds
+        </div>
+      </div>
+    </div>
+    {% endif %}
     {% endif %}
     <div class="container border-bottom p-2">
       <div class="row">
@@ -561,6 +578,10 @@ btn-primary
 <span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
 {% endfunc %}
 
+{% func badgeStabilizing() %}
+<span class="badge bg-warning text-dark" title="This firing state is kept because of `keep_firing_for`">stabilizing</span>
+{% endfunc %}
+
 {% func seriesFetchedWarn(r APIRule) %}
 {% if isNoMatch(r) %}
 <svg xmlns="http://www.w3.org/2000/svg"
@@ -578,4 +599,4 @@ btn-primary
   func isNoMatch (r APIRule) bool {
     return r.LastSamples == 0 && r.LastSeriesFetched != nil && *r.LastSeriesFetched == 0
   }
-%}
+%}

app/vmalert/web_test.go

@@ -145,23 +145,6 @@ func TestHandler(t *testing.T) {
 			t.Errorf("expected 1 group got %d", length)
 		}
 	})
-
-	// check deprecated links support
-	// TODO: remove as soon as deprecated links removed
-	t.Run("/api/v1/0/0/status", func(t *testing.T) {
-		alert := &APIAlert{}
-		getResp(ts.URL+"/api/v1/0/0/status", alert, 200)
-		expAlert := ar.newAlertAPI(*ar.alerts[0])
-		if !reflect.DeepEqual(alert, expAlert) {
-			t.Errorf("expected %v is equal to %v", alert, expAlert)
-		}
-	})
-	t.Run("/api/v1/0/1/status", func(t *testing.T) {
-		getResp(ts.URL+"/api/v1/0/1/status", nil, 404)
-	})
-	t.Run("/api/v1/1/0/status", func(t *testing.T) {
-		getResp(ts.URL+"/api/v1/1/0/status", nil, 404)
-	})
 }
 
 func TestEmptyResponse(t *testing.T) {

app/vmalert/web_types.go

@@ -32,6 +32,9 @@ type APIAlert struct {
 	SourceLink string `json:"source"`
 	// Restored shows whether Alert's state was restored on restart
 	Restored bool `json:"restored"`
+	// Stabilizing shows when firing state is kept because of
+	// `keep_firing_for` instead of real alert
+	Stabilizing bool `json:"stabilizing"`
 }
 
 // WebLink returns a link to the alert which can be used in UI.
@@ -96,9 +99,11 @@ type APIRule struct {
 	// Query represents Rule's `expression` field
 	Query string `json:"query"`
 	// Duration represents Rule's `for` field
-	Duration    float64           `json:"duration"`
-	Labels      map[string]string `json:"labels,omitempty"`
-	Annotations map[string]string `json:"annotations,omitempty"`
+	Duration float64 `json:"duration"`
+	// Alert will continue firing for this long even when the alerting expression no longer has results.
+	KeepFiringFor float64           `json:"keep_firing_for"`
+	Labels        map[string]string `json:"labels,omitempty"`
+	Annotations   map[string]string `json:"annotations,omitempty"`
 	// LastError contains the error faced while executing the rule.
 	LastError string `json:"lastError"`
 	// EvaluationTime is the time taken to completely evaluate the rule in float seconds.

app/vmauth/README.md

@@ -85,7 +85,7 @@ ip_filters:
   deny_list: [10.0.0.42]
 ```
 
-The following config allows requests for the user 'foobar' only from the ip `127.0.0.1`:
+The following config allows requests for the user 'foobar' only from the IP `127.0.0.1`:
 
 ```yml
 users:
@@ -96,6 +96,8 @@ users:
     allow_list: [127.0.0.1]
 ```
 
+See config example of using IP filters [here](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmauth/example_config_ent.yml).
+
 ## Auth config
 
 `-auth.config` is represented in the following simple `yml` format:
@@ -249,7 +251,12 @@ It is recommended protecting the following endpoints with authKeys:
 `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
 either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
 
-`vmauth` exports `vmauth_user_requests_total` metric with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file. It is possible to override or hide the value in the label by specifying `name` field. For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
+`vmauth` exports `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric 
+and `vmauth_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) metric 
+with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file.
+It is possible to override or hide the value in the label by specifying `name` field. 
+For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` 
+instead of `vmauth_user_requests_total{username="secret_user"}`:
 
 ```yml
 users:
@@ -258,7 +265,10 @@ users:
   # other config options here
 ```
 
-For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` metric without label (if `unauthorized_user` section of config is used).
+For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` 
+[counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric and 
+`vmauth_unauthorized_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary)
+metric without label (if `unauthorized_user` section of config is used).
 
 ## How to build from sources
 
@@ -267,13 +277,13 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 ### Development build
 
 1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
-2. Run `make vmauth` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make vmauth` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmauth` binary and puts it into the `bin` folder.
 
 ### Production build
 
 1. [Install docker](https://docs.docker.com/install/).
-2. Run `make vmauth-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+1. Run `make vmauth-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmauth-prod` binary and puts it into the `bin` folder.
 
 ### Building docker images
@@ -332,15 +342,17 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -auth.config string
      Path to auth config. It can point either to local file or to http url. See https://docs.victoriametrics.com/vmauth.html for details on the format of this auth config
   -configCheckInterval duration
-     Interval for config file re-read. Zero value disables config re-reading. By default, refreshing is disabled, send SIGHUP for config refresh.
+     interval for config file re-read. Zero value disables config re-reading. By default, refreshing is disabled, send SIGHUP for config refresh.
   -enableTCP6
-     Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP is used
+     Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable
-     Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
+     Whether to enable reading flags from environment variables in addition to the command line. Command line flag values have priority over values from environment vars. Flags are read only from the command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
      By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -failTimeout duration
+     Sets a delay period for load balancing to skip a malfunctioning backend. (defaults 3s)
   -flagsAuthKey string
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -fs.disableMmap
@@ -358,15 +370,19 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -http.shutdownDelay duration
      Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers
   -httpAuth.password string
-     Password for HTTP Basic Auth. The authentication is disabled if -httpAuth.username is empty
+     Password for HTTP server's Basic Auth. The authentication is disabled if -httpAuth.username is empty
   -httpAuth.username string
-     Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
+     Username for HTTP server's Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
      TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8427")
   -httpListenAddr.useProxyProtocol
-     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
+     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing
+  -internStringCacheExpireDuration duration
+     The expiry duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
+  -internStringDisableCache
+     Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 500)
+     The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
   -logInvalidAuthTokens
      Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
@@ -392,10 +408,10 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -maxIdleConnsPerBackend int
      The maximum number of idle connections vmauth can open per each backend host. See also -maxConcurrentRequests (default 100)
   -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
+     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
   -memory.allowedPercent float
-     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
+     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -pprofAuthKey string
@@ -406,7 +422,7 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -pushmetrics.interval duration
      Interval for pushing metrics to -pushmetrics.url (default 10s)
   -pushmetrics.url array
-     Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default metrics exposed at /metrics page aren't pushed to any remote storage
+     Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage
      Supports an array of values separated by comma or specified via multiple flags.
   -reloadAuthKey string
      Auth key for /-/reload http endpoint. It must be passed as authKey=...

app/vmauth/auth_config.go

@@ -50,7 +50,8 @@ type UserInfo struct {
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
-	requests *metrics.Counter
+	requests         *metrics.Counter
+	requestsDuration *metrics.Summary
 }
 
 func (ui *UserInfo) beginConcurrencyLimit() error {
@@ -133,7 +134,7 @@ func (bu *backendURL) isBroken() bool {
 }
 
 func (bu *backendURL) setBroken() {
-	deadline := fasttime.UnixTimestamp() + 3
+	deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds())
 	atomic.StoreUint64(&bu.brokenDeadline, deadline)
 }
 
@@ -378,6 +379,7 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 	ui := ac.UnauthorizedUser
 	if ui != nil {
 		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
+		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
 		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
 		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
@@ -441,9 +443,11 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
 			}
 			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
+			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
 		}
 		if ui.Username != "" {
 			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
+			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
 		}
 		mcr := ui.getMaxConcurrentRequests()
 		ui.concurrencyLimitCh = make(chan struct{}, mcr)

app/vmauth/example_config.yml

@@ -89,8 +89,6 @@ users:
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
     headers:
     - "X-Scope-OrgID: abc"
-    ip_filters:
-      deny_list: [127.0.0.1]
   default_url:
   - "http://default1:8888/unsupported_url_handler"
   - "http://default2:8888/unsupported_url_handler"
@@ -104,10 +102,3 @@ unauthorized_user:
     url_prefix:
     - http://vmselect1:8481/select/0/prometheus
     - http://vmselect2:8481/select/0/prometheus
-    ip_filters:
-      allow_list: [8.8.8.8]
-
-ip_filters:
-  allow_list: ["1.2.3.0/24", "127.0.0.1"]
-  deny_list:
-  - 10.1.0.1

app/vmauth/example_config_ent.yml

@@ -0,0 +1,57 @@
+# Arbitrary number of usernames may be put here.
+# It is possible to set multiple identical usernames with different passwords.
+# Such usernames can be differentiated by `name` option.
+
+users:
+  # A single user for querying and inserting data:
+  #
+  # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
+  #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
+  #     - http://vmselect1:8481/select/42/prometheus
+  #     - http://vmselect2:8481/select/42/prometheus
+  #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
+  #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
+  #
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  #   The "X-Scope-OrgID: abc" http header is added to these requests.
+  #
+  # Request which do not match `src_paths` from the `url_map` are proxied to the urls from `default_url`
+  # in a round-robin manner. The original request path is passed in `request_path` query arg.
+  # For example, request to http://vmauth:8427/non/existing/path are proxied:
+  #  - to http://default1:8888/unsupported_url_handler?request_path=/non/existing/path
+  #  - or http://default2:8888/unsupported_url_handler?request_path=/non/existing/path
+- username: "foobar"
+  url_map:
+  - src_paths:
+    - "/api/v1/query"
+    - "/api/v1/query_range"
+    - "/api/v1/label/[^/]+/values"
+    url_prefix:
+    - "http://vmselect1:8481/select/42/prometheus"
+    - "http://vmselect2:8481/select/42/prometheus"
+  - src_paths: ["/api/v1/write"]
+    url_prefix: "http://vminsert:8480/insert/42/prometheus"
+    headers:
+    - "X-Scope-OrgID: abc"
+    ip_filters:
+      deny_list: [127.0.0.1]
+  default_url:
+  - "http://default1:8888/unsupported_url_handler"
+  - "http://default2:8888/unsupported_url_handler"
+
+# Requests without Authorization header are routed according to `unauthorized_user` section.
+unauthorized_user:
+  url_map:
+  - src_paths:
+    - /api/v1/query
+    - /api/v1/query_range
+    url_prefix:
+    - http://vmselect1:8481/select/0/prometheus
+    - http://vmselect2:8481/select/0/prometheus
+    ip_filters:
+      allow_list: [8.8.8.8]
+
+ip_filters:
+  allow_list: ["1.2.3.0/24", "127.0.0.1"]
+  deny_list:
+  - 10.1.0.1

app/vmauth/main.go

@@ -41,6 +41,7 @@ var (
 	reloadAuthKey        = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
+	failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend.")
 )
 
 func main() {
@@ -123,6 +124,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
+	startTime := time.Now()
+	defer ui.requestsDuration.UpdateDuration(startTime)
+
 	ui.requests.Inc()
 
 	// Limit the concurrency of requests to backends