test build

### Describe Your Changes

Set missing `Authorization` header when querying a storage node and
Basic Auth is enabled.

See: #9080 

### Checklist

The following checks are **mandatory**:

- [X] My change adheres to [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -5,10 +5,10 @@ body:
   - type: markdown
     attributes:
       value: |
-        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
+        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade) 
         to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
         and verify whether the bug is reproducible there.
-        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/troubleshooting/) first.
+        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) first.
   - type: textarea
     id: describe-the-bug
     attributes:
@@ -64,8 +64,8 @@ body:
           * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
         
         See how to setup monitoring here:
-          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
-          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/#monitoring)
+          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
+          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
     validations:
       required: false
   - type: textarea

.github/ISSUE_TEMPLATE/question.yml

@@ -24,9 +24,9 @@ body:
       label: Troubleshooting docs
       description: I am familiar with the following troubleshooting docs
       options:
-        - label: General - https://docs.victoriametrics.com/troubleshooting/
+        - label: General - https://docs.victoriametrics.com/victoriametrics/troubleshooting/
           required: false
-        - label: vmagent - https://docs.victoriametrics.com/vmagent/#troubleshooting
+        - label: vmagent - https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
           required: false
-        - label: vmalert - https://docs.victoriametrics.com/vmalert/#troubleshooting
+        - label: vmalert - https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting
           required: false

.github/pull_request_template.md

@@ -6,4 +6,4 @@ Please provide a brief description of the changes you made. Be as specific as po
 
 The following checks are **mandatory**:
 
-- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).
+- [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).

.gitignore

@@ -27,3 +27,4 @@ _site
 coverage.txt
 cspell.json
 *~
+deployment/docker/provisioning/plugins/

CONTRIBUTING.md

@@ -1 +1 @@
-The document has been moved [here](https://docs.victoriametrics.com/contributing/).
+The document has been moved [here](https://docs.victoriametrics.com/victoriametrics/contributing/).

Makefile

@@ -5,7 +5,6 @@ MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
 	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
-LATEST_TAG ?= cluster-latest
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
@@ -108,12 +107,31 @@ package: \
 	package-vmselect \
 	package-vmstorage
 
+publish-latest:
+	PKG_TAG=$(TAG) APP_NAME=victoria-metrics $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG) APP_NAME=vmagent $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG) APP_NAME=vmalert $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG) APP_NAME=vmalert-tool $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG) APP_NAME=vmauth $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG) APP_NAME=vmbackup $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG) APP_NAME=vmrestore $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG) APP_NAME=vmctl $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG)-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG)-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG)-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-latest && \
+	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-latest
+	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-latest
+
+publish-victoria-logs-latest:
+	PKG_TAG=$(TAG) APP_NAME=victoria-logs $(MAKE) publish-via-docker-latest
+	PKG_TAG=$(TAG) APP_NAME=vlogscli $(MAKE) publish-via-docker-latest
+
 publish-release:
 	rm -rf bin/*
-	git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
+	git checkout $(TAG) && $(MAKE) release && $(MAKE) publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release && $(MAKE) publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release && $(MAKE) publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && $(MAKE) publish
 
 release:
 	$(MAKE_PARALLEL) release-vmcluster
@@ -195,7 +213,7 @@ fmt:
 	gofmt -l -w -s ./apptest
 
 vet:
-	go vet ./lib/...
+	GOEXPERIMENT=synctest go vet ./lib/...
 	go vet ./app/...
 	go vet ./apptest/...
 
@@ -204,35 +222,35 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	go test ./lib/... ./app/...
+	GOEXPERIMENT=synctest go test ./lib/... ./app/...
 
 test-race:
-	go test -race ./lib/... ./app/...
+	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
 
 test-pure:
-	CGO_ENABLED=0 go test ./lib/... ./app/...
+	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 integration-test: all
 	go test ./apptest/... -skip="^TestSingle.*"
 
 benchmark:
-	go test -bench=. ./lib/...
+	GOEXPERIMENT=synctest go test -bench=. ./lib/...
 	go test -bench=. ./app/...
 
 benchmark-pure:
-	CGO_ENABLED=0 go test -bench=. ./lib/...
+	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
 	go get -u ./lib/...
 	go get -u ./app/...
-	go mod tidy -compat=1.23
+	go mod tidy -compat=1.24
 	go mod vendor
 
 app-local:
@@ -255,7 +273,7 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	golangci-lint run
+	GOEXPERIMENT=synctest golangci-lint run
 
 install-golangci-lint:
 	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7

README.md

@@ -11,9 +11,9 @@
 ![Reddit](https://img.shields.io/reddit/subreddit-subscribers/VictoriaMetrics?style=flat&label=Join&labelColor=red&logoColor=white&logo=reddit&link=https%3A%2F%2Fwww.reddit.com%2Fr%2FVictoriaMetrics)
 
 <picture>
-  <source srcset="docs/logo_white.webp" media="(prefers-color-scheme: dark)">
-  <source srcset="docs/logo.webp" media="(prefers-color-scheme: light)">
-  <img src="docs/logo.webp" width="300" alt="VictoriaMetrics logo">
+  <source srcset="docs/victoriametrics/logo_white.webp" media="(prefers-color-scheme: dark)">
+  <source srcset="docs/victoriametrics/logo.webp" media="(prefers-color-scheme: light)">
+  <img src="docs/victoriametrics/logo.webp" width="300" alt="VictoriaMetrics logo">
 </picture>
 
 VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
@@ -21,10 +21,10 @@ VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and
 Here are some resources and information about VictoriaMetrics:
 
 - Documentation: [docs.victoriametrics.com](https://docs.victoriametrics.com)
-- Case studies: [Grammarly, Roblox, Wix,...](https://docs.victoriametrics.com/casestudies/).
+- Case studies: [Grammarly, Roblox, Wix,...](https://docs.victoriametrics.com/victoriametrics/casestudies/).
 - Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), docker images [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
-- Deployment types: [Single-node version](https://docs.victoriametrics.com/), [Cluster version](https://docs.victoriametrics.com/cluster-victoriametrics/), and [Enterprise version](https://docs.victoriametrics.com/enterprise/)
-- Changelog: [CHANGELOG](https://docs.victoriametrics.com/changelog/), and [How to upgrade](https://docs.victoriametrics.com/#how-to-upgrade-victoriametrics)
+- Deployment types: [Single-node version](https://docs.victoriametrics.com/), [Cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/)
+- Changelog: [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/), and [How to upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics)
 - Community: [Slack](https://slack.victoriametrics.com/), [X (Twitter)](https://x.com/VictoriaMetrics), [LinkedIn](https://www.linkedin.com/company/victoriametrics/), [YouTube](https://www.youtube.com/@VictoriaMetrics)
 
 Yes, we open-source both the single-node VictoriaMetrics and the cluster version.
@@ -35,22 +35,22 @@ VictoriaMetrics is optimized for timeseries data, even when old time series are
 
 * **Long-term storage for Prometheus** or as a drop-in replacement for Prometheus and Graphite in Grafana.
 * **Powerful stream aggregation**: Can be used as a StatsD alternative.
-* **Ideal for big data**: Works well with large amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://docs.victoriametrics.com/enterprise/).
+* **Ideal for big data**: Works well with large amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://docs.victoriametrics.com/victoriametrics/enterprise/).
 * **Query language**: Supports both PromQL and the more performant MetricsQL.
 * **Easy to setup**: No dependencies, single [small binary](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d), configuration through command-line flags, but the default is also fine-tuned; backup and restore with [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
 * **Global query view**: Multiple Prometheus instances or any other data sources may ingest data into VictoriaMetrics and queried via a single query.
 * **Various Protocols**: Support metric scraping, ingestion and backfilling in various protocol.
-    * [Prometheus exporters](https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter), [Prometheus remote write API](https://docs.victoriametrics.com/#prometheus-setup), [Prometheus exposition format](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format).
-    * [InfluxDB line protocol](https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) over HTTP, TCP and UDP.
-    * [Graphite plaintext protocol](https://docs.victoriametrics.com/#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon).
-    * [OpenTSDB put message](https://docs.victoriametrics.com/#sending-data-via-telnet-put-protocol).
-    * [HTTP OpenTSDB /api/put requests](https://docs.victoriametrics.com/#sending-opentsdb-data-via-http-apiput-requests).
-    * [JSON line format](https://docs.victoriametrics.com/#how-to-import-data-in-json-line-format).
-    * [Arbitrary CSV data](https://docs.victoriametrics.com/#how-to-import-csv-data).
-    * [Native binary format](https://docs.victoriametrics.com/#how-to-import-data-in-native-format).
-    * [DataDog agent or DogStatsD](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent).
-    * [NewRelic infrastructure agent](https://docs.victoriametrics.com/#how-to-send-data-from-newrelic-agent).
-    * [OpenTelemetry metrics format](https://docs.victoriametrics.com/#sending-data-via-opentelemetry).
+    * [Prometheus exporters](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter), [Prometheus remote write API](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/), [Prometheus exposition format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format).
+    * [InfluxDB line protocol](https://docs.victoriametrics.com/victoriametrics/integrations/influxdb/) over HTTP, TCP and UDP.
+    * [Graphite plaintext protocol](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon).
+    * [OpenTSDB put message](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-telnet).
+    * [HTTP OpenTSDB /api/put requests](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http).
+    * [JSON line format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-json-line-format).
+    * [Arbitrary CSV data](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data).
+    * [Native binary format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-native-format).
+    * [DataDog agent or DogStatsD](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/).
+    * [NewRelic infrastructure agent](https://docs.victoriametrics.com/victoriametrics/integrations/newrelic/#sending-data-from-agent).
+    * [OpenTelemetry metrics format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#sending-data-via-opentelemetry).
 * **NFS-based storages**: Supports storing data on NFS-based storages such as Amazon EFS, Google Filestore.
 * And many other features such as metrics relabeling, cardinality limiter, etc.
 
@@ -62,9 +62,9 @@ In addition, the Enterprise version includes extra features:
 - **Backup automation**: Automates regular backup procedures.
 - **Multiple retentions**: Reducing storage costs by specifying different retentions for different datasets.
 - **Downsampling**: Reducing storage costs and increasing performance for queries over historical data.
-- **Stable releases** with long-term support lines ([LTS](https://docs.victoriametrics.com/lts-releases/)).
+- **Stable releases** with long-term support lines ([LTS](https://docs.victoriametrics.com/victoriametrics/lts-releases/)).
 - **Comprehensive support**: First-class consulting, feature requests and technical support provided by the core VictoriaMetrics dev team.
-- Many other features, which you can read about on [the Enterprise page](https://docs.victoriametrics.com/enterprise/).
+- Many other features, which you can read about on [the Enterprise page](https://docs.victoriametrics.com/victoriametrics/enterprise/).
 
 [Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics. Or you can request a free trial license [here](https://victoriametrics.com/products/enterprise/trial/), downloaded Enterprise binaries are available at [Github Releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
 
@@ -77,7 +77,7 @@ Some good benchmarks VictoriaMetrics achieved:
 * **Minimal memory footprint**: handling millions of unique timeseries with [10x less RAM](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) than InfluxDB, up to [7x less RAM](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f) than Prometheus, Thanos or Cortex.
 * **Highly scalable and performance** for [data ingestion](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b) and [querying](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4), [20x outperforms](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) InfluxDB and TimescaleDB.
 * **High data compression**: [70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) may be stored into limited storage than TimescaleDB, [7x less storage](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f) space is required than Prometheus, Thanos or Cortex.
-* **Reducing storage costs**: [10x more effective](https://docs.victoriametrics.com/casestudies/#grammarly) than Graphite according to the Grammarly case study.
+* **Reducing storage costs**: [10x more effective](https://docs.victoriametrics.com/victoriametrics/casestudies/#grammarly) than Graphite according to the Grammarly case study.
 * **A single-node VictoriaMetrics** can replace medium-sized clusters built with competing solutions such as Thanos, M3DB, Cortex, InfluxDB or TimescaleDB. See [VictoriaMetrics vs Thanos](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683), [Measuring vertical scalability](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae), [Remote write storage wars - PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
 * **Optimized for storage**: [Works well with high-latency IO](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b) and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc.).
 
@@ -93,7 +93,7 @@ Feel free asking any questions regarding VictoriaMetrics:
 * [Telegram-ru](https://t.me/VictoriaMetrics_ru1)
 * [Mastodon](https://mastodon.social/@victoriametrics/)
 
-If you like VictoriaMetrics and want to contribute, then please [read these docs](https://docs.victoriametrics.com/contributing/).
+If you like VictoriaMetrics and want to contribute, then please [read these docs](https://docs.victoriametrics.com/victoriametrics/contributing/).
 
 ## VictoriaMetrics Logo
 

SECURITY.md

@@ -6,9 +6,9 @@ The following versions of VictoriaMetrics receive regular security fixes:
 
 | Version | Supported          |
 |---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.110.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| [latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
+| v1.102.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
+| v1.110.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.

app/victoria-logs/main.go

@@ -46,7 +46,9 @@ func main() {
 	vlselect.Init()
 	vlinsert.Init()
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
+		UseProxyProtocol: useProxyProtocol,
+	})
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/victorialogs/", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()

app/vlinsert/datadog/datadog.go

@@ -95,6 +95,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 	// There is no need in updating v2LogsRequestDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
 	v2LogsRequestDuration.UpdateDuration(startTime)
+	w.WriteHeader(http.StatusAccepted)
 	fmt.Fprintf(w, `{}`)
 	return true
 }

app/vlinsert/elasticsearch/elasticsearch.go

@@ -102,7 +102,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		lmp := cp.NewLogMessageProcessor("elasticsearch_bulk", true)
 		encoding := r.Header.Get("Content-Encoding")
 		streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-		n, err := readBulkRequest(streamName, r.Body, encoding, cp.TimeField, cp.MsgFields, lmp)
+		n, err := readBulkRequest(streamName, r.Body, encoding, cp.TimeFields, cp.MsgFields, lmp)
 		lmp.MustClose()
 		if err != nil {
 			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
@@ -131,7 +131,7 @@ var (
 	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
-func readBulkRequest(streamName string, r io.Reader, encoding string, timeField string, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {
+func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
 
 	reader, err := protoparserutil.GetUncompressedReader(r, encoding)
@@ -147,7 +147,7 @@ func readBulkRequest(streamName string, r io.Reader, encoding string, timeField
 
 	n := 0
 	for {
-		ok, err := readBulkLine(lr, timeField, msgFields, lmp)
+		ok, err := readBulkLine(lr, timeFields, msgFields, lmp)
 		wcr.DecConcurrency()
 		if err != nil || !ok {
 			return n, err
@@ -156,7 +156,7 @@ func readBulkRequest(streamName string, r io.Reader, encoding string, timeField
 	}
 }
 
-func readBulkLine(lr *insertutil.LineReader, timeField string, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
+func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
 	var line []byte
 
 	// Read the command, must be "create" or "index"
@@ -190,7 +190,7 @@ func readBulkLine(lr *insertutil.LineReader, timeField string, msgFields []strin
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
 
-	ts, err := extractTimestampFromFields(timeField, p.Fields)
+	ts, err := extractTimestampFromFields(timeFields, p.Fields)
 	if err != nil {
 		return false, fmt.Errorf("cannot parse timestamp: %w", err)
 	}
@@ -204,18 +204,20 @@ func readBulkLine(lr *insertutil.LineReader, timeField string, msgFields []strin
 	return true, nil
 }
 
-func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
+func extractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
+	for _, timeField := range timeFields {
+		for i := range fields {
+			f := &fields[i]
+			if f.Name != timeField {
+				continue
+			}
+			timestamp, err := parseElasticsearchTimestamp(f.Value)
+			if err != nil {
+				return 0, err
+			}
+			f.Value = ""
+			return timestamp, nil
 		}
-		timestamp, err := parseElasticsearchTimestamp(f.Value)
-		if err != nil {
-			return 0, err
-		}
-		f.Value = ""
-		return timestamp, nil
 	}
 	return 0, nil
 }

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -19,7 +19,7 @@ func TestReadBulkRequest_Failure(t *testing.T) {
 
 		tlp := &insertutil.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", "_time", []string{"_msg"}, tlp)
+		rows, err := readBulkRequest("test", r, "", []string{"_time"}, []string{"_msg"}, tlp)
 		if err == nil {
 			t.Fatalf("expecting non-empty error")
 		}
@@ -40,12 +40,13 @@ func TestReadBulkRequest_Success(t *testing.T) {
 	f := func(data, encoding, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
+		timeFields := []string{"non_existing_foo", timeField, "non_existing_bar"}
 		msgFields := []string{"non_existing_foo", msgField, "non_exiting_bar"}
 		tlp := &insertutil.TestLogMessageProcessor{}
 
 		// Read the request without compression
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", timeField, msgFields, tlp)
+		rows, err := readBulkRequest("test", r, "", timeFields, msgFields, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -62,7 +63,7 @@ func TestReadBulkRequest_Success(t *testing.T) {
 			data = compressData(data, encoding)
 		}
 		r = bytes.NewBufferString(data)
-		rows, err = readBulkRequest("test", r, encoding, timeField, msgFields, tlp)
+		rows, err = readBulkRequest("test", r, encoding, timeFields, msgFields, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -111,7 +112,7 @@ func compressData(s string, encoding string) string {
 	case "zstd":
 		zw, _ = zstd.NewWriter(&bb)
 	case "snappy":
-		zw = snappy.NewBufferedWriter(&bb)
+		return string(snappy.Encode(nil, []byte(s)))
 	case "deflate":
 		zw = zlib.NewWriter(&bb)
 	default:

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -40,7 +40,7 @@ func benchmarkReadBulkRequest(b *testing.B, encoding string) {
 	}
 	dataBytes := bytesutil.ToUnsafeBytes(data)
 
-	timeField := "@timestamp"
+	timeFields := []string{"@timestamp"}
 	msgFields := []string{"message"}
 	blp := &insertutil.BenchmarkLogMessageProcessor{}
 
@@ -50,7 +50,7 @@ func benchmarkReadBulkRequest(b *testing.B, encoding string) {
 		r := &bytes.Reader{}
 		for pb.Next() {
 			r.Reset(dataBytes)
-			_, err := readBulkRequest("test", r, encoding, timeField, msgFields, blp)
+			_, err := readBulkRequest("test", r, encoding, timeFields, msgFields, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/insertutil/common_params.go

@@ -28,12 +28,13 @@ var (
 //
 // See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
 type CommonParams struct {
-	TenantID     logstorage.TenantID
-	TimeField    string
-	MsgFields    []string
-	StreamFields []string
-	IgnoreFields []string
-	ExtraFields  []logstorage.Field
+	TenantID         logstorage.TenantID
+	TimeFields       []string
+	MsgFields        []string
+	StreamFields     []string
+	IgnoreFields     []string
+	DecolorizeFields []string
+	ExtraFields      []logstorage.Field
 
 	Debug           bool
 	DebugRequestURI string
@@ -48,14 +49,15 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 		return nil, err
 	}
 
-	timeField := "_time"
-	if tf := httputil.GetRequestValue(r, "_time_field", "VL-Time-Field"); tf != "" {
-		timeField = tf
+	timeFields := []string{"_time"}
+	if tfs := httputil.GetArray(r, "_time_field", "VL-Time-Field"); len(tfs) > 0 {
+		timeFields = tfs
 	}
 
 	msgFields := httputil.GetArray(r, "_msg_field", "VL-Msg-Field")
 	streamFields := httputil.GetArray(r, "_stream_fields", "VL-Stream-Fields")
 	ignoreFields := httputil.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
+	decolorizeFields := httputil.GetArray(r, "decolorize_fields", "VL-Decolorize-Fields")
 
 	extraFields, err := getExtraFields(r)
 	if err != nil {
@@ -77,15 +79,16 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 	}
 
 	cp := &CommonParams{
-		TenantID:        tenantID,
-		TimeField:       timeField,
-		MsgFields:       msgFields,
-		StreamFields:    streamFields,
-		IgnoreFields:    ignoreFields,
-		ExtraFields:     extraFields,
-		Debug:           debug,
-		DebugRequestURI: debugRequestURI,
-		DebugRemoteAddr: debugRemoteAddr,
+		TenantID:         tenantID,
+		TimeFields:       timeFields,
+		MsgFields:        msgFields,
+		StreamFields:     streamFields,
+		IgnoreFields:     ignoreFields,
+		DecolorizeFields: decolorizeFields,
+		ExtraFields:      extraFields,
+		Debug:            debug,
+		DebugRequestURI:  debugRequestURI,
+		DebugRemoteAddr:  debugRemoteAddr,
 	}
 
 	return cp, nil
@@ -112,7 +115,7 @@ func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
 }
 
 // GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
-func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields []string, extraFields []logstorage.Field) *CommonParams {
+func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) *CommonParams {
 	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
 	if streamFields == nil {
 		streamFields = []string{
@@ -122,14 +125,17 @@ func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignore
 		}
 	}
 	cp := &CommonParams{
-		TenantID:  tenantID,
-		TimeField: "timestamp",
+		TenantID: tenantID,
+		TimeFields: []string{
+			"timestamp",
+		},
 		MsgFields: []string{
 			"message",
 		},
-		StreamFields: streamFields,
-		IgnoreFields: ignoreFields,
-		ExtraFields:  extraFields,
+		StreamFields:     streamFields,
+		IgnoreFields:     ignoreFields,
+		DecolorizeFields: decolorizeFields,
+		ExtraFields:      extraFields,
 	}
 
 	return cp
@@ -219,6 +225,42 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
 	}
 }
 
+// InsertRowProcessor is used by native data ingestion protocol parser.
+type InsertRowProcessor interface {
+	// AddInsertRow must add r to the underlying storage.
+	AddInsertRow(r *logstorage.InsertRow)
+}
+
+// AddInsertRow adds r to lmp.
+func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
+	lmp.rowsIngestedTotal.Inc()
+	n := logstorage.EstimatedJSONRowLen(r.Fields)
+	lmp.bytesIngestedTotal.Add(n)
+
+	if len(r.Fields) > *MaxFieldsPerLine {
+		line := logstorage.MarshalFieldsToJSON(nil, r.Fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(r.Fields), *MaxFieldsPerLine, line)
+		rowsDroppedTotalTooManyFields.Inc()
+		return
+	}
+
+	lmp.mu.Lock()
+	defer lmp.mu.Unlock()
+
+	lmp.lr.MustAddInsertRow(r)
+
+	if lmp.cp.Debug {
+		s := lmp.lr.GetRowString(0)
+		lmp.lr.ResetKeepSettings()
+		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+		rowsDroppedTotalDebug.Inc()
+		return
+	}
+	if lmp.lr.NeedFlush() {
+		lmp.flushLocked()
+	}
+}
+
 // flushLocked must be called under locked lmp.mu.
 func (lmp *logMessageProcessor) flushLocked() {
 	lmp.lastFlushTime = time.Now()
@@ -240,7 +282,7 @@ func (lmp *logMessageProcessor) MustClose() {
 //
 // MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
 func (cp *CommonParams) NewLogMessageProcessor(protocolName string, isStreamMode bool) LogMessageProcessor {
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.ExtraFields, *defaultMsgValue)
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.DecolorizeFields, cp.ExtraFields, *defaultMsgValue)
 	rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
 	bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
 	lmp := &logMessageProcessor{

app/vlinsert/insertutil/timestamp.go

@@ -10,27 +10,29 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-// ExtractTimestampFromFields extracts timestamp in nanoseconds from the field with the name timeField at fields.
+// ExtractTimestampFromFields extracts timestamp in nanoseconds from the first field the name from timeFields at fields.
 //
-// The value for the timeField is set to empty string after returning from the function,
+// The value for the corresponding timeFields is set to empty string after returning from the function,
 // so it could be ignored during data ingestion.
 //
 // The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
+func ExtractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
+	for _, timeField := range timeFields {
+		for i := range fields {
+			f := &fields[i]
+			if f.Name != timeField {
+				continue
+			}
+			nsecs, err := parseTimestamp(f.Value)
+			if err != nil {
+				return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", f.Name, err)
+			}
+			f.Value = ""
+			if nsecs == 0 {
+				nsecs = time.Now().UnixNano()
+			}
+			return nsecs, nil
 		}
-		nsecs, err := parseTimestamp(f.Value)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", timeField, err)
-		}
-		f.Value = ""
-		if nsecs == 0 {
-			nsecs = time.Now().UnixNano()
-		}
-		return nsecs, nil
 	}
 	return time.Now().UnixNano(), nil
 }

app/vlinsert/insertutil/timestamp_test.go

@@ -67,7 +67,7 @@ func TestExtractTimestampFromFields_Success(t *testing.T) {
 	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
 		t.Helper()
 
-		nsecs, err := ExtractTimestampFromFields(timeField, fields)
+		nsecs, err := ExtractTimestampFromFields([]string{timeField}, fields)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -140,7 +140,7 @@ func TestExtractTimestampFromFields_Error(t *testing.T) {
 		fields := []logstorage.Field{
 			{Name: "time", Value: s},
 		}
-		nsecs, err := ExtractTimestampFromFields("time", fields)
+		nsecs, err := ExtractTimestampFromFields([]string{"time"}, fields)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}

app/vlinsert/internalinsert/internalinsert.go

@@ -0,0 +1,96 @@
+package internalinsert
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+)
+
+var (
+	disableInsert  = flag.Bool("internalinsert.disable", false, "Whether to disable /internal/insert HTTP endpoint")
+	maxRequestSize = flagutil.NewBytes("internalinsert.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single request, which can be accepted at /internal/insert HTTP endpoint")
+)
+
+// RequestHandler processes /internal/insert requests.
+func RequestHandler(w http.ResponseWriter, r *http.Request) {
+	if *disableInsert {
+		httpserver.Errorf(w, r, "requests to /internal/insert are disabled with -internalinsert.disable command-line flag")
+		return
+	}
+
+	startTime := time.Now()
+	if r.Method != "POST" {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+	version := r.FormValue("version")
+	if version != netinsert.ProtocolVersion {
+		httpserver.Errorf(w, r, "unsupported protocol version=%q; want %q", version, netinsert.ProtocolVersion)
+		return
+	}
+
+	requestsTotal.Inc()
+
+	cp, err := insertutil.GetCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	encoding := r.Header.Get("Content-Encoding")
+	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
+		lmp := cp.NewLogMessageProcessor("internalinsert", false)
+		irp := lmp.(insertutil.InsertRowProcessor)
+		err := parseData(irp, data)
+		lmp.MustClose()
+		return err
+	})
+	if err != nil {
+		errorsTotal.Inc()
+		httpserver.Errorf(w, r, "cannot parse internal insert request: %s", err)
+		return
+	}
+
+	requestDuration.UpdateDuration(startTime)
+}
+
+func parseData(irp insertutil.InsertRowProcessor, data []byte) error {
+	r := logstorage.GetInsertRow()
+	src := data
+	i := 0
+	for len(src) > 0 {
+		tail, err := r.UnmarshalInplace(src)
+		if err != nil {
+			return fmt.Errorf("cannot parse row #%d: %s", i, err)
+		}
+		src = tail
+		i++
+
+		irp.AddInsertRow(r)
+	}
+	logstorage.PutInsertRow(r)
+
+	return nil
+}
+
+var (
+	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
+	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
+
+	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/internal/insert"}`)
+)

app/vlinsert/journald/journald.go

@@ -53,8 +53,8 @@ func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
 		}
 		cp.TenantID = tenantID
 	}
-	if cp.TimeField != "" {
-		cp.TimeField = *journaldTimeField
+	if len(cp.TimeFields) == 0 {
+		cp.TimeFields = []string{*journaldTimeField}
 	}
 	if len(cp.StreamFields) == 0 {
 		cp.StreamFields = *journaldStreamFields
@@ -207,7 +207,7 @@ func parseJournaldRequest(data []byte, lmp insertutil.LogMessageProcessor, cp *i
 		if !allowedJournaldEntryNameChars.MatchString(name) {
 			return fmt.Errorf("journald entry name should consist of `A-Z0-9_` characters and must start from non-digit symbol")
 		}
-		if name == cp.TimeField {
+		if slices.Contains(cp.TimeFields, name) {
 			n, err := strconv.ParseInt(value, 10, 64)
 			if err != nil {
 				return fmt.Errorf("failed to parse Journald timestamp, %w", err)

app/vlinsert/journald/journald_test.go

@@ -11,8 +11,8 @@ func TestPushJournaldOk(t *testing.T) {
 		t.Helper()
 		tlp := &insertutil.TestLogMessageProcessor{}
 		cp := &insertutil.CommonParams{
-			TimeField: "__REALTIME_TIMESTAMP",
-			MsgFields: []string{"MESSAGE"},
+			TimeFields: []string{"__REALTIME_TIMESTAMP"},
+			MsgFields:  []string{"MESSAGE"},
 		}
 		if err := parseJournaldRequest([]byte(src), tlp, cp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
@@ -46,8 +46,8 @@ func TestPushJournald_Failure(t *testing.T) {
 		t.Helper()
 		tlp := &insertutil.TestLogMessageProcessor{}
 		cp := &insertutil.CommonParams{
-			TimeField: "__REALTIME_TIMESTAMP",
-			MsgFields: []string{"MESSAGE"},
+			TimeFields: []string{"__REALTIME_TIMESTAMP"},
+			MsgFields:  []string{"MESSAGE"},
 		}
 		if err := parseJournaldRequest([]byte(data), tlp, cp); err == nil {
 			t.Fatalf("expected non nil error")

app/vlinsert/jsonline/jsonline.go

@@ -48,34 +48,49 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 
 	lmp := cp.NewLogMessageProcessor("jsonline", true)
 	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	processStreamInternal(streamName, reader, cp.TimeField, cp.MsgFields, lmp)
+	err = processStreamInternal(streamName, reader, cp.TimeFields, cp.MsgFields, lmp)
 	lmp.MustClose()
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot process jsonline request; error: %s", err)
+		return
+	}
 
 	requestDuration.UpdateDuration(startTime)
 }
 
-func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutil.LogMessageProcessor) {
+func processStreamInternal(streamName string, r io.Reader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) error {
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
 	lr := insertutil.NewLineReader(streamName, wcr)
 
 	n := 0
+	errors := 0
+	var lastError error
 	for {
-		ok, err := readLine(lr, timeField, msgFields, lmp)
+		ok, err := readLine(lr, timeFields, msgFields, lmp)
 		wcr.DecConcurrency()
 		if err != nil {
-			errorsTotal.Inc()
+			lastError = err
+			errors++
 			logger.Warnf("jsonline: cannot read line #%d in /jsonline request: %s", n, err)
 		}
 		if !ok {
-			return
+			break
 		}
 		n++
 	}
+	errorsTotal.Add(errors)
+
+	if errors > 0 && n == errors {
+		// Return an error if no logs were processed and there were errors
+		return lastError
+	}
+
+	return nil
 }
 
-func readLine(lr *insertutil.LineReader, timeField string, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
+func readLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
 	var line []byte
 	for len(line) == 0 {
 		if !lr.NextLine() {
@@ -89,11 +104,11 @@ func readLine(lr *insertutil.LineReader, timeField string, msgFields []string, l
 	defer logstorage.PutJSONParser(p)
 
 	if err := p.ParseLogMessage(line); err != nil {
-		return true, fmt.Errorf("cannot parse json-encoded line: %w; line contents: %q", err, line)
+		return true, fmt.Errorf("%s; line contents: %q", err, line)
 	}
-	ts, err := insertutil.ExtractTimestampFromFields(timeField, p.Fields)
+	ts, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
 	if err != nil {
-		return true, fmt.Errorf("cannot get timestamp from json-encoded line: %w; line contents: %q", err, line)
+		return true, fmt.Errorf("%s; line contents: %q", err, line)
 	}
 	logstorage.RenameField(p.Fields, msgFields, "_msg")
 	lmp.AddRow(ts, p.Fields, nil)

app/vlinsert/jsonline/jsonline_test.go

@@ -2,19 +2,23 @@ package jsonline
 
 import (
 	"bytes"
+	"strings"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
 )
 
-func TestProcessStreamInternal(t *testing.T) {
+func TestProcessStreamInternalSuccess(t *testing.T) {
 	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
+		timeFields := []string{timeField}
 		msgFields := []string{msgField}
 		tlp := &insertutil.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		processStreamInternal("test", r, timeField, msgFields, tlp)
+		if err := processStreamInternal("test", r, timeFields, msgFields, tlp); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
 
 		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
@@ -44,22 +48,6 @@ func TestProcessStreamInternal(t *testing.T) {
 {"message":"baz"}`
 	f(data, timeField, msgField, timestampsExpected, resultExpected)
 
-	// invalid json
-	data = "foobar"
-	timeField = "@timestamp"
-	msgField = "aaa"
-	timestampsExpected = nil
-	resultExpected = ``
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
-	// invalid timestamp field
-	data = `{"time":"foobar"}`
-	timeField = "time"
-	msgField = "abc"
-	timestampsExpected = nil
-	resultExpected = ``
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
 	// invalid lines among valid lines
 	data = `
 dsfodmasd
@@ -77,3 +65,33 @@ asbsdf
 {"_msg":"baz"}`
 	f(data, timeField, msgField, timestampsExpected, resultExpected)
 }
+
+func TestProcessStreamInternalFailure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		tlp := &insertutil.TestLogMessageProcessor{}
+		r := strings.NewReader(data)
+		if err := processStreamInternal("test", r, []string{"time"}, nil, tlp); err == nil {
+			t.Fatalf("expected error, got nil")
+		}
+
+		if err := tlp.Verify(nil, ""); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	}
+
+	// invalid json
+	f("foobar")
+
+	f(`foo
+bar`)
+
+	f(`
+foo
+
+`)
+
+	// invalid timestamp field
+	f(`{"time":"foobar"}`)
+}

app/vlinsert/main.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/internalinsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
@@ -28,6 +29,11 @@ func Stop() {
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 
+	if path == "/internal/insert" {
+		internalinsert.RequestHandler(w, r)
+		return true
+	}
+
 	if !strings.HasPrefix(path, "/insert/") {
 		// Skip requests, which do not start with /insert/, since these aren't our requests.
 		return false

app/vlinsert/opentelemetry/opentelemetry.go

@@ -12,7 +12,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -53,7 +52,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
 		lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf", false)
 		useDefaultStreamFields := len(cp.StreamFields) == 0
-		err := pushProtobufRequest(data, lmp, useDefaultStreamFields)
+		err := pushProtobufRequest(data, lmp, cp.MsgFields, useDefaultStreamFields)
 		lmp.MustClose()
 		return err
 	})
@@ -75,7 +74,7 @@ var (
 	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
 )
 
-func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, useDefaultStreamFields bool) error {
+func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) error {
 	var req pb.ExportLogsServiceRequest
 	if err := req.UnmarshalProtobuf(data); err != nil {
 		errorsTotal.Inc()
@@ -84,35 +83,31 @@ func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, useDef
 
 	var commonFields []logstorage.Field
 	for _, rl := range req.ResourceLogs {
-		attributes := rl.Resource.Attributes
-		commonFields = slicesutil.SetLength(commonFields, len(attributes))
-		for i, attr := range attributes {
-			commonFields[i].Name = attr.Key
-			commonFields[i].Value = attr.Value.FormatString(true)
-		}
+		commonFields = commonFields[:0]
+		commonFields = appendKeyValues(commonFields, rl.Resource.Attributes, "")
 		commonFieldsLen := len(commonFields)
 		for _, sc := range rl.ScopeLogs {
-			commonFields = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp, useDefaultStreamFields)
+			commonFields = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp, msgFields, useDefaultStreamFields)
 		}
 	}
 
 	return nil
 }
 
-func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutil.LogMessageProcessor, useDefaultStreamFields bool) []logstorage.Field {
+func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) []logstorage.Field {
 	fields := commonFields
 	for _, lr := range sc.LogRecords {
 		fields = fields[:len(commonFields)]
-		fields = append(fields, logstorage.Field{
-			Name:  "_msg",
-			Value: lr.Body.FormatString(true),
-		})
-		for _, attr := range lr.Attributes {
+		if lr.Body.KeyValueList != nil {
+			fields = appendKeyValues(fields, lr.Body.KeyValueList.Values, "")
+			logstorage.RenameField(fields[len(commonFields):], msgFields, "_msg")
+		} else {
 			fields = append(fields, logstorage.Field{
-				Name:  attr.Key,
-				Value: attr.Value.FormatString(true),
+				Name:  "_msg",
+				Value: lr.Body.FormatString(true),
 			})
 		}
+		fields = appendKeyValues(fields, lr.Attributes, "")
 		if len(lr.TraceID) > 0 {
 			fields = append(fields, logstorage.Field{
 				Name:  "trace_id",
@@ -138,3 +133,22 @@ func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field,
 	}
 	return fields
 }
+
+func appendKeyValues(fields []logstorage.Field, kvs []*pb.KeyValue, parentField string) []logstorage.Field {
+	for _, attr := range kvs {
+		fieldName := attr.Key
+		if parentField != "" {
+			fieldName = parentField + "." + fieldName
+		}
+
+		if attr.Value.KeyValueList != nil {
+			fields = appendKeyValues(fields, attr.Value.KeyValueList.Values, fieldName)
+		} else {
+			fields = append(fields, logstorage.Field{
+				Name:  fieldName,
+				Value: attr.Value.FormatString(true),
+			})
+		}
+	}
+	return fields
+}

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -16,7 +16,7 @@ func TestPushProtoOk(t *testing.T) {
 
 		pData := lr.MarshalProtobuf(nil)
 		tlp := &insertutil.TestLogMessageProcessor{}
-		if err := pushProtobufRequest(pData, tlp, false); err != nil {
+		if err := pushProtobufRequest(pData, tlp, nil, false); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 
@@ -41,6 +41,26 @@ func TestPushProtoOk(t *testing.T) {
 		`{"_msg":"log-line-message","severity":"Trace"}`,
 	)
 
+	// severities mapping
+	f([]pb.ResourceLogs{
+		{
+			ScopeLogs: []pb.ScopeLogs{
+				{
+					LogRecords: []pb.LogRecord{
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 13, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 24, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
+					},
+				},
+			},
+		},
+	},
+		[]int64{1234, 1234, 1234},
+		`{"_msg":"log-line-message","severity":"Trace"}
+{"_msg":"log-line-message","severity":"Warn"}
+{"_msg":"log-line-message","severity":"Fatal4"}`,
+	)
+
 	// multi-line with resource attributes
 	f([]pb.ResourceLogs{
 		{
@@ -60,7 +80,7 @@ func TestPushProtoOk(t *testing.T) {
 				{
 					LogRecords: []pb.LogRecord{
 						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 21, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 25, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
 						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
 					},
 				},
@@ -68,9 +88,9 @@ func TestPushProtoOk(t *testing.T) {
 		},
 	},
 		[]int64{1234, 1235, 1236},
-		`{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Unspecified"}
-{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
+		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}
+{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
 	)
 
 	// multi-scope with resource attributes and multi-line
@@ -116,14 +136,71 @@ func TestPushProtoOk(t *testing.T) {
 		},
 	},
 		[]int64{1234, 1235, 2345, 2346, 2347, 2348, 3333},
-		`{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Debug"}
+		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Debug"}
 {"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
 {"_msg":"log-line-resource-scope-1-1-1","trace_id":"1234","span_id":"45","severity":"Info4"}
 {"_msg":"log-line-resource-scope-1-1-2","trace_id":"4bf92f3577b34da6a3ce929d0e0e4736","span_id":"00f067aa0ba902b7","severity":"Unspecified"}`,
 	)
+
+	// nested fields
+	f([]pb.ResourceLogs{
+		{
+			ScopeLogs: []pb.ScopeLogs{
+				{
+					LogRecords: []pb.LogRecord{
+						{
+							TimeUnixNano: 1234,
+							Body:         pb.AnyValue{StringValue: ptrTo("nested fields")},
+							Attributes: []*pb.KeyValue{
+								{Key: "error", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
+									{
+										Key:   "type",
+										Value: &pb.AnyValue{StringValue: ptrTo("document_parsing_exception")},
+									},
+									{
+										Key:   "reason",
+										Value: &pb.AnyValue{StringValue: ptrTo("failed to parse field [_msg] of type [text]")},
+									},
+									{
+										Key: "caused_by",
+										Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
+											{
+												Key:   "type",
+												Value: &pb.AnyValue{StringValue: ptrTo("x_content_parse_exception")},
+											},
+											{
+												Key:   "reason",
+												Value: &pb.AnyValue{StringValue: ptrTo("unexpected end-of-input in VALUE_STRING")},
+											},
+											{
+												Key: "caused_by",
+												Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
+													{
+														Key:   "type",
+														Value: &pb.AnyValue{StringValue: ptrTo("json_e_o_f_exception")},
+													},
+													{
+														Key:   "reason",
+														Value: &pb.AnyValue{StringValue: ptrTo("eof")},
+													},
+												}}},
+											},
+										}}},
+									},
+								}}}},
+							},
+						},
+					},
+				},
+			},
+		},
+	}, []int64{1234},
+		`{"_msg":"nested fields","error.type":"document_parsing_exception","error.reason":"failed to parse field [_msg] of type [text]",`+
+			`"error.caused_by.type":"x_content_parse_exception","error.caused_by.reason":"unexpected end-of-input in VALUE_STRING",`+
+			`"error.caused_by.caused_by.type":"json_e_o_f_exception","error.caused_by.caused_by.reason":"eof","severity":"Unspecified"}`)
 }
 
 func ptrTo[T any](s T) *T {

app/vlinsert/opentelemetry/opentelemetry_timing_test.go

@@ -27,7 +27,7 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			if err := pushProtobufRequest(body, blp, false); err != nil {
+			if err := pushProtobufRequest(body, blp, nil, false); err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}

app/vlinsert/syslog/syslog.go

@@ -45,6 +45,11 @@ var (
 	ignoreFieldsUDP = flagutil.NewArrayString("syslog.ignoreFields.udp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. "+
 		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
 
+	decolorizeFieldsTCP = flagutil.NewArrayString("syslog.decolorizeFields.tcp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.tcp. "+
+		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
+	decolorizeFieldsUDP = flagutil.NewArrayString("syslog.decolorizeFields.udp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.udp. "+
+		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
+
 	extraFieldsTCP = flagutil.NewArrayString("syslog.extraFields.tcp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. "+
 		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
 	extraFieldsUDP = flagutil.NewArrayString("syslog.extraFields.udp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. "+
@@ -188,6 +193,12 @@ func runUDPListener(addr string, argIdx int) {
 		logger.Fatalf("cannot parse -syslog.ignoreFields.udp=%q for -syslog.listenAddr.udp=%q: %s", ignoreFieldsStr, addr, err)
 	}
 
+	decolorizeFieldsStr := decolorizeFieldsUDP.GetOptionalArg(argIdx)
+	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.decolorizeFields.udp=%q for -syslog.listenAddr.udp=%q: %s", decolorizeFieldsStr, addr, err)
+	}
+
 	extraFieldsStr := extraFieldsUDP.GetOptionalArg(argIdx)
 	extraFields, err := parseExtraFields(extraFieldsStr)
 	if err != nil {
@@ -196,7 +207,7 @@ func runUDPListener(addr string, argIdx int) {
 
 	doneCh := make(chan struct{})
 	go func() {
-		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
+		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
 		close(doneCh)
 	}()
 
@@ -249,6 +260,12 @@ func runTCPListener(addr string, argIdx int) {
 		logger.Fatalf("cannot parse -syslog.ignoreFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", ignoreFieldsStr, addr, err)
 	}
 
+	decolorizeFieldsStr := decolorizeFieldsTCP.GetOptionalArg(argIdx)
+	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.decolorizeFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", decolorizeFieldsStr, addr, err)
+	}
+
 	extraFieldsStr := extraFieldsTCP.GetOptionalArg(argIdx)
 	extraFields, err := parseExtraFields(extraFieldsStr)
 	if err != nil {
@@ -257,7 +274,7 @@ func runTCPListener(addr string, argIdx int) {
 
 	doneCh := make(chan struct{})
 	go func() {
-		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
+		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
 		close(doneCh)
 	}()
 
@@ -279,7 +296,7 @@ func checkCompressMethod(compressMethod, addr, protocol string) {
 	}
 }
 
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
+func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
 	gomaxprocs := cgroup.AvailableCPUs()
 	var wg sync.WaitGroup
 	localAddr := ln.LocalAddr()
@@ -287,7 +304,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string,
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
+			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
 			var bb bytesutil.ByteBuffer
 			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
 			for {
@@ -321,7 +338,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string,
 	wg.Wait()
 }
 
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
+func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
 	var cm ingestserver.ConnsMap
 	cm.Init("syslog")
 
@@ -351,7 +368,7 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us
 
 		wg.Add(1)
 		go func() {
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
+			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
 			if err := processStream("tcp", c, encoding, useLocalTimestamp, cp); err != nil {
 				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
 			}
@@ -535,7 +552,7 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
 	if useLocalTimestamp {
 		ts = time.Now().UnixNano()
 	} else {
-		nsecs, err := insertutil.ExtractTimestampFromFields("timestamp", p.Fields)
+		nsecs, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
 		if err != nil {
 			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
 		}
@@ -548,6 +565,7 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
 	return nil
 }
 
+var timeFields = []string{"timestamp"}
 var msgFields = []string{"message"}
 
 var (

app/vlogscli/less_wrapper.go

@@ -17,7 +17,7 @@ func isTerminal() bool {
 	return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
 }
 
-func readWithLess(r io.Reader, wrapLongLines bool) error {
+func readWithLess(r io.Reader, disableColors, wrapLongLines bool) error {
 	if !isTerminal() {
 		// Just write everything to stdout if no terminal is available.
 		_, err := io.Copy(os.Stdout, r)
@@ -49,6 +49,9 @@ func readWithLess(r io.Reader, wrapLongLines bool) error {
 		return fmt.Errorf("cannot find 'less' command: %w", err)
 	}
 	opts := []string{"less", "-F", "-X"}
+	if !disableColors {
+		opts = append(opts, "-R")
+	}
 	if !wrapLongLines {
 		opts = append(opts, "-S")
 	}

app/vlogscli/main.go

@@ -91,6 +91,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 	}
 
 	outputMode := outputModeJSONMultiline
+	disableColors := true
 	wrapLongLines := false
 	s := ""
 	for {
@@ -100,7 +101,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 			case io.EOF:
 				if s != "" {
 					// This is non-interactive query execution.
-					executeQuery(context.Background(), rl, s, outputMode, wrapLongLines)
+					executeQuery(context.Background(), rl, s, outputMode, disableColors, wrapLongLines)
 				}
 				return
 			case readline.ErrInterrupt:
@@ -176,6 +177,24 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 			s = ""
 			continue
 		}
+		if s == `\disable_colors` {
+			if !disableColors {
+				disableColors = true
+				fmt.Fprintf(rl, `disabled colors in compact output mode; enter \enable_colors for enabling it`+"\n")
+			}
+			historyLines = pushToHistory(rl, historyLines, s)
+			s = ""
+			continue
+		}
+		if s == `\enable_colors` {
+			if disableColors {
+				disableColors = false
+				fmt.Fprintf(rl, `enabled colors in compact output mode; type \disable_colors for disabling it`+"\n")
+			}
+			historyLines = pushToHistory(rl, historyLines, s)
+			s = ""
+			continue
+		}
 		if line != "" && !strings.HasSuffix(line, ";") {
 			// Assume the query is incomplete and allow the user finishing the query on the next line
 			s += "\n"
@@ -185,7 +204,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 
 		// Execute the query
 		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		executeQuery(ctx, rl, s, outputMode, wrapLongLines)
+		executeQuery(ctx, rl, s, outputMode, disableColors, wrapLongLines)
 		cancel()
 
 		historyLines = pushToHistory(rl, historyLines, s)
@@ -273,13 +292,15 @@ func printCommandsHelp(w io.Writer) {
 \c - compact output mode
 \logfmt - logfmt output mode
 \wrap_long_lines - toggles wrapping long lines
+\enable_colors - enable ANSI colors in compact output mode
+\disable_colors - disable ANSI colors in compact output mode
 \tail <query> - live tail <query> results
 
 See https://docs.victoriametrics.com/victorialogs/querying/vlogscli/ for more details
 `)
 }
 
-func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, wrapLongLines bool) {
+func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, disableColors, wrapLongLines bool) {
 	if strings.HasPrefix(qStr, `\tail `) {
 		tailQuery(ctx, output, qStr, outputMode)
 		return
@@ -293,7 +314,7 @@ func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode
 		_ = respBody.Close()
 	}()
 
-	if err := readWithLess(respBody, wrapLongLines); err != nil {
+	if err := readWithLess(respBody, disableColors, wrapLongLines); err != nil {
 		fmt.Fprintf(output, "error when reading query response: %s\n", err)
 		return
 	}

app/vlogsgenerator/main.go

@@ -25,8 +25,8 @@ var (
 	addr    = flag.String("addr", "stdout", "HTTP address to push the generated logs to; if it is set to stdout, then logs are generated to stdout")
 	workers = flag.Int("workers", 1, "The number of workers to use to push logs to -addr")
 
-	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
-	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
+	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
+	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
 	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
 	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
 		"during data generation")
@@ -206,6 +206,8 @@ func generateAndPushLogs(cfg *workerConfig, workerID int) {
 	if err != nil {
 		logger.Fatalf("cannot perform request to %q: %s", cfg.url, err)
 	}
+	defer resp.Body.Close()
+
 	if resp.StatusCode/100 != 2 {
 		logger.Fatalf("unexpected status code got from %q: %d; want 2xx", cfg.url, err)
 	}

app/vlselect/internalselect/internalselect.go

@@ -0,0 +1,324 @@
+package internalselect
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+)
+
+var disableSelect = flag.Bool("internalselect.disable", false, "Whether to disable /internal/select/* HTTP endpoints")
+
+// RequestHandler processes requests to /internal/select/*
+func RequestHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if *disableSelect {
+		httpserver.Errorf(w, r, "requests to /internal/select/* are disabled with -internalselect.disable command-line flag")
+		return
+	}
+
+	startTime := time.Now()
+
+	path := r.URL.Path
+	rh := requestHandlers[path]
+	if rh == nil {
+		httpserver.Errorf(w, r, "unsupported endpoint requested: %s", path)
+		return
+	}
+
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_requests_total{path=%q}`, path)).Inc()
+	if err := rh(ctx, w, r); err != nil && !netutil.IsTrivialNetworkError(err) {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_request_errors_total{path=%q}`, path)).Inc()
+		httpserver.Errorf(w, r, "%s", err)
+		// The return is skipped intentionally in order to track the duration of failed queries.
+	}
+	metrics.GetOrCreateSummary(fmt.Sprintf(`vl_http_request_duration_seconds{path=%q}`, path)).UpdateDuration(startTime)
+}
+
+var requestHandlers = map[string]func(ctx context.Context, w http.ResponseWriter, r *http.Request) error{
+	"/internal/select/query":               processQueryRequest,
+	"/internal/select/field_names":         processFieldNamesRequest,
+	"/internal/select/field_values":        processFieldValuesRequest,
+	"/internal/select/stream_field_names":  processStreamFieldNamesRequest,
+	"/internal/select/stream_field_values": processStreamFieldValuesRequest,
+	"/internal/select/streams":             processStreamsRequest,
+	"/internal/select/stream_ids":          processStreamIDsRequest,
+}
+
+func processQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.QueryProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	w.Header().Set("Content-Type", "application/octet-stream")
+
+	var wLock sync.Mutex
+	var dataLenBuf []byte
+
+	sendBuf := func(bb *bytesutil.ByteBuffer) error {
+		if len(bb.B) == 0 {
+			return nil
+		}
+
+		data := bb.B
+		if !cp.DisableCompression {
+			bufLen := len(bb.B)
+			bb.B = zstd.CompressLevel(bb.B, bb.B, 1)
+			data = bb.B[bufLen:]
+		}
+
+		wLock.Lock()
+		dataLenBuf = encoding.MarshalUint64(dataLenBuf[:0], uint64(len(data)))
+		_, err := w.Write(dataLenBuf)
+		if err == nil {
+			_, err = w.Write(data)
+		}
+		wLock.Unlock()
+
+		// Reset the sent buf
+		bb.Reset()
+
+		return err
+	}
+
+	var bufs atomicutil.Slice[bytesutil.ByteBuffer]
+
+	var errGlobalLock sync.Mutex
+	var errGlobal error
+
+	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
+		if errGlobal != nil {
+			return
+		}
+
+		bb := bufs.Get(workerID)
+
+		bb.B = db.Marshal(bb.B)
+
+		if len(bb.B) < 1024*1024 {
+			// Fast path - the bb is too small to be sent to the client yet.
+			return
+		}
+
+		// Slow path - the bb must be sent to the client.
+		if err := sendBuf(bb); err != nil {
+			errGlobalLock.Lock()
+			if errGlobal != nil {
+				errGlobal = err
+			}
+			errGlobalLock.Unlock()
+		}
+	}
+
+	if err := vlstorage.RunQuery(ctx, cp.TenantIDs, cp.Query, writeBlock); err != nil {
+		return err
+	}
+	if errGlobal != nil {
+		return errGlobal
+	}
+
+	// Send the remaining data
+	for _, bb := range bufs.All() {
+		if err := sendBuf(bb); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func processFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.FieldNamesProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	fieldNames, err := vlstorage.GetFieldNames(ctx, cp.TenantIDs, cp.Query)
+	if err != nil {
+		return fmt.Errorf("cannot obtain field names: %w", err)
+	}
+
+	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
+}
+
+func processFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.FieldValuesProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	fieldName := r.FormValue("field")
+
+	limit, err := getInt64FromRequest(r, "limit")
+	if err != nil {
+		return err
+	}
+
+	fieldValues, err := vlstorage.GetFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
+	if err != nil {
+		return fmt.Errorf("cannot obtain field values: %w", err)
+	}
+
+	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
+}
+
+func processStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.StreamFieldNamesProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	fieldNames, err := vlstorage.GetStreamFieldNames(ctx, cp.TenantIDs, cp.Query)
+	if err != nil {
+		return fmt.Errorf("cannot obtain stream field names: %w", err)
+	}
+
+	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
+}
+
+func processStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.StreamFieldValuesProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	fieldName := r.FormValue("field")
+
+	limit, err := getInt64FromRequest(r, "limit")
+	if err != nil {
+		return err
+	}
+
+	fieldValues, err := vlstorage.GetStreamFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
+	if err != nil {
+		return fmt.Errorf("cannot obtain stream field values: %w", err)
+	}
+
+	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
+}
+
+func processStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.StreamsProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	limit, err := getInt64FromRequest(r, "limit")
+	if err != nil {
+		return err
+	}
+
+	streams, err := vlstorage.GetStreams(ctx, cp.TenantIDs, cp.Query, uint64(limit))
+	if err != nil {
+		return fmt.Errorf("cannot obtain streams: %w", err)
+	}
+
+	return writeValuesWithHits(w, streams, cp.DisableCompression)
+}
+
+func processStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.StreamIDsProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	limit, err := getInt64FromRequest(r, "limit")
+	if err != nil {
+		return err
+	}
+
+	streamIDs, err := vlstorage.GetStreamIDs(ctx, cp.TenantIDs, cp.Query, uint64(limit))
+	if err != nil {
+		return fmt.Errorf("cannot obtain streams: %w", err)
+	}
+
+	return writeValuesWithHits(w, streamIDs, cp.DisableCompression)
+}
+
+type commonParams struct {
+	TenantIDs []logstorage.TenantID
+	Query     *logstorage.Query
+
+	DisableCompression bool
+}
+
+func getCommonParams(r *http.Request, expectedProtocolVersion string) (*commonParams, error) {
+	version := r.FormValue("version")
+	if version != expectedProtocolVersion {
+		return nil, fmt.Errorf("unexpected version=%q; want %q", version, expectedProtocolVersion)
+	}
+
+	tenantIDsStr := r.FormValue("tenant_ids")
+	tenantIDs, err := logstorage.UnmarshalTenantIDs([]byte(tenantIDsStr))
+	if err != nil {
+		return nil, fmt.Errorf("cannot unmarshal tenant_ids=%q: %w", tenantIDsStr, err)
+	}
+
+	timestamp, err := getInt64FromRequest(r, "timestamp")
+	if err != nil {
+		return nil, err
+	}
+
+	qStr := r.FormValue("query")
+	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
+	if err != nil {
+		return nil, fmt.Errorf("cannot unmarshal query=%q: %w", qStr, err)
+	}
+
+	s := r.FormValue("disable_compression")
+	disableCompression, err := strconv.ParseBool(s)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse disable_compression=%q: %w", s, err)
+	}
+
+	cp := &commonParams{
+		TenantIDs: tenantIDs,
+		Query:     q,
+
+		DisableCompression: disableCompression,
+	}
+	return cp, nil
+}
+
+func writeValuesWithHits(w http.ResponseWriter, vhs []logstorage.ValueWithHits, disableCompression bool) error {
+	var b []byte
+	for i := range vhs {
+		b = vhs[i].Marshal(b)
+	}
+
+	if !disableCompression {
+		b = zstd.CompressLevel(nil, b, 1)
+	}
+
+	w.Header().Set("Content-Type", "application/octet-stream")
+
+	if _, err := w.Write(b); err != nil {
+		return fmt.Errorf("cannot send response to the client: %w", err)
+	}
+
+	return nil
+}
+
+func getInt64FromRequest(r *http.Request, argName string) (int64, error) {
+	s := r.FormValue(argName)
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse %s=%q: %w", argName, s, err)
+	}
+	return n, nil
+}

app/vlselect/logsql/buffered_writer.go

@@ -1,47 +0,0 @@
-package logsql
-
-import (
-	"bufio"
-	"io"
-	"sync"
-)
-
-func getBufferedWriter(w io.Writer) *bufferedWriter {
-	v := bufferedWriterPool.Get()
-	if v == nil {
-		return &bufferedWriter{
-			bw: bufio.NewWriter(w),
-		}
-	}
-	bw := v.(*bufferedWriter)
-	bw.bw.Reset(w)
-	return bw
-}
-
-func putBufferedWriter(bw *bufferedWriter) {
-	bw.reset()
-	bufferedWriterPool.Put(bw)
-}
-
-var bufferedWriterPool sync.Pool
-
-type bufferedWriter struct {
-	mu sync.Mutex
-	bw *bufio.Writer
-}
-
-func (bw *bufferedWriter) reset() {
-	// nothing to do
-}
-
-func (bw *bufferedWriter) WriteIgnoreErrors(p []byte) {
-	bw.mu.Lock()
-	_, _ = bw.bw.Write(p)
-	bw.mu.Unlock()
-}
-
-func (bw *bufferedWriter) FlushIgnoreErrors() {
-	bw.mu.Lock()
-	_ = bw.bw.Flush()
-	bw.mu.Unlock()
-}

app/vlselect/logsql/logsql.go

@@ -3,6 +3,7 @@ package logsql
 import (
 	"context"
 	"fmt"
+	"io"
 	"math"
 	"net/http"
 	"regexp"
@@ -11,12 +12,14 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastjson"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
@@ -52,15 +55,21 @@ func ProcessFacetsRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
 	}
 	keepConstFields := httputil.GetBool(r, "keep_const_fields")
 
+	// Pipes must be dropped, since it is expected facets are obtained
+	// from the real logs stored in the database.
 	q.DropAllPipes()
+
 	q.AddFacetsPipe(limit, maxValuesPerField, maxValueLen, keepConstFields)
 
 	var mLock sync.Mutex
 	m := make(map[string][]facetEntry)
-	writeBlock := func(_ uint, _ []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 || len(columns[0].Values) == 0 {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+		if rowsCount == 0 {
 			return
 		}
+
+		columns := db.Columns
 		if len(columns) != 3 {
 			logger.Panicf("BUG: expecting 3 columns; got %d columns", len(columns))
 		}
@@ -150,23 +159,27 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		fieldsLimit = 0
 	}
 
-	// Prepare the query for hits count.
+	// Pipes must be dropped, since it is expected hits are obtained
+	// from the real logs stored in the database.
 	q.DropAllPipes()
+
 	q.AddCountByTimePipe(int64(step), int64(offset), fields)
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 || len(columns[0].Values) == 0 {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+		if rowsCount == 0 {
 			return
 		}
 
+		columns := db.Columns
 		timestampValues := columns[0].Values
 		hitsValues := columns[len(columns)-1].Values
 		columns = columns[1 : len(columns)-1]
 
 		bb := blockResultPool.Get()
-		for i := range timestamps {
+		for i := 0; i < rowsCount; i++ {
 			timestampStr := strings.Clone(timestampValues[i])
 			hitsStr := strings.Clone(hitsValues[i])
 			hits, err := strconv.ParseUint(hitsStr, 10, 64)
@@ -205,6 +218,8 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	WriteHitsSeries(w, m)
 }
 
+var blockResultPool bytesutil.ByteBufferPool
+
 func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
 	if fieldsLimit <= 0 || fieldsLimit >= len(m) {
 		return m
@@ -280,6 +295,10 @@ func ProcessFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *htt
 		return
 	}
 
+	// Pipes must be dropped, since it is expected field names are obtained
+	// from the real logs stored in the database.
+	q.DropAllPipes()
+
 	// Obtain field names for the given query
 	fieldNames, err := vlstorage.GetFieldNames(ctx, tenantIDs, q)
 	if err != nil {
@@ -319,6 +338,10 @@ func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *ht
 		limit = 0
 	}
 
+	// Pipes must be dropped, since it is expected field values are obtained
+	// from the real logs stored in the database.
+	q.DropAllPipes()
+
 	// Obtain unique values for the given field
 	values, err := vlstorage.GetFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
 	if err != nil {
@@ -341,6 +364,10 @@ func ProcessStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter,
 		return
 	}
 
+	// Pipes must be dropped, since it is expected stream field names are obtained
+	// from the real logs stored in the database.
+	q.DropAllPipes()
+
 	// Obtain stream field names for the given query
 	names, err := vlstorage.GetStreamFieldNames(ctx, tenantIDs, q)
 	if err != nil {
@@ -379,6 +406,10 @@ func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter,
 		limit = 0
 	}
 
+	// Pipes must be dropped, since it is expected stream field values are obtained
+	// from the real logs stored in the database.
+	q.DropAllPipes()
+
 	// Obtain stream field values for the given query and the given fieldName
 	values, err := vlstorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
 	if err != nil {
@@ -410,6 +441,10 @@ func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http
 		limit = 0
 	}
 
+	// Pipes must be dropped, since it is expected stream ids are obtained
+	// from the real logs stored in the database.
+	q.DropAllPipes()
+
 	// Obtain streamIDs for the given query
 	streamIDs, err := vlstorage.GetStreamIDs(ctx, tenantIDs, q, uint64(limit))
 	if err != nil {
@@ -441,6 +476,10 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
 		limit = 0
 	}
 
+	// Pipes must be dropped, since it is expected stream are obtained
+	// from the real logs stored in the database.
+	q.DropAllPipes()
+
 	// Obtain streams for the given query
 	streams, err := vlstorage.GetStreams(ctx, tenantIDs, q, uint64(limit))
 	if err != nil {
@@ -504,6 +543,11 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 	if !ok {
 		logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
 	}
+
+	w.Header().Set("Content-Type", "application/x-ndjson")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	flusher.Flush()
+
 	qOrig := q
 	for {
 		q = qOrig.CloneWithTimeFilter(end, start, end)
@@ -566,8 +610,8 @@ func newTailProcessor(cancel func()) *tailProcessor {
 	}
 }
 
-func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-	if len(timestamps) == 0 {
+func (tp *tailProcessor) writeBlock(_ uint, db *logstorage.DataBlock) {
+	if db.RowsCount() == 0 {
 		return
 	}
 
@@ -579,14 +623,8 @@ func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logsto
 	}
 
 	// Make sure columns contain _time field, since it is needed for proper tail work.
-	hasTime := false
-	for _, c := range columns {
-		if c.Name == "_time" {
-			hasTime = true
-			break
-		}
-	}
-	if !hasTime {
+	timestamps, ok := db.GetTimestamps(nil)
+	if !ok {
 		tp.err = fmt.Errorf("missing _time field")
 		tp.cancel()
 		return
@@ -595,8 +633,8 @@ func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logsto
 	// Copy block rows to tp.perStreamRows
 	for i, timestamp := range timestamps {
 		streamID := ""
-		fields := make([]logstorage.Field, len(columns))
-		for j, c := range columns {
+		fields := make([]logstorage.Field, len(db.Columns))
+		for j, c := range db.Columns {
 			name := strings.Clone(c.Name)
 			value := strings.Clone(c.Values[i])
 
@@ -688,12 +726,15 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 	m := make(map[string]*statsSeries)
 	var mLock sync.Mutex
 
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+
+		columns := db.Columns
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
-		for i := range timestamps {
+		for i := 0; i < rowsCount; i++ {
 			// Do not move q.GetTimestamp() outside writeBlock, since ts
 			// must be initialized to query timestamp for every processed log row.
 			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8312
@@ -802,12 +843,14 @@ func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *htt
 	var rowsLock sync.Mutex
 
 	timestamp := q.GetTimestamp()
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+		columns := db.Columns
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
-		for i := range timestamps {
+		for i := 0; i < rowsCount; i++ {
 			labels := make([]logstorage.Field, 0, len(byFields))
 			for j, c := range columns {
 				if slices.Contains(byFields, c.Name) {
@@ -869,11 +912,21 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	bw := getBufferedWriter(w)
+	sw := &syncWriter{
+		w: w,
+	}
+
+	var bwShards atomicutil.Slice[bufferedWriter]
+	bwShards.Init = func(shard *bufferedWriter) {
+		shard.sw = sw
+	}
 	defer func() {
-		bw.FlushIgnoreErrors()
-		putBufferedWriter(bw)
+		shards := bwShards.All()
+		for _, shard := range shards {
+			shard.FlushIgnoreErrors()
+		}
 	}()
+
 	w.Header().Set("Content-Type", "application/stream+json")
 
 	if limit > 0 {
@@ -883,32 +936,34 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 				httpserver.Errorf(w, r, "%s", err)
 				return
 			}
-			bb := blockResultPool.Get()
-			b := bb.B
+			bw := bwShards.Get(0)
 			for i := range rows {
-				b = logstorage.MarshalFieldsToJSON(b[:0], rows[i].fields)
-				b = append(b, '\n')
-				bw.WriteIgnoreErrors(b)
+				bw.buf = logstorage.MarshalFieldsToJSON(bw.buf, rows[i].fields)
+				bw.buf = append(bw.buf, '\n')
+				if len(bw.buf) > 16*1024 {
+					bw.FlushIgnoreErrors()
+				}
 			}
-			bb.B = b
-			blockResultPool.Put(bb)
 			return
 		}
 
 		q.AddPipeLimit(uint64(limit))
 	}
 
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 || len(columns[0].Values) == 0 {
+	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+		if rowsCount == 0 {
 			return
 		}
+		columns := db.Columns
 
-		bb := blockResultPool.Get()
-		for i := range timestamps {
-			WriteJSONRow(bb, columns, i)
+		bw := bwShards.Get(workerID)
+		for i := 0; i < rowsCount; i++ {
+			WriteJSONRow(bw, columns, i)
+			if len(bw.buf) > 16*1024 {
+				bw.FlushIgnoreErrors()
+			}
 		}
-		bw.WriteIgnoreErrors(bb.B)
-		blockResultPool.Put(bb)
 	}
 
 	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
@@ -917,14 +972,37 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 	}
 }
 
-var blockResultPool bytesutil.ByteBufferPool
-
-type row struct {
-	timestamp int64
-	fields    []logstorage.Field
+type syncWriter struct {
+	mu sync.Mutex
+	w  io.Writer
 }
 
-func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+func (sw *syncWriter) Write(p []byte) (int, error) {
+	sw.mu.Lock()
+	n, err := sw.w.Write(p)
+	sw.mu.Unlock()
+	return n, err
+}
+
+type bufferedWriter struct {
+	buf []byte
+	sw  *syncWriter
+}
+
+func (bw *bufferedWriter) Write(p []byte) (int, error) {
+	bw.buf = append(bw.buf, p...)
+
+	// Do not send bw.buf to bw.sw here, since the data at bw.buf may be incomplete (it must end with '\n')
+
+	return len(p), nil
+}
+
+func (bw *bufferedWriter) FlushIgnoreErrors() {
+	_, _ = bw.sw.Write(bw.buf)
+	bw.buf = bw.buf[:0]
+}
+
+func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
 	limitUpper := 2 * limit
 	q.AddPipeLimit(uint64(limitUpper))
 
@@ -993,28 +1071,39 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 	}
 }
 
-func getLastNRows(rows []row, limit int) []row {
-	sort.Slice(rows, func(i, j int) bool {
-		return rows[i].timestamp < rows[j].timestamp
-	})
+func getLastNRows(rows []logRow, limit int) []logRow {
+	sortLogRows(rows)
 	if len(rows) > limit {
 		rows = rows[len(rows)-limit:]
 	}
 	return rows
 }
 
-func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
 	ctxWithCancel, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	var rows []row
+	var missingTimeColumn atomic.Bool
+	var rows []logRow
 	var rowsLock sync.Mutex
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		if missingTimeColumn.Load() {
+			return
+		}
+
+		columns := db.Columns
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
 
+		timestamps, ok := db.GetTimestamps(nil)
+		if !ok {
+			missingTimeColumn.Store(true)
+			cancel()
+			return
+		}
+
 		for i, timestamp := range timestamps {
 			fields := make([]logstorage.Field, len(columns))
 			for j := range columns {
@@ -1024,7 +1113,7 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 			}
 
 			rowsLock.Lock()
-			rows = append(rows, row{
+			rows = append(rows, logRow{
 				timestamp: timestamp,
 				fields:    fields,
 			})
@@ -1035,11 +1124,13 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 			cancel()
 		}
 	}
-	if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock); err != nil {
-		return nil, err
+	err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock)
+
+	if missingTimeColumn.Load() {
+		return nil, fmt.Errorf("missing _time column in the result for the query [%s]", q)
 	}
 
-	return rows, nil
+	return rows, err
 }
 
 func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, error) {
@@ -1098,20 +1189,22 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 	}
 
 	// Parse optional extra_filters
-	extraFiltersStr := r.FormValue("extra_filters")
-	extraFilters, err := parseExtraFilters(extraFiltersStr)
-	if err != nil {
-		return nil, nil, err
+	for _, extraFiltersStr := range r.Form["extra_filters"] {
+		extraFilters, err := parseExtraFilters(extraFiltersStr)
+		if err != nil {
+			return nil, nil, err
+		}
+		q.AddExtraFilters(extraFilters)
 	}
-	q.AddExtraFilters(extraFilters)
 
 	// Parse optional extra_stream_filters
-	extraStreamFiltersStr := r.FormValue("extra_stream_filters")
-	extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr)
-	if err != nil {
-		return nil, nil, err
+	for _, extraStreamFiltersStr := range r.Form["extra_stream_filters"] {
+		extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr)
+		if err != nil {
+			return nil, nil, err
+		}
+		q.AddExtraFilters(extraStreamFilters)
 	}
-	q.AddExtraFilters(extraStreamFilters)
 
 	return q, tenantIDs, nil
 }

app/vlselect/main.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/internalselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -72,7 +73,7 @@ var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 
-	if !strings.HasPrefix(path, "/select/") {
+	if !strings.HasPrefix(path, "/select/") && !strings.HasPrefix(path, "/internal/select/") {
 		// Skip requests, which do not start with /select/, since these aren't our requests.
 		return false
 	}
@@ -119,12 +120,24 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	defer decRequestConcurrency()
 
+	if strings.HasPrefix(path, "/internal/select/") {
+		// Process internal request from vlselect without timeout (e.g. use ctx instead of ctxWithTimeout),
+		// since the timeout must be controlled by the vlselect.
+		internalselect.RequestHandler(ctx, w, r)
+		return true
+	}
+
 	ok := processSelectRequest(ctxWithTimeout, w, r, path)
 	if !ok {
 		return false
 	}
 
-	err := ctxWithTimeout.Err()
+	logRequestErrorIfNeeded(ctxWithTimeout, w, r, startTime)
+	return true
+}
+
+func logRequestErrorIfNeeded(ctx context.Context, w http.ResponseWriter, r *http.Request, startTime time.Time) {
+	err := ctx.Err()
 	switch err {
 	case nil:
 		// nothing to do
@@ -140,8 +153,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	default:
 		httpserver.Errorf(w, r, "unexpected error: %s", err)
 	}
-
-	return true
 }
 
 func incRequestConcurrency(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

app/vlselect/vmui/index.html

@@ -35,10 +35,10 @@
   <meta property="og:title" content="UI for VictoriaLogs">
   <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
   <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-BgdvCSTM.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
+  <script type="module" crossorigin src="./assets/index-BaRvaPfA.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-u4IOGr0E.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-C85_NB5q.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vlstorage/main.go

@@ -10,11 +10,14 @@ import (
 
 	"github.com/VictoriaMetrics/metrics"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 var (
@@ -38,15 +41,59 @@ var (
 	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
 		"the storage stops accepting new data")
 
-	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
+	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge . It overrides -httpAuth.* . "+
+		"See https://docs.victoriametrics.com/victorialogs/#forced-merge")
+	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush . It overrides -httpAuth.* . "+
+		"See https://docs.victoriametrics.com/victorialogs/#forced-flush")
+
+	storageNodeAddrs = flagutil.NewArrayString("storageNode", "Comma-separated list of TCP addresses for storage nodes to route the ingested logs to and to send select queries to. "+
+		"If the list is empty, then the ingested logs are stored and queried locally from -storageDataPath")
+	insertConcurrency        = flag.Int("insert.concurrency", 2, "The average number of concurrent data ingestion requests, which can be sent to every -storageNode")
+	insertDisableCompression = flag.Bool("insert.disableCompression", false, "Whether to disable compression when sending the ingested data to -storageNode nodes. "+
+		"Disabled compression reduces CPU usage at the cost of higher network usage")
+	selectDisableCompression = flag.Bool("select.disableCompression", false, "Whether to disable compression for select query responses received from -storageNode nodes. "+
+		"Disabled compression reduces CPU usage at the cost of higher network usage")
+
+	storageNodeUsername     = flagutil.NewArrayString("storageNode.username", "Optional basic auth username to use for the corresponding -storageNode")
+	storageNodePassword     = flagutil.NewArrayString("storageNode.password", "Optional basic auth password to use for the corresponding -storageNode")
+	storageNodePasswordFile = flagutil.NewArrayString("storageNode.passwordFile", "Optional path to basic auth password to use for the corresponding -storageNode. "+
+		"The file is re-read every second")
+	storageNodeBearerToken     = flagutil.NewArrayString("storageNode.bearerToken", "Optional bearer auth token to use for the corresponding -storageNode")
+	storageNodeBearerTokenFile = flagutil.NewArrayString("storageNode.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -storageNode. "+
+		"The token is re-read from the file every second")
+
+	storageNodeTLS = flagutil.NewArrayBool("storageNode.tls", "Whether to use TLS (HTTPS) protocol for communicating with the corresponding -storageNode. "+
+		"By default communication is performed via HTTP")
+	storageNodeTLSCAFile = flagutil.NewArrayString("storageNode.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -storageNode. "+
+		"By default, system CA is used")
+	storageNodeTLSCertFile = flagutil.NewArrayString("storageNode.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
+		"to the corresponding -storageNode")
+	storageNodeTLSKeyFile    = flagutil.NewArrayString("storageNode.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -storageNode")
+	storageNodeTLSServerName = flagutil.NewArrayString("storageNode.tlsServerName", "Optional TLS server name to use for connections to the corresponding -storageNode. "+
+		"By default, the server name from -storageNode is used")
+	storageNodeTLSInsecureSkipVerify = flagutil.NewArrayBool("storageNode.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -storageNode")
 )
 
+var localStorage *logstorage.Storage
+var localStorageMetrics *metrics.Set
+
+var netstorageInsert *netinsert.Storage
+var netstorageSelect *netselect.Storage
+
 // Init initializes vlstorage.
 //
 // Stop must be called when vlstorage is no longer needed
 func Init() {
-	if strg != nil {
-		logger.Panicf("BUG: Init() has been already called")
+	if len(*storageNodeAddrs) == 0 {
+		initLocalStorage()
+	} else {
+		initNetworkStorage()
+	}
+}
+
+func initLocalStorage() {
+	if localStorage != nil {
+		logger.Panicf("BUG: initLocalStorage() has been already called")
 	}
 
 	if retentionPeriod.Duration() < 24*time.Hour {
@@ -63,60 +110,157 @@ func Init() {
 	}
 	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
 	startTime := time.Now()
-	strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
+	localStorage = logstorage.MustOpenStorage(*storageDataPath, cfg)
 
 	var ss logstorage.StorageStats
-	strg.UpdateStats(&ss)
+	localStorage.UpdateStats(&ss)
 	logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
 		"smallPartSize: %d bytes; bigPartSize: %d bytes",
 		time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
 		ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
 
-	// register storage metrics
-	storageMetrics = metrics.NewSet()
-	storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, strg)
+	// register local storage metrics
+	localStorageMetrics = metrics.NewSet()
+	localStorageMetrics.RegisterMetricsWriter(func(w io.Writer) {
+		writeStorageMetrics(w, localStorage)
 	})
-	metrics.RegisterSet(storageMetrics)
+	metrics.RegisterSet(localStorageMetrics)
+}
+
+func initNetworkStorage() {
+	if netstorageInsert != nil || netstorageSelect != nil {
+		logger.Panicf("BUG: initNetworkStorage() has been already called")
+	}
+
+	authCfgs := make([]*promauth.Config, len(*storageNodeAddrs))
+	isTLSs := make([]bool, len(*storageNodeAddrs))
+	for i := range authCfgs {
+		authCfgs[i] = newAuthConfigForStorageNode(i)
+		isTLSs[i] = storageNodeTLS.GetOptionalArg(i)
+	}
+
+	logger.Infof("starting insert service for nodes %s", *storageNodeAddrs)
+	netstorageInsert = netinsert.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *insertConcurrency, *insertDisableCompression)
+
+	logger.Infof("initializing select service for nodes %s", *storageNodeAddrs)
+	netstorageSelect = netselect.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *selectDisableCompression)
+
+	logger.Infof("initialized all the network services")
+}
+
+func newAuthConfigForStorageNode(argIdx int) *promauth.Config {
+	username := storageNodeUsername.GetOptionalArg(argIdx)
+	password := storageNodePassword.GetOptionalArg(argIdx)
+	passwordFile := storageNodePasswordFile.GetOptionalArg(argIdx)
+	var basicAuthCfg *promauth.BasicAuthConfig
+	if username != "" || password != "" || passwordFile != "" {
+		basicAuthCfg = &promauth.BasicAuthConfig{
+			Username:     username,
+			Password:     promauth.NewSecret(password),
+			PasswordFile: passwordFile,
+		}
+	}
+
+	token := storageNodeBearerToken.GetOptionalArg(argIdx)
+	tokenFile := storageNodeBearerTokenFile.GetOptionalArg(argIdx)
+
+	tlsCfg := &promauth.TLSConfig{
+		CAFile:             storageNodeTLSCAFile.GetOptionalArg(argIdx),
+		CertFile:           storageNodeTLSCertFile.GetOptionalArg(argIdx),
+		KeyFile:            storageNodeTLSKeyFile.GetOptionalArg(argIdx),
+		ServerName:         storageNodeTLSServerName.GetOptionalArg(argIdx),
+		InsecureSkipVerify: storageNodeTLSInsecureSkipVerify.GetOptionalArg(argIdx),
+	}
+
+	opts := &promauth.Options{
+		BasicAuth:       basicAuthCfg,
+		BearerToken:     token,
+		BearerTokenFile: tokenFile,
+		TLSConfig:       tlsCfg,
+	}
+	ac, err := opts.NewConfig()
+	if err != nil {
+		logger.Panicf("FATAL: cannot populate auth config for storage node #%d: %s", argIdx, err)
+	}
+
+	return ac
 }
 
 // Stop stops vlstorage.
 func Stop() {
-	metrics.UnregisterSet(storageMetrics, true)
-	storageMetrics = nil
+	if localStorage != nil {
+		metrics.UnregisterSet(localStorageMetrics, true)
+		localStorageMetrics = nil
 
-	strg.MustClose()
-	strg = nil
+		localStorage.MustClose()
+		localStorage = nil
+	} else {
+		netstorageInsert.MustStop()
+		netstorageInsert = nil
+
+		netstorageSelect.MustStop()
+		netstorageSelect = nil
+	}
 }
 
 // RequestHandler is a storage request handler.
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
-	if path == "/internal/force_merge" {
-		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
-			return true
-		}
-		// Run force merge in background
-		partitionNamePrefix := r.FormValue("partition_prefix")
-		go func() {
-			activeForceMerges.Inc()
-			defer activeForceMerges.Dec()
-			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
-			startTime := time.Now()
-			strg.MustForceMerge(partitionNamePrefix)
-			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
-		}()
-		return true
+	switch path {
+	case "/internal/force_merge":
+		return processForceMerge(w, r)
+	case "/internal/force_flush":
+		return processForceFlush(w, r)
 	}
 	return false
 }
 
-var strg *logstorage.Storage
-var storageMetrics *metrics.Set
+func processForceMerge(w http.ResponseWriter, r *http.Request) bool {
+	if localStorage == nil {
+		// Force merge isn't supported by non-local storage
+		return false
+	}
 
-// CanWriteData returns non-nil error if it cannot write data to vlstorage.
+	if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
+		return true
+	}
+
+	// Run force merge in background
+	partitionNamePrefix := r.FormValue("partition_prefix")
+	go func() {
+		activeForceMerges.Inc()
+		defer activeForceMerges.Dec()
+		logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
+		startTime := time.Now()
+		localStorage.MustForceMerge(partitionNamePrefix)
+		logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
+	}()
+	return true
+}
+
+func processForceFlush(w http.ResponseWriter, r *http.Request) bool {
+	if localStorage == nil {
+		// Force merge isn't supported by non-local storage
+		return false
+	}
+
+	if !httpserver.CheckAuthFlag(w, r, forceFlushAuthKey) {
+		return true
+	}
+
+	logger.Infof("flushing storage to make pending data available for reading")
+	localStorage.DebugFlush()
+	return true
+}
+
+// CanWriteData returns non-nil error if it cannot write data to vlstorage
 func CanWriteData() error {
-	if strg.IsReadOnly() {
+	if localStorage == nil {
+		// The data can be always written in non-local mode.
+		return nil
+	}
+
+	if localStorage.IsReadOnly() {
 		return &httpserver.ErrorWithStatusCode{
 			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
 				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
@@ -130,50 +274,77 @@ func CanWriteData() error {
 //
 // It is advised to call CanWriteData() before calling MustAddRows()
 func MustAddRows(lr *logstorage.LogRows) {
-	strg.MustAddRows(lr)
+	if localStorage != nil {
+		// Store lr in the local storage.
+		localStorage.MustAddRows(lr)
+	} else {
+		// Store lr across the remote storage nodes.
+		lr.ForEachRow(netstorageInsert.AddRow)
+	}
 }
 
 // RunQuery runs the given q and calls writeBlock for the returned data blocks
-func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteBlockFunc) error {
-	return strg.RunQuery(ctx, tenantIDs, q, writeBlock)
+func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
+	if localStorage != nil {
+		return localStorage.RunQuery(ctx, tenantIDs, q, writeBlock)
+	}
+	return netstorageSelect.RunQuery(ctx, tenantIDs, q, writeBlock)
 }
 
 // GetFieldNames executes q and returns field names seen in results.
 func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return strg.GetFieldNames(ctx, tenantIDs, q)
+	if localStorage != nil {
+		return localStorage.GetFieldNames(ctx, tenantIDs, q)
+	}
+	return netstorageSelect.GetFieldNames(ctx, tenantIDs, q)
 }
 
 // GetFieldValues executes q and returns unique values for the fieldName seen in results.
 //
 // If limit > 0, then up to limit unique values are returned.
 func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	if localStorage != nil {
+		return localStorage.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	}
+	return netstorageSelect.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
 }
 
 // GetStreamFieldNames executes q and returns stream field names seen in results.
 func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamFieldNames(ctx, tenantIDs, q)
+	if localStorage != nil {
+		return localStorage.GetStreamFieldNames(ctx, tenantIDs, q)
+	}
+	return netstorageSelect.GetStreamFieldNames(ctx, tenantIDs, q)
 }
 
 // GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
 //
 // If limit > 0, then up to limit unique stream field values are returned.
 func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	if localStorage != nil {
+		return localStorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	}
+	return netstorageSelect.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
 }
 
 // GetStreams executes q and returns streams seen in query results.
 //
 // If limit > 0, then up to limit unique streams are returned.
 func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreams(ctx, tenantIDs, q, limit)
+	if localStorage != nil {
+		return localStorage.GetStreams(ctx, tenantIDs, q, limit)
+	}
+	return netstorageSelect.GetStreams(ctx, tenantIDs, q, limit)
 }
 
 // GetStreamIDs executes q and returns streamIDs seen in query results.
 //
 // If limit > 0, then up to limit unique streamIDs are returned.
 func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamIDs(ctx, tenantIDs, q, limit)
+	if localStorage != nil {
+		return localStorage.GetStreamIDs(ctx, tenantIDs, q, limit)
+	}
+	return netstorageSelect.GetStreamIDs(ctx, tenantIDs, q, limit)
 }
 
 func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {

app/vlstorage/netinsert/netinsert.go

@@ -0,0 +1,369 @@
+package netinsert
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/valyala/fastrand"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+)
+
+// the maximum size of a single data block sent to storage node.
+const maxInsertBlockSize = 2 * 1024 * 1024
+
+// ProtocolVersion is the version of the data ingestion protocol.
+//
+// It must be changed every time the data encoding at /internal/insert HTTP endpoint is changed.
+const ProtocolVersion = "v1"
+
+// Storage is a network storage for sending data to remote storage nodes in the cluster.
+type Storage struct {
+	sns []*storageNode
+
+	disableCompression bool
+
+	srt *streamRowsTracker
+
+	pendingDataBuffers chan *bytesutil.ByteBuffer
+
+	stopCh chan struct{}
+	wg     sync.WaitGroup
+}
+
+type storageNode struct {
+	// scheme is http or https scheme to communicate with addr
+	scheme string
+
+	// addr is TCP address of storage node to send the ingested data to
+	addr string
+
+	// s is a storage, which holds the given storageNode
+	s *Storage
+
+	// c is an http client used for sending data blocks to addr.
+	c *http.Client
+
+	// ac is auth config used for setting request headers such as Authorization and Host.
+	ac *promauth.Config
+
+	// pendingData contains pending data, which must be sent to the storage node at the addr.
+	pendingDataMu        sync.Mutex
+	pendingData          *bytesutil.ByteBuffer
+	pendingDataLastFlush time.Time
+
+	// the unix timestamp until the storageNode is disabled for data writing.
+	disabledUntil atomic.Uint64
+}
+
+func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
+	tr := httputil.NewTransport(false, "vlinsert_backend")
+	tr.TLSHandshakeTimeout = 20 * time.Second
+	tr.DisableCompression = true
+
+	scheme := "http"
+	if isTLS {
+		scheme = "https"
+	}
+
+	sn := &storageNode{
+		scheme: scheme,
+		addr:   addr,
+		s:      s,
+		c: &http.Client{
+			Transport: ac.NewRoundTripper(tr),
+		},
+		ac: ac,
+
+		pendingData: &bytesutil.ByteBuffer{},
+	}
+
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		sn.backgroundFlusher()
+	}()
+
+	return sn
+}
+
+func (sn *storageNode) backgroundFlusher() {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-sn.s.stopCh:
+			return
+		case <-t.C:
+			sn.flushPendingData()
+		}
+	}
+}
+
+func (sn *storageNode) flushPendingData() {
+	sn.pendingDataMu.Lock()
+	if time.Since(sn.pendingDataLastFlush) < time.Second {
+		// nothing to flush
+		sn.pendingDataMu.Unlock()
+		return
+	}
+
+	pendingData := sn.grabPendingDataForFlushLocked()
+	sn.pendingDataMu.Unlock()
+
+	sn.mustSendInsertRequest(pendingData)
+}
+
+func (sn *storageNode) addRow(r *logstorage.InsertRow) {
+	bb := bbPool.Get()
+	b := bb.B
+
+	b = r.Marshal(b)
+
+	if len(b) > maxInsertBlockSize {
+		logger.Warnf("skipping too long log entry, since its length exceeds %d bytes; the actual log entry length is %d bytes; log entry contents: %s", maxInsertBlockSize, len(b), b)
+		return
+	}
+
+	var pendingData *bytesutil.ByteBuffer
+	sn.pendingDataMu.Lock()
+	if sn.pendingData.Len()+len(b) > maxInsertBlockSize {
+		pendingData = sn.grabPendingDataForFlushLocked()
+	}
+	sn.pendingData.MustWrite(b)
+	sn.pendingDataMu.Unlock()
+
+	bb.B = b
+	bbPool.Put(bb)
+
+	if pendingData != nil {
+		sn.mustSendInsertRequest(pendingData)
+	}
+}
+
+var bbPool bytesutil.ByteBufferPool
+
+func (sn *storageNode) grabPendingDataForFlushLocked() *bytesutil.ByteBuffer {
+	sn.pendingDataLastFlush = time.Now()
+	pendingData := sn.pendingData
+	sn.pendingData = <-sn.s.pendingDataBuffers
+
+	return pendingData
+}
+
+func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) {
+	defer func() {
+		pendingData.Reset()
+		sn.s.pendingDataBuffers <- pendingData
+	}()
+
+	err := sn.sendInsertRequest(pendingData)
+	if err == nil {
+		return
+	}
+
+	if !errors.Is(err, errTemporarilyDisabled) {
+		logger.Warnf("%s; re-routing the data block to the remaining nodes", err)
+	}
+	for !sn.s.sendInsertRequestToAnyNode(pendingData) {
+		logger.Errorf("cannot send pending data to all storage nodes, since all of them are unavailable; re-trying to send the data in a second")
+
+		t := timerpool.Get(time.Second)
+		select {
+		case <-sn.s.stopCh:
+			timerpool.Put(t)
+			logger.Errorf("dropping %d bytes of data, since there are no available storage nodes", pendingData.Len())
+			return
+		case <-t.C:
+			timerpool.Put(t)
+		}
+	}
+}
+
+func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) error {
+	dataLen := pendingData.Len()
+	if dataLen == 0 {
+		// Nothing to send.
+		return nil
+	}
+
+	if sn.disabledUntil.Load() > fasttime.UnixTimestamp() {
+		return errTemporarilyDisabled
+	}
+
+	ctx, cancel := contextutil.NewStopChanContext(sn.s.stopCh)
+	defer cancel()
+
+	var body io.Reader
+	if !sn.s.disableCompression {
+		bb := zstdBufPool.Get()
+		defer zstdBufPool.Put(bb)
+
+		bb.B = zstd.CompressLevel(bb.B[:0], pendingData.B, 1)
+		body = bb.NewReader()
+	} else {
+		body = pendingData.NewReader()
+	}
+
+	reqURL := sn.getRequestURL("/internal/insert")
+	req, err := http.NewRequestWithContext(ctx, "POST", reqURL, body)
+	if err != nil {
+		logger.Panicf("BUG: unexpected error when creating an http request: %s", err)
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	if !sn.s.disableCompression {
+		req.Header.Set("Content-Encoding", "zstd")
+	}
+	if err := sn.ac.SetHeaders(req, true); err != nil {
+		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
+	}
+
+	resp, err := sn.c.Do(req)
+	if err != nil {
+		// Disable sn for data writing for 10 seconds.
+		sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
+
+		return fmt.Errorf("cannot send data block with the length %d to %q: %s", pendingData.Len(), reqURL, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode/100 == 2 {
+		return nil
+	}
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		respBody = []byte(fmt.Sprintf("%s", err))
+	}
+
+	// Disable sn for data writing for 10 seconds.
+	sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
+
+	return fmt.Errorf("unexpected status code returned when sending data block to %q: %d; want 2xx; response body: %q", reqURL, resp.StatusCode, respBody)
+}
+
+func (sn *storageNode) getRequestURL(path string) string {
+	return fmt.Sprintf("%s://%s%s?version=%s", sn.scheme, sn.addr, path, url.QueryEscape(ProtocolVersion))
+}
+
+var zstdBufPool bytesutil.ByteBufferPool
+
+// NewStorage returns new Storage for the given addrs with the given authCfgs.
+//
+// The concurrency is the average number of concurrent connections per every addr.
+//
+// If disableCompression is set, then the data is sent uncompressed to the remote storage.
+//
+// Call MustStop on the returned storage when it is no longer needed.
+func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, concurrency int, disableCompression bool) *Storage {
+	pendingDataBuffers := make(chan *bytesutil.ByteBuffer, concurrency*len(addrs))
+	for i := 0; i < cap(pendingDataBuffers); i++ {
+		pendingDataBuffers <- &bytesutil.ByteBuffer{}
+	}
+
+	s := &Storage{
+		disableCompression: disableCompression,
+		pendingDataBuffers: pendingDataBuffers,
+		stopCh:             make(chan struct{}),
+	}
+
+	sns := make([]*storageNode, len(addrs))
+	for i, addr := range addrs {
+		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
+	}
+	s.sns = sns
+
+	s.srt = newStreamRowsTracker(len(sns))
+
+	return s
+}
+
+// MustStop stops the s.
+func (s *Storage) MustStop() {
+	close(s.stopCh)
+	s.wg.Wait()
+	s.sns = nil
+}
+
+// AddRow adds the given log row into s.
+func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
+	idx := s.srt.getNodeIdx(streamHash)
+	sn := s.sns[idx]
+	sn.addRow(r)
+}
+
+func (s *Storage) sendInsertRequestToAnyNode(pendingData *bytesutil.ByteBuffer) bool {
+	startIdx := int(fastrand.Uint32n(uint32(len(s.sns))))
+	for i := range s.sns {
+		idx := (startIdx + i) % len(s.sns)
+		sn := s.sns[idx]
+		err := sn.sendInsertRequest(pendingData)
+		if err == nil {
+			return true
+		}
+		if !errors.Is(err, errTemporarilyDisabled) {
+			logger.Warnf("cannot send pending data to the storage node %q: %s; trying to send it to another storage node", sn.addr, err)
+		}
+	}
+	return false
+}
+
+var errTemporarilyDisabled = fmt.Errorf("writing to the node is temporarily disabled")
+
+type streamRowsTracker struct {
+	mu sync.Mutex
+
+	nodesCount    int64
+	rowsPerStream map[uint64]uint64
+}
+
+func newStreamRowsTracker(nodesCount int) *streamRowsTracker {
+	return &streamRowsTracker{
+		nodesCount:    int64(nodesCount),
+		rowsPerStream: make(map[uint64]uint64),
+	}
+}
+
+func (srt *streamRowsTracker) getNodeIdx(streamHash uint64) uint64 {
+	if srt.nodesCount == 1 {
+		// Fast path for a single node.
+		return 0
+	}
+
+	srt.mu.Lock()
+	defer srt.mu.Unlock()
+
+	streamRows := srt.rowsPerStream[streamHash] + 1
+	srt.rowsPerStream[streamHash] = streamRows
+
+	if streamRows <= 1000 {
+		// Write the initial rows for the stream to a single storage node for better locality.
+		// This should work great for log streams containing small number of logs, since will be distributed
+		// evenly among available storage nodes because they have different streamHash.
+		return streamHash % uint64(srt.nodesCount)
+	}
+
+	// The log stream contains more than 1000 rows. Distribute them among storage nodes at random
+	// in order to improve query performance over this stream (the data for the log stream
+	// can be processed in parallel on all the storage nodes).
+	//
+	// The random distribution is preferred over round-robin distribution in order to avoid possible
+	// dependency between the order of the ingested logs and the number of storage nodes,
+	// which may lead to non-uniform distribution of logs among storage nodes.
+	return uint64(fastrand.Uint32n(uint32(srt.nodesCount)))
+}

app/vlstorage/netinsert/netinsert_test.go

@@ -0,0 +1,57 @@
+package netinsert
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"testing"
+
+	"github.com/cespare/xxhash/v2"
+)
+
+func TestStreamRowsTracker(t *testing.T) {
+	f := func(rowsCount, streamsCount, nodesCount int) {
+		t.Helper()
+
+		// generate stream hashes
+		streamHashes := make([]uint64, streamsCount)
+		for i := range streamHashes {
+			streamHashes[i] = xxhash.Sum64([]byte(fmt.Sprintf("stream %d.", i)))
+		}
+
+		srt := newStreamRowsTracker(nodesCount)
+
+		rng := rand.New(rand.NewSource(0))
+		rowsPerNode := make([]uint64, nodesCount)
+		for i := 0; i < rowsCount; i++ {
+			streamIdx := rng.Intn(streamsCount)
+			h := streamHashes[streamIdx]
+			nodeIdx := srt.getNodeIdx(h)
+			rowsPerNode[nodeIdx]++
+		}
+
+		// Verify that rows are uniformly distributed among nodes.
+		expectedRowsPerNode := float64(rowsCount) / float64(nodesCount)
+		for nodeIdx, nodeRows := range rowsPerNode {
+			if math.Abs(float64(nodeRows)-expectedRowsPerNode)/expectedRowsPerNode > 0.15 {
+				t.Fatalf("non-uniform distribution of rows among nodes; node %d has %d rows, while it must have %v rows; rowsPerNode=%d",
+					nodeIdx, nodeRows, expectedRowsPerNode, rowsPerNode)
+			}
+		}
+	}
+
+	rowsCount := 10000
+	streamsCount := 9
+	nodesCount := 2
+	f(rowsCount, streamsCount, nodesCount)
+
+	rowsCount = 10000
+	streamsCount = 100
+	nodesCount = 2
+	f(rowsCount, streamsCount, nodesCount)
+
+	rowsCount = 100000
+	streamsCount = 1000
+	nodesCount = 9
+	f(rowsCount, streamsCount, nodesCount)
+}

app/vlstorage/netselect/netselect.go

@@ -0,0 +1,472 @@
+package netselect
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
+)
+
+const (
+	// FieldNamesProtocolVersion is the version of the protocol used for /internal/select/field_names HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	FieldNamesProtocolVersion = "v1"
+
+	// FieldValuesProtocolVersion is the version of the protocol used for /internal/select/field_values HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	FieldValuesProtocolVersion = "v1"
+
+	// StreamFieldNamesProtocolVersion is the version of the protocol used for /internal/select/stream_field_names HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	StreamFieldNamesProtocolVersion = "v1"
+
+	// StreamFieldValuesProtocolVersion is the version of the protocol used for /internal/select/stream_field_values HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	StreamFieldValuesProtocolVersion = "v1"
+
+	// StreamsProtocolVersion is the version of the protocol used for /internal/select/streams HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	StreamsProtocolVersion = "v1"
+
+	// StreamIDsProtocolVersion is the version of the protocol used for /internal/select/stream_ids HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	StreamIDsProtocolVersion = "v1"
+
+	// QueryProtocolVersion is the version of the protocol used for /internal/select/query HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	QueryProtocolVersion = "v1"
+)
+
+// Storage is a network storage for querying remote storage nodes in the cluster.
+type Storage struct {
+	sns []*storageNode
+
+	disableCompression bool
+}
+
+type storageNode struct {
+	// scheme is http or https scheme to communicate with addr
+	scheme string
+
+	// addr is TCP address of the storage node to query
+	addr string
+
+	// s is a storage, which holds the given storageNode
+	s *Storage
+
+	// c is an http client used for querying storage node at addr.
+	c *http.Client
+
+	// ac is auth config used for setting request headers such as Authorization and Host.
+	ac *promauth.Config
+}
+
+func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
+	tr := httputil.NewTransport(false, "vlselect_backend")
+	tr.TLSHandshakeTimeout = 20 * time.Second
+	tr.DisableCompression = true
+
+	scheme := "http"
+	if isTLS {
+		scheme = "https"
+	}
+
+	sn := &storageNode{
+		scheme: scheme,
+		addr:   addr,
+		s:      s,
+		c: &http.Client{
+			Transport: ac.NewRoundTripper(tr),
+		},
+		ac: ac,
+	}
+	return sn
+}
+
+func (sn *storageNode) runQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, processBlock func(db *logstorage.DataBlock)) error {
+	args := sn.getCommonArgs(QueryProtocolVersion, tenantIDs, q)
+
+	reqURL := sn.getRequestURL("/internal/select/query", args)
+	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
+	if err != nil {
+		logger.Panicf("BUG: unexpected error when creating a request: %s", err)
+	}
+	if err := sn.ac.SetHeaders(req, true); err != nil {
+		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
+	}
+
+	// send the request to the storage node
+	resp, err := sn.c.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		responseBody, err := io.ReadAll(resp.Body)
+		if err != nil {
+			responseBody = []byte(err.Error())
+		}
+		return fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
+	}
+
+	// read the response
+	var dataLenBuf [8]byte
+	var buf []byte
+	var db logstorage.DataBlock
+	var valuesBuf []string
+	for {
+		if _, err := io.ReadFull(resp.Body, dataLenBuf[:]); err != nil {
+			if errors.Is(err, io.EOF) {
+				// The end of response stream
+				return nil
+			}
+			return fmt.Errorf("cannot read block size from %q: %w", reqURL, err)
+		}
+		blockLen := encoding.UnmarshalUint64(dataLenBuf[:])
+		if blockLen > math.MaxInt {
+			return fmt.Errorf("too big data block: %d bytes; mustn't exceed %v bytes", blockLen, math.MaxInt)
+		}
+
+		buf = slicesutil.SetLength(buf, int(blockLen))
+		if _, err := io.ReadFull(resp.Body, buf); err != nil {
+			return fmt.Errorf("cannot read block with size of %d bytes from %q: %w", blockLen, reqURL, err)
+		}
+
+		src := buf
+		if !sn.s.disableCompression {
+			bufLen := len(buf)
+			var err error
+			buf, err = zstd.Decompress(buf, buf)
+			if err != nil {
+				return fmt.Errorf("cannot decompress data block: %w", err)
+			}
+			src = buf[bufLen:]
+		}
+
+		for len(src) > 0 {
+			tail, vb, err := db.UnmarshalInplace(src, valuesBuf[:0])
+			if err != nil {
+				return fmt.Errorf("cannot unmarshal data block received from %q: %w", reqURL, err)
+			}
+			valuesBuf = vb
+			src = tail
+
+			processBlock(&db)
+
+			clear(valuesBuf)
+		}
+	}
+}
+
+func (sn *storageNode) getFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(FieldNamesProtocolVersion, tenantIDs, q)
+
+	return sn.getValuesWithHits(ctx, "/internal/select/field_names", args)
+}
+
+func (sn *storageNode) getFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(FieldValuesProtocolVersion, tenantIDs, q)
+	args.Set("field", fieldName)
+	args.Set("limit", fmt.Sprintf("%d", limit))
+
+	return sn.getValuesWithHits(ctx, "/internal/select/field_values", args)
+}
+
+func (sn *storageNode) getStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(StreamFieldNamesProtocolVersion, tenantIDs, q)
+
+	return sn.getValuesWithHits(ctx, "/internal/select/stream_field_names", args)
+}
+
+func (sn *storageNode) getStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(StreamFieldValuesProtocolVersion, tenantIDs, q)
+	args.Set("field", fieldName)
+	args.Set("limit", fmt.Sprintf("%d", limit))
+
+	return sn.getValuesWithHits(ctx, "/internal/select/stream_field_values", args)
+}
+
+func (sn *storageNode) getStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(StreamsProtocolVersion, tenantIDs, q)
+	args.Set("limit", fmt.Sprintf("%d", limit))
+
+	return sn.getValuesWithHits(ctx, "/internal/select/streams", args)
+}
+
+func (sn *storageNode) getStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(StreamIDsProtocolVersion, tenantIDs, q)
+	args.Set("limit", fmt.Sprintf("%d", limit))
+
+	return sn.getValuesWithHits(ctx, "/internal/select/stream_ids", args)
+}
+
+func (sn *storageNode) getCommonArgs(version string, tenantIDs []logstorage.TenantID, q *logstorage.Query) url.Values {
+	args := url.Values{}
+	args.Set("version", version)
+	args.Set("tenant_ids", string(logstorage.MarshalTenantIDs(nil, tenantIDs)))
+	args.Set("query", q.String())
+	args.Set("timestamp", fmt.Sprintf("%d", q.GetTimestamp()))
+	args.Set("disable_compression", fmt.Sprintf("%v", sn.s.disableCompression))
+	return args
+}
+
+func (sn *storageNode) getValuesWithHits(ctx context.Context, path string, args url.Values) ([]logstorage.ValueWithHits, error) {
+	data, err := sn.executeRequestAt(ctx, path, args)
+	if err != nil {
+		return nil, err
+	}
+	return unmarshalValuesWithHits(data)
+}
+
+func (sn *storageNode) executeRequestAt(ctx context.Context, path string, args url.Values) ([]byte, error) {
+	reqURL := sn.getRequestURL(path, args)
+	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
+	if err != nil {
+		logger.Panicf("BUG: unexpected error when creating a request: %s", err)
+	}
+	if err := sn.ac.SetHeaders(req, true); err != nil {
+		return nil, fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
+	}
+
+	// send the request to the storage node
+	resp, err := sn.c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		responseBody, err := io.ReadAll(resp.Body)
+		if err != nil {
+			responseBody = []byte(err.Error())
+		}
+		return nil, fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
+	}
+
+	// read the response
+	var bb bytesutil.ByteBuffer
+	if _, err := bb.ReadFrom(resp.Body); err != nil {
+		return nil, fmt.Errorf("cannot read response from %q: %w", reqURL, err)
+	}
+
+	if sn.s.disableCompression {
+		return bb.B, nil
+	}
+
+	bbLen := len(bb.B)
+	bb.B, err = zstd.Decompress(bb.B, bb.B)
+	if err != nil {
+		return nil, err
+	}
+	return bb.B[bbLen:], nil
+}
+
+func (sn *storageNode) getRequestURL(path string, args url.Values) string {
+	return fmt.Sprintf("%s://%s%s?%s", sn.scheme, sn.addr, path, args.Encode())
+}
+
+// NewStorage returns new Storage for the given addrs and the given authCfgs.
+//
+// If disableCompression is set, then uncompressed responses are received from storage nodes.
+//
+// Call MustStop on the returned storage when it is no longer needed.
+func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, disableCompression bool) *Storage {
+	s := &Storage{
+		disableCompression: disableCompression,
+	}
+
+	sns := make([]*storageNode, len(addrs))
+	for i, addr := range addrs {
+		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
+	}
+	s.sns = sns
+
+	return s
+}
+
+// MustStop stops the s.
+func (s *Storage) MustStop() {
+	s.sns = nil
+}
+
+// RunQuery runs the given q and calls writeBlock for the returned data blocks
+func (s *Storage) RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
+	nqr, err := logstorage.NewNetQueryRunner(ctx, tenantIDs, q, s.RunQuery, writeBlock)
+	if err != nil {
+		return err
+	}
+
+	search := func(stopCh <-chan struct{}, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
+		return s.runQuery(stopCh, tenantIDs, q, writeBlock)
+	}
+
+	concurrency := q.GetConcurrency()
+	return nqr.Run(ctx, concurrency, search)
+}
+
+func (s *Storage) runQuery(stopCh <-chan struct{}, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
+	ctxWithCancel, cancel := contextutil.NewStopChanContext(stopCh)
+	defer cancel()
+
+	errs := make([]error, len(s.sns))
+
+	var wg sync.WaitGroup
+	for i := range s.sns {
+		wg.Add(1)
+		go func(nodeIdx int) {
+			defer wg.Done()
+			sn := s.sns[nodeIdx]
+			err := sn.runQuery(ctxWithCancel, tenantIDs, q, func(db *logstorage.DataBlock) {
+				writeBlock(uint(nodeIdx), db)
+			})
+			if err != nil {
+				// Cancel the remaining parallel queries
+				cancel()
+			}
+
+			errs[nodeIdx] = err
+		}(i)
+	}
+	wg.Wait()
+
+	return getFirstNonCancelError(errs)
+}
+
+// GetFieldNames executes q and returns field names seen in results.
+func (s *Storage) GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getFieldNames(ctx, tenantIDs, q)
+	})
+}
+
+// GetFieldValues executes q and returns unique values for the fieldName seen in results.
+//
+// If limit > 0, then up to limit unique values are returned.
+func (s *Storage) GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	})
+}
+
+// GetStreamFieldNames executes q and returns stream field names seen in results.
+func (s *Storage) GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getStreamFieldNames(ctx, tenantIDs, q)
+	})
+}
+
+// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
+//
+// If limit > 0, then up to limit unique stream field values are returned.
+func (s *Storage) GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	})
+}
+
+// GetStreams executes q and returns streams seen in query results.
+//
+// If limit > 0, then up to limit unique streams are returned.
+func (s *Storage) GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getStreams(ctx, tenantIDs, q, limit)
+	})
+}
+
+// GetStreamIDs executes q and returns streamIDs seen in query results.
+//
+// If limit > 0, then up to limit unique streamIDs are returned.
+func (s *Storage) GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getStreamIDs(ctx, tenantIDs, q, limit)
+	})
+}
+
+func (s *Storage) getValuesWithHits(ctx context.Context, limit uint64, resetHitsOnLimitExceeded bool,
+	callback func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error)) ([]logstorage.ValueWithHits, error) {
+
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	results := make([][]logstorage.ValueWithHits, len(s.sns))
+	errs := make([]error, len(s.sns))
+
+	var wg sync.WaitGroup
+	for i := range s.sns {
+		wg.Add(1)
+		go func(nodeIdx int) {
+			defer wg.Done()
+
+			sn := s.sns[nodeIdx]
+			vhs, err := callback(ctxWithCancel, sn)
+			results[nodeIdx] = vhs
+			errs[nodeIdx] = err
+
+			if err != nil {
+				// Cancel the remaining parallel requests
+				cancel()
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	if err := getFirstNonCancelError(errs); err != nil {
+		return nil, err
+	}
+
+	vhs := logstorage.MergeValuesWithHits(results, limit, resetHitsOnLimitExceeded)
+
+	return vhs, nil
+}
+
+func getFirstNonCancelError(errs []error) error {
+	for _, err := range errs {
+		if err != nil && !errors.Is(err, context.Canceled) {
+			return err
+		}
+	}
+	return nil
+}
+
+func unmarshalValuesWithHits(src []byte) ([]logstorage.ValueWithHits, error) {
+	var vhs []logstorage.ValueWithHits
+	for len(src) > 0 {
+		var vh logstorage.ValueWithHits
+		tail, err := vh.UnmarshalInplace(src)
+		if err != nil {
+			return nil, fmt.Errorf("cannot unmarshal ValueWithHits #%d: %w", len(vhs), err)
+		}
+		src = tail
+
+		// Clone vh.Value, since it points to src.
+		vh.Value = strings.Clone(vh.Value)
+
+		vhs = append(vhs, vh)
+	}
+
+	return vhs, nil
+}

app/vmagent/README.md

@@ -1,3 +1,3 @@
-See vmagent docs [here](https://docs.victoriametrics.com/vmagent/).
+See vmagent docs [here](https://docs.victoriametrics.com/victoriametrics/vmagent/).
 
-vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmagent.md).
+vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmagent.md).

app/vmagent/main.go

@@ -165,7 +165,9 @@ func main() {
 
 	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
+		UseProxyProtocol: useProxyProtocol,
+	})
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -242,7 +244,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent/'>https://docs.victoriametrics.com/vmagent/</a></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"targets", "status for discovered active targets"},
@@ -752,7 +754,7 @@ func usage() {
 	const s = `
 vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
 
-See the docs at https://docs.victoriametrics.com/vmagent/ .
+See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
 `
 	flagutil.Usage(s)
 }

app/vmagent/multiarch/Dockerfile

@@ -9,4 +9,5 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8429
 ENTRYPOINT ["/vmagent-prod"]
 ARG TARGETARCH
-COPY vmagent-linux-${TARGETARCH}-prod ./vmagent-prod
+ARG BINARY_SUFFIX=non-existing
+COPY vmagent-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmagent-prod

app/vmagent/remotewrite/client.go

@@ -10,26 +10,29 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
 )
 
 var (
 	forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
 	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
 
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
@@ -87,7 +90,8 @@ type client struct {
 	remoteWriteURL string
 
 	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
-	useVMProto bool
+	useVMProto          atomic.Bool
+	canDowngradeVMProto atomic.Bool
 
 	fq *persistentqueue.FastQueue
 	hc *http.Client
@@ -166,17 +170,11 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
 	}
 	if !useVMProto && !usePromProto {
-		// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
-		doRequest := func(url string) (*http.Response, error) {
-			return c.doRequest(url, nil)
-		}
-		useVMProto = protoparserutil.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
-		if !useVMProto {
-			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
-				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", sanitizedURL)
-		}
+		// The VM protocol could be downgraded later at runtime if unsupported media type response status is received.
+		useVMProto = true
+		c.canDowngradeVMProto.Store(true)
 	}
-	c.useVMProto = useVMProto
+	c.useVMProto.Store(useVMProto)
 
 	return c
 }
@@ -384,7 +382,7 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
-	if c.useVMProto {
+	if encoding.IsZstd(body) {
 		h.Set("Content-Encoding", "zstd")
 		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
 	} else {
@@ -420,7 +418,7 @@ again:
 		if retryDuration > maxRetryDuration {
 			retryDuration = maxRetryDuration
 		}
-		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+		remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
 		t := timerpool.Get(retryDuration)
 		select {
@@ -433,6 +431,7 @@ again:
 		c.retriesCount.Inc()
 		goto again
 	}
+
 	statusCode := resp.StatusCode
 	if statusCode/100 == 2 {
 		_ = resp.Body.Close()
@@ -441,24 +440,46 @@ again:
 		c.blocksSent.Inc()
 		return true
 	}
+
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
-	if statusCode == 409 || statusCode == 400 {
-		body, err := io.ReadAll(resp.Body)
-		_ = resp.Body.Close()
-		if err != nil {
-			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
-				"failed to read response body: %s",
-				len(block), c.sanitizedURL, statusCode, err)
-		} else {
-			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
-				len(block), c.sanitizedURL, statusCode, string(body))
-		}
-		// Just drop block on 409 and 400 status codes like Prometheus does.
+	if statusCode == 409 {
+		logBlockRejected(block, c.sanitizedURL, resp)
+
+		// Just drop block on 409 status code like Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
 		_ = resp.Body.Close()
 		c.packetsDropped.Inc()
 		return true
+		// - Remote Write v1 specification implicitly expects a `400 Bad Request` when the encoding is not supported.
+		// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
+		// - Real-world implementations of v1 use both 400 and 415 status codes.
+		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
+	} else if statusCode == 415 || statusCode == 400 {
+		if c.canDowngradeVMProto.Swap(false) {
+			logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
+				"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
+			c.useVMProto.Store(false)
+		}
+
+		if encoding.IsZstd(block) {
+			logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
+				"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
+
+			block = mustRepackBlockFromZstdToSnappy(block)
+
+			c.retriesCount.Inc()
+			_ = resp.Body.Close()
+			goto again
+		}
+
+		// Just drop snappy blocks on 400 or 415 status codes like Prometheus does.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
+		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
+		logBlockRejected(block, c.sanitizedURL, resp)
+		_ = resp.Body.Close()
+		c.packetsDropped.Inc()
+		return true
 	}
 
 	// Unexpected status code returned
@@ -488,6 +509,7 @@ again:
 }
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
 
 // getRetryDuration returns retry duration.
 // retryAfterDuration has the highest priority.
@@ -510,6 +532,28 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
 	return retryDuration
 }
 
+func mustRepackBlockFromZstdToSnappy(zstdBlock []byte) []byte {
+	plainBlock := make([]byte, 0, len(zstdBlock)*2)
+	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
+	if err != nil {
+		logger.Panicf("FATAL: cannot re-pack block with size %d bytes from Zstd to Snappy: %s", len(zstdBlock), err)
+	}
+
+	return snappy.Encode(nil, plainBlock)
+}
+
+func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+			"failed to read response body: %s",
+			len(block), sanitizedURL, resp.StatusCode, err)
+	} else {
+		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+			len(block), sanitizedURL, resp.StatusCode, string(body))
+	}
+}
+
 // parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
 // retryAfterString should be in either HTTP-date or a number of seconds.
 // It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.

app/vmagent/remotewrite/client_test.go

@@ -5,6 +5,9 @@ import (
 	"net/http"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/golang/snappy"
 )
 
 func TestCalculateRetryDuration(t *testing.T) {
@@ -97,3 +100,19 @@ func helper(d time.Duration) time.Duration {
 
 	return d + dv
 }
+
+func TestRepackBlockFromZstdToSnappy(t *testing.T) {
+	expectedPlainBlock := []byte(`foobar`)
+
+	zstdBlock := encoding.CompressZSTDLevel(nil, expectedPlainBlock, 1)
+	snappyBlock := mustRepackBlockFromZstdToSnappy(zstdBlock)
+
+	actualPlainBlock, err := snappy.Decode(nil, snappyBlock)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	if string(actualPlainBlock) != string(expectedPlainBlock) {
+		t.Fatalf("unexpected plain block; got %q; want %q", actualPlainBlock, expectedPlainBlock)
+	}
+}

app/vmagent/remotewrite/pendingseries.go

@@ -29,7 +29,7 @@ var (
 	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
 	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
 		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
-		"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
 )
 
 type pendingSeries struct {
@@ -40,7 +40,7 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Bool, significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
 	ps.wr.fq = fq
 	ps.wr.isVMRemoteWrite = isVMRemoteWrite
@@ -100,7 +100,7 @@ type writeRequest struct {
 	fq *persistentqueue.FastQueue
 
 	// Whether to encode the write request with VictoriaMetrics remote write protocol.
-	isVMRemoteWrite bool
+	isVMRemoteWrite *atomic.Bool
 
 	// How many significant figures must be left before sending the writeRequest to fq.
 	significantFigures int
@@ -138,7 +138,7 @@ func (wr *writeRequest) reset() {
 // This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
 func (wr *writeRequest) mustFlushOnStop() {
 	wr.wr.Timeseries = wr.tss
-	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite) {
+	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite.Load()) {
 		logger.Panicf("BUG: final flush must always return true")
 	}
 	wr.reset()
@@ -152,7 +152,7 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
 func (wr *writeRequest) tryFlush() bool {
 	wr.wr.Timeseries = wr.tss
 	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
-	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
+	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite.Load()) {
 		return false
 	}
 	wr.reset()

app/vmagent/remotewrite/relabel.go

@@ -7,10 +7,13 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -19,10 +22,10 @@ var (
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
 		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
 		"The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmagent/#relabeling")
+		"See https://docs.victoriametrics.com/victoriametrics/relabeling/")
 	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
 		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmagent/#relabeling")
+		"See https://docs.victoriametrics.com/victoriametrics/relabeling/")
 
 	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
 		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
@@ -31,12 +34,59 @@ var (
 
 var labelsGlobal []prompbmarshal.Label
 
+var (
+	relabelConfigReloads      *metrics.Counter
+	relabelConfigReloadErrors *metrics.Counter
+	relabelConfigSuccess      *metrics.Gauge
+	relabelConfigTimestamp    *metrics.Counter
+)
+
+func initRelabelMetrics() {
+	relabelConfigReloads = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
+	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
+	relabelConfigSuccess = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
+	relabelConfigTimestamp = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
+}
+
 // CheckRelabelConfigs checks -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig.
 func CheckRelabelConfigs() error {
 	_, err := loadRelabelConfigs()
 	return err
 }
 
+func initRelabelConfigs() {
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		logger.Fatalf("cannot initialize relabel configs: %s", err)
+	}
+	allRelabelConfigs.Store(rcs)
+	if rcs.isSet() {
+		initRelabelMetrics()
+		relabelConfigSuccess.Set(1)
+		relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
+	}
+}
+
+func reloadRelabelConfigs() {
+	rcs := allRelabelConfigs.Load()
+	if !rcs.isSet() {
+		return
+	}
+	relabelConfigReloads.Inc()
+	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		relabelConfigReloadErrors.Inc()
+		relabelConfigSuccess.Set(0)
+		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
+		return
+	}
+	allRelabelConfigs.Store(rcs)
+	relabelConfigSuccess.Set(1)
+	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
+	logger.Infof("successfully reloaded relabel configs")
+}
+
 func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
@@ -70,6 +120,21 @@ type relabelConfigs struct {
 	perURL []*promrelabel.ParsedConfigs
 }
 
+func (rcs *relabelConfigs) isSet() bool {
+	if rcs == nil {
+		return false
+	}
+	if rcs.global.Len() > 0 {
+		return true
+	}
+	for _, pc := range rcs.perURL {
+		if pc.Len() > 0 {
+			return true
+		}
+	}
+	return false
+}
+
 // initLabelsGlobal must be called after parsing command-line flags.
 func initLabelsGlobal() {
 	labelsGlobal = nil

app/vmagent/remotewrite/remotewrite.go

@@ -15,7 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -39,22 +39,21 @@ var (
 		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. "+
 		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set")
 	enableMultitenantHandlers = flag.Bool("enableMultitenantHandlers", false, "Whether to process incoming data via multitenant insert handlers according to "+
-		"https://docs.victoriametrics.com/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers "+
-		"according to https://docs.victoriametrics.com/#how-to-import-time-series-data ."+
-		"See https://docs.victoriametrics.com/vmagent/#multitenancy for details")
+		"https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers "+
+		"according to https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data ."+
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy for details")
 
-	shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . "+
-		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages . "+
+	shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url. "+
+		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages . "+
 		"See also -remoteWrite.shardByURLReplicas")
 	shardByURLReplicas = flag.Int("remoteWrite.shardByURLReplicas", 1, "How many copies of data to make among remote storage systems enumerated via -remoteWrite.url "+
-		"when -remoteWrite.shardByURL is set. See https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
+		"when -remoteWrite.shardByURL is set. See https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
 	shardByURLLabels = flagutil.NewArrayString("remoteWrite.shardByURL.labels", "Optional list of labels, which must be used for sharding outgoing samples "+
 		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
 		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.ignoreLabels")
 	shardByURLIgnoreLabels = flagutil.NewArrayString("remoteWrite.shardByURL.ignoreLabels", "Optional list of labels, which must be ignored when sharding outgoing samples "+
 		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
 		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.labels")
-
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
@@ -81,28 +80,30 @@ var (
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
 		`Enabled sorting for labels can slow down ingestion performance a bit`)
 	maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
-		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter")
+		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
 	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
-		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter")
+		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
 	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmagent can receive per second. Data ingestion is paused when the limit is exceeded. "+
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
 	disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
 		"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
-		"See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
 	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
+		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
 )
 
 var (
 	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
-	rwctxsGlobal []*remoteWriteCtx
+	rwctxsGlobal              []*remoteWriteCtx
+	rwctxsGlobalIdx           []int
+	rwctxConsistentHashGlobal *consistenthash.ConsistentHash
 
 	// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
 	ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
 		Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
 			"or remove -remoteWrite.disableOnDiskQueue from vmagent command-line flags, so it could save pending data to -remoteWrite.tmpDataPath; " +
-			"see https://docs.victoriametrics.com/vmagent/#disabling-on-disk-persistence"),
+			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence"),
 		StatusCode: http.StatusTooManyRequests,
 	}
 
@@ -183,7 +184,7 @@ func Init() {
 
 	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
-			"see https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
+			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
 	}
 	shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
 	shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
@@ -195,17 +196,11 @@ func Init() {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
 	sighupCh := procutil.NewSighupChan()
 
-	rcs, err := loadRelabelConfigs()
-	if err != nil {
-		logger.Fatalf("cannot load relabel configs: %s", err)
-	}
-	allRelabelConfigs.Store(rcs)
-	relabelConfigSuccess.Set(1)
-	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
+	initRelabelConfigs()
 
 	initStreamAggrConfigGlobal()
 
-	rwctxsGlobal = newRemoteWriteCtxs(*remoteWriteURLs)
+	initRemoteWriteCtxs(*remoteWriteURLs)
 
 	disableOnDiskQueues := []bool(*disableOnDiskQueue)
 	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
@@ -267,30 +262,7 @@ func dropDanglingQueues() {
 	}
 }
 
-func reloadRelabelConfigs() {
-	relabelConfigReloads.Inc()
-	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
-	rcs, err := loadRelabelConfigs()
-	if err != nil {
-		relabelConfigReloadErrors.Inc()
-		relabelConfigSuccess.Set(0)
-		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
-		return
-	}
-	allRelabelConfigs.Store(rcs)
-	relabelConfigSuccess.Set(1)
-	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
-	logger.Infof("successfully reloaded relabel configs")
-}
-
-var (
-	relabelConfigReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
-	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	relabelConfigSuccess      = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
-	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
-)
-
-func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
+func initRemoteWriteCtxs(urls []string) {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
@@ -306,6 +278,7 @@ func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
 		maxInmemoryBlocks = 2
 	}
 	rwctxs := make([]*remoteWriteCtx, len(urls))
+	rwctxIdx := make([]int, len(urls))
 	for i, remoteWriteURLRaw := range urls {
 		remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
 		if err != nil {
@@ -316,8 +289,19 @@ func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
 		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+		rwctxIdx[i] = i
 	}
-	return rwctxs
+
+	if *shardByURL {
+		consistentHashNodes := make([]string, 0, len(urls))
+		for i, url := range urls {
+			consistentHashNodes = append(consistentHashNodes, fmt.Sprintf("%d:%s", i+1, url))
+		}
+		rwctxConsistentHashGlobal = consistenthash.NewConsistentHash(consistentHashNodes, 0)
+	}
+
+	rwctxsGlobal = rwctxs
+	rwctxsGlobalIdx = rwctxIdx
 }
 
 var (
@@ -501,6 +485,10 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 	return true
 }
 
+// getEligibleRemoteWriteCtxs checks whether writes to configured remote storage systems are blocked and
+// returns only the unblocked rwctx.
+//
+// calculateHealthyRwctxIdx will rely on the order of rwctx to be in ascending order.
 func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
 	if !disableOnDiskQueueAny {
 		return rwctxsGlobal, true
@@ -517,6 +505,12 @@ func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamples
 				return nil, false
 			}
 			rowsCount := getRowsCount(tss)
+			if *shardByURL {
+				// Todo: When shardByURL is enabled, the following metrics won't be 100% accurate. Because vmagent don't know
+				// which rwctx should data be pushed to yet. Let's consider the hashing algorithm fair and will distribute
+				// data to all rwctxs evenly.
+				rowsCount = rowsCount / len(rwctxsGlobal)
+			}
 			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 		}
 	}
@@ -528,6 +522,7 @@ func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
 	if len(rwctxs) == 0 {
 		return
 	}
+
 	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
 		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
 	}
@@ -578,42 +573,7 @@ func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []pr
 	defer putTSSShards(x)
 
 	shards := x.shards
-	tmpLabels := promutil.GetLabels()
-	for _, ts := range tssBlock {
-		hashLabels := ts.Labels
-		if len(shardByURLLabelsMap) > 0 {
-			hashLabels = tmpLabels.Labels[:0]
-			for _, label := range ts.Labels {
-				if _, ok := shardByURLLabelsMap[label.Name]; ok {
-					hashLabels = append(hashLabels, label)
-				}
-			}
-			tmpLabels.Labels = hashLabels
-		} else if len(shardByURLIgnoreLabelsMap) > 0 {
-			hashLabels = tmpLabels.Labels[:0]
-			for _, label := range ts.Labels {
-				if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
-					hashLabels = append(hashLabels, label)
-				}
-			}
-			tmpLabels.Labels = hashLabels
-		}
-		h := getLabelsHash(hashLabels)
-		idx := h % uint64(len(shards))
-		i := 0
-		for {
-			shards[idx] = append(shards[idx], ts)
-			i++
-			if i >= replicas {
-				break
-			}
-			idx++
-			if idx >= uint64(len(shards)) {
-				idx = 0
-			}
-		}
-	}
-	promutil.PutLabels(tmpLabels)
+	shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
 
 	// Push sharded samples to remote storage systems in parallel in order to reduce
 	// the time needed for sending the data to multiple remote storage systems.
@@ -636,6 +596,86 @@ func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []pr
 	return !anyPushFailed.Load()
 }
 
+// calculateHealthyRwctxIdx returns the index of healthyRwctxs in rwctxsGlobal.
+// It relies on the order of rwctx in healthyRwctxs, which is appended by getEligibleRemoteWriteCtxs.
+func calculateHealthyRwctxIdx(healthyRwctxs []*remoteWriteCtx) ([]int, []int) {
+	// fast path: all rwctxs are healthy.
+	if len(healthyRwctxs) == len(rwctxsGlobal) {
+		return rwctxsGlobalIdx, nil
+	}
+
+	unhealthyIdx := make([]int, 0, len(rwctxsGlobal))
+	healthyIdx := make([]int, 0, len(rwctxsGlobal))
+
+	var i int
+	for j := range rwctxsGlobal {
+		if i < len(healthyRwctxs) && rwctxsGlobal[j].idx == healthyRwctxs[i].idx {
+			healthyIdx = append(healthyIdx, j)
+			i++
+		} else {
+			unhealthyIdx = append(unhealthyIdx, j)
+		}
+	}
+
+	return healthyIdx, unhealthyIdx
+}
+
+// shardAmountRemoteWriteCtx distribute time series to shards by consistent hashing.
+func shardAmountRemoteWriteCtx(tssBlock []prompbmarshal.TimeSeries, shards [][]prompbmarshal.TimeSeries, rwctxs []*remoteWriteCtx, replicas int) {
+	tmpLabels := promutil.GetLabels()
+	defer promutil.PutLabels(tmpLabels)
+
+	healthyIdx, unhealthyIdx := calculateHealthyRwctxIdx(rwctxs)
+
+	// shardsIdxMap is a map to find which the shard idx by rwctxs idx.
+	// rwctxConsistentHashGlobal will tell which the rwctxs idx a time series should be written to.
+	// And this time series should be appended to the shards by correct shard idx.
+	shardsIdxMap := make(map[int]int, len(healthyIdx))
+	for idx, rwctxsIdx := range healthyIdx {
+		shardsIdxMap[rwctxsIdx] = idx
+	}
+
+	for _, ts := range tssBlock {
+		hashLabels := ts.Labels
+		if len(shardByURLLabelsMap) > 0 {
+			hashLabels = tmpLabels.Labels[:0]
+			for _, label := range ts.Labels {
+				if _, ok := shardByURLLabelsMap[label.Name]; ok {
+					hashLabels = append(hashLabels, label)
+				}
+			}
+			tmpLabels.Labels = hashLabels
+		} else if len(shardByURLIgnoreLabelsMap) > 0 {
+			hashLabels = tmpLabels.Labels[:0]
+			for _, label := range ts.Labels {
+				if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
+					hashLabels = append(hashLabels, label)
+				}
+			}
+			tmpLabels.Labels = hashLabels
+		}
+		h := getLabelsHash(hashLabels)
+
+		// Get the rwctxIdx through consistent hashing and then map it to the index in shards.
+		// The rwctxIdx is not always equal to the shardIdx, for example, when some rwctx are not available.
+		rwctxIdx := rwctxConsistentHashGlobal.GetNodeIdx(h, unhealthyIdx)
+		shardIdx := shardsIdxMap[rwctxIdx]
+
+		replicated := 0
+		for {
+			shards[shardIdx] = append(shards[shardIdx], ts)
+			replicated++
+			if replicated >= replicas {
+				break
+			}
+			shardIdx++
+			if shardIdx >= len(shards) {
+				shardIdx = 0
+			}
+		}
+	}
+}
+
 type tssShards struct {
 	shards [][]prompbmarshal.TimeSeries
 }
@@ -807,7 +847,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 	pss := make([]*pendingSeries, pssLen)
 	for i := range pss {
-		pss[i] = newPendingSeries(fq, c.useVMProto, sf, rd)
+		pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
 	}
 
 	rwctx := &remoteWriteCtx{

app/vmagent/remotewrite/remotewrite_test.go

@@ -4,13 +4,17 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+	"strconv"
+	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
+
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -68,7 +72,9 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 		allRelabelConfigs.Store(rcs)
 
 		pss := make([]*pendingSeries, 1)
-		pss[0] = newPendingSeries(nil, true, 0, 100)
+		isVMProto := &atomic.Bool{}
+		isVMProto.Store(true)
+		pss[0] = newPendingSeries(nil, isVMProto, 0, 100)
 		rwctx := &remoteWriteCtx{
 			idx:                    0,
 			streamAggrKeepInput:    keepInput,
@@ -171,3 +177,173 @@ metric{env="dev"} 15
 metric{env="bar"} 25
 `)
 }
+
+func TestShardAmountRemoteWriteCtx(t *testing.T) {
+	// 1. distribute 100000 series to n nodes.
+	// 2. remove the last node from healthy list.
+	// 3. distribute the same 10000 series to (n-1) node again.
+	// 4. check active time series change rate:
+	// change rate must < (3/total nodes). e.g. +30% if 10 you have 10 nodes.
+
+	f := func(remoteWriteCount int, healthyIdx []int, replicas int) {
+		t.Helper()
+		defer func() {
+			rwctxsGlobal = nil
+			rwctxsGlobalIdx = nil
+			rwctxConsistentHashGlobal = nil
+		}()
+
+		rwctxsGlobal = make([]*remoteWriteCtx, remoteWriteCount)
+		rwctxsGlobalIdx = make([]int, remoteWriteCount)
+		rwctxs := make([]*remoteWriteCtx, 0, len(healthyIdx))
+
+		for i := range remoteWriteCount {
+			rwCtx := &remoteWriteCtx{
+				idx: i,
+			}
+			rwctxsGlobalIdx[i] = i
+
+			if i >= len(healthyIdx) {
+				rwctxsGlobal[i] = rwCtx
+				continue
+			}
+			hIdx := healthyIdx[i]
+			if hIdx != i {
+				rwctxs = append(rwctxs, &remoteWriteCtx{
+					idx: hIdx,
+				})
+			} else {
+				rwctxs = append(rwctxs, rwCtx)
+			}
+			rwctxsGlobal[i] = rwCtx
+		}
+
+		seriesCount := 100000
+		// build 1000000 series
+		tssBlock := make([]prompbmarshal.TimeSeries, 0, seriesCount)
+		for i := 0; i < seriesCount; i++ {
+			tssBlock = append(tssBlock, prompbmarshal.TimeSeries{
+				Labels: []prompbmarshal.Label{
+					{
+						Name:  "label",
+						Value: strconv.Itoa(i),
+					},
+				},
+				Samples: []prompbmarshal.Sample{
+					{
+						Timestamp: 0,
+						Value:     0,
+					},
+				},
+			})
+		}
+
+		// build consistent hash for x remote write context
+		// build active time series set
+		nodes := make([]string, 0, remoteWriteCount)
+		activeTimeSeriesByNodes := make([]map[string]struct{}, remoteWriteCount)
+		for i := 0; i < remoteWriteCount; i++ {
+			nodes = append(nodes, fmt.Sprintf("node%d", i))
+			activeTimeSeriesByNodes[i] = make(map[string]struct{})
+		}
+		rwctxConsistentHashGlobal = consistenthash.NewConsistentHash(nodes, 0)
+
+		// create shards
+		x := getTSSShards(len(rwctxs))
+		shards := x.shards
+
+		// execute
+		shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
+
+		for i, nodeIdx := range healthyIdx {
+			for _, ts := range shards[i] {
+				// add it to node[nodeIdx]'s active time series
+				activeTimeSeriesByNodes[nodeIdx][prompbmarshal.LabelsToString(ts.Labels)] = struct{}{}
+			}
+		}
+
+		totalActiveTimeSeries := 0
+		for _, activeTimeSeries := range activeTimeSeriesByNodes {
+			totalActiveTimeSeries += len(activeTimeSeries)
+		}
+		avgActiveTimeSeries1 := totalActiveTimeSeries / remoteWriteCount
+		putTSSShards(x)
+
+		// removed last node
+		rwctxs = rwctxs[:len(rwctxs)-1]
+		healthyIdx = healthyIdx[:len(healthyIdx)-1]
+
+		x = getTSSShards(len(rwctxs))
+		shards = x.shards
+
+		// execute
+		shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
+		for i, nodeIdx := range healthyIdx {
+			for _, ts := range shards[i] {
+				// add it to node[nodeIdx]'s active time series
+				activeTimeSeriesByNodes[nodeIdx][prompbmarshal.LabelsToString(ts.Labels)] = struct{}{}
+			}
+		}
+
+		totalActiveTimeSeries = 0
+		for _, activeTimeSeries := range activeTimeSeriesByNodes {
+			totalActiveTimeSeries += len(activeTimeSeries)
+		}
+		avgActiveTimeSeries2 := totalActiveTimeSeries / remoteWriteCount
+
+		changed := math.Abs(float64(avgActiveTimeSeries2-avgActiveTimeSeries1) / float64(avgActiveTimeSeries1))
+		threshold := 3 / float64(remoteWriteCount)
+
+		if changed >= threshold {
+			t.Fatalf("average active time series before: %d, after: %d, changed: %.2f. threshold: %.2f", avgActiveTimeSeries1, avgActiveTimeSeries2, changed, threshold)
+		}
+
+	}
+
+	f(5, []int{0, 1, 2, 3, 4}, 1)
+
+	f(5, []int{0, 1, 2, 3, 4}, 2)
+
+	f(10, []int{0, 1, 2, 3, 4, 5, 6, 7, 9}, 1)
+
+	f(10, []int{0, 1, 2, 3, 4, 5, 6, 7, 9}, 3)
+}
+
+func TestCalculateHealthyRwctxIdx(t *testing.T) {
+	f := func(total int, healthyIdx []int, unhealthyIdx []int) {
+		t.Helper()
+
+		healthyMap := make(map[int]bool)
+		for _, idx := range healthyIdx {
+			healthyMap[idx] = true
+		}
+		rwctxsGlobal = make([]*remoteWriteCtx, total)
+		rwctxsGlobalIdx = make([]int, total)
+		rwctxs := make([]*remoteWriteCtx, 0, len(healthyIdx))
+		for i := range rwctxsGlobal {
+			rwctx := &remoteWriteCtx{idx: i}
+			rwctxsGlobal[i] = rwctx
+			if healthyMap[i] {
+				rwctxs = append(rwctxs, rwctx)
+			}
+			rwctxsGlobalIdx[i] = i
+		}
+
+		gotHealthyIdx, gotUnhealthyIdx := calculateHealthyRwctxIdx(rwctxs)
+		if !reflect.DeepEqual(healthyIdx, gotHealthyIdx) {
+			t.Errorf("calculateHealthyRwctxIdx want healthyIdx = %v, got %v", healthyIdx, gotHealthyIdx)
+		}
+		if !reflect.DeepEqual(unhealthyIdx, gotUnhealthyIdx) {
+			t.Errorf("calculateHealthyRwctxIdx want unhealthyIdx = %v, got %v", unhealthyIdx, gotUnhealthyIdx)
+		}
+	}
+
+	f(5, []int{0, 1, 2, 3, 4}, nil)
+	f(5, []int{0, 1, 2, 4}, []int{3})
+	f(5, []int{2, 4}, []int{0, 1, 3})
+	f(5, []int{0, 2, 4}, []int{1, 3})
+	f(5, []int{}, []int{0, 1, 2, 3, 4})
+	f(5, []int{4}, []int{0, 1, 2, 3})
+	f(1, []int{0}, nil)
+	f(1, []int{}, []int{0})
+}

app/vmagent/remotewrite/streamaggr.go

@@ -16,55 +16,55 @@ import (
 var (
 	// Global config
 	streamAggrGlobalConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
 		"See also -streamAggr.keepInput, -streamAggr.dropInput and -streamAggr.dedupInterval")
 	streamAggrGlobalKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep all the input samples after the aggregation "+
 		"with -streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
+		"are written to remote storages write. See also -streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
 	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
+		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
 	streamAggrGlobalDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval on "+
 		"aggregator before optional aggregation with -streamAggr.config . "+
-		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
+		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
 	streamAggrGlobalIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the "+
 		"current aggregation interval for aggregator. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples")
 	streamAggrGlobalIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start for "+
 		"aggregator. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from "+
-		"clients pushing data into the vmagent. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
+		"clients pushing data into the vmagent. See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrGlobalDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples for aggregator "+
-		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels")
 	streamAggrGlobalEnableWindows = flag.Bool("streamAggr.enableWindows", false, "Enables aggregation within fixed windows for all global aggregators. "+
 		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows.")
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-windows.")
 
 	// Per URL config
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
 		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
 	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
 	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
 	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
 		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples")
 	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
 		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
 		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving buffered delayed data from clients pushing data into the vmagent. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
 		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
 		"Multiple labels per remoteWrite.url must be delimited by '^^': -remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels")
 	streamAggrEnableWindows = flagutil.NewArrayBool("remoteWrite.streamAggr.enableWindows", "Enables aggregation within fixed windows for all remote write's aggregators. "+
 		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows.")
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-windows.")
 )
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.

app/vmalert-tool/README.md

@@ -1,3 +1,3 @@
-See vmalert-tool docs [here](https://docs.victoriametrics.com/vmalert-tool.html).
+See vmalert-tool docs [here](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/).
 
-vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmalert-tool.md).
+vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmalert-tool.md).

app/vmalert-tool/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image=non-existing
+FROM $base_image
+
+EXPOSE 8880
+
+ENTRYPOINT ["/vmalert-tool-prod"]
+ARG src_binary=non-existing
+COPY $src_binary ./vmalert-tool-prod

app/vmalert-tool/multiarch/Dockerfile

@@ -9,4 +9,5 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8429
 ENTRYPOINT ["/vmalert-tool-prod"]
 ARG TARGETARCH
-COPY vmalert-tool-linux-${TARGETARCH}-prod ./vmalert-tool-prod
+ARG BINARY_SUFFIX=non-existing
+COPY vmalert-tool-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmalert-tool-prod

app/vmalert/Makefile

@@ -74,7 +74,7 @@ test-vmalert:
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
 	go test -v -race -cover ./app/vmalert/remotewrite
-	go test -v -race -cover ./app/vmalert/utils
+	go test -v -race -cover ./app/vmalert/vmalertutil
 
 run-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules2-good.rules \

app/vmalert/README.md

@@ -1,3 +1,3 @@
-See vmalert docs [here](https://docs.victoriametrics.com/vmalert/).
+See vmalert docs [here](https://docs.victoriametrics.com/victoriametrics/vmalert/).
 
-vmalert docs can be edited at [docs/vmalert.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmalert.md).
+vmalert docs can be edited at [docs/vmalert.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmalert.md).

app/vmalert/config/config.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"bytes"
-	"crypto/md5"
 	"flag"
 	"fmt"
 	"hash/fnv"
@@ -11,15 +10,16 @@ import (
 	"sort"
 	"strings"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/log"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
-	"gopkg.in/yaml.v2"
 )
 
 var (
-	defaultRuleType = flag.String("rule.defaultRuleType", "prometheus", `Default type for rule expressions, can be overridden via "type" parameter on the group level, see https://docs.victoriametrics.com/vmalert/#groups. Supported values: "graphite", "prometheus" and "vlogs".`)
+	defaultRuleType = flag.String("rule.defaultRuleType", "prometheus", `Default type for rule expressions, can be overridden via "type" parameter on the group level, see https://docs.victoriametrics.com/victoriametrics/vmalert/#groups. Supported values: "graphite", "prometheus" and "vlogs".`)
 )
 
 // Group contains list of Rules grouped into
@@ -50,6 +50,8 @@ type Group struct {
 	NotifierHeaders []Header `yaml:"notifier_headers,omitempty"`
 	// EvalAlignment will make the timestamp of group query requests be aligned with interval
 	EvalAlignment *bool `yaml:"eval_alignment,omitempty"`
+	// Debug enables debug logs for the group
+	Debug bool `yaml:"debug,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]any `yaml:",inline"`
 }
@@ -67,7 +69,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
 	if g.Type.Get() == "" {
 		g.Type = NewRawType(*defaultRuleType)
 	}
-	h := md5.New()
+	h := fnv.New64a()
 	h.Write(b)
 	g.Checksum = fmt.Sprintf("%x", h.Sum(nil))
 	return nil
@@ -143,7 +145,7 @@ type Rule struct {
 	KeepFiringFor *promutil.Duration `yaml:"keep_firing_for,omitempty"`
 	Labels        map[string]string  `yaml:"labels,omitempty"`
 	Annotations   map[string]string  `yaml:"annotations,omitempty"`
-	Debug         bool               `yaml:"debug,omitempty"`
+	Debug         *bool              `yaml:"debug,omitempty"`
 	// UpdateEntriesLimit defines max number of rule's state updates stored in memory.
 	// Overrides `-rule.updateEntriesLimit`.
 	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`

app/vmalert/main.go

@@ -44,7 +44,7 @@ Enterprise version of vmalert supports S3 and GCS paths to rules.
 For example: gs://bucket/path/to/rules, s3://bucket/path/to/rules
 S3 and GCS paths support only matching by prefix, e.g. s3://bucket/dir/rule_ matches
 all files with prefix rule_ in folder dir.
-See https://docs.victoriametrics.com/vmalert/#reading-rules-from-object-storage
+See https://docs.victoriametrics.com/victoriametrics/vmalert/#reading-rules-from-object-storage
 `)
 
 	ruleTemplatesPath = flagutil.NewArrayString("rule.templates", `Path or glob pattern to location with go template definitions `+
@@ -71,7 +71,7 @@ absolute path to all .tpl files in root.
 	externalURL         = flag.String("external.url", "", "External URL is used as alert's source for sent alerts to the notifier. By default, hostname is used as address.")
 	externalAlertSource = flag.String("external.alert.source", "", `External Alert Source allows to override the Source link for alerts sent to AlertManager `+
 		`for cases where you want to build a custom link to Grafana, Prometheus or any other service. `+
-		`Supports templating - see https://docs.victoriametrics.com/vmalert/#templating . `+
+		`Supports templating - see https://docs.victoriametrics.com/victoriametrics/vmalert/#templating . `+
 		`For example, link to Grafana: -external.alert.source='explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{.Expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"now-1h","to":"now"}}'. `+
 		`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
 		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
@@ -182,7 +182,9 @@ func main() {
 		listenAddrs = []string{":8880"}
 	}
 	rh := &requestHandler{m: manager}
-	go httpserver.Serve(listenAddrs, useProxyProtocol, rh.handler)
+	go httpserver.Serve(listenAddrs, rh.handler, httpserver.ServeOptions{
+		UseProxyProtocol: useProxyProtocol,
+	})
 
 	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
@@ -318,7 +320,7 @@ func usage() {
 	const s = `
 vmalert processes alerts and recording rules.
 
-See the docs at https://docs.victoriametrics.com/vmalert/ .
+See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
 `
 	flagutil.Usage(s)
 }

app/vmalert/multiarch/Dockerfile

@@ -9,4 +9,5 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8880
 ENTRYPOINT ["/vmalert-prod"]
 ARG TARGETARCH
-COPY vmalert-linux-${TARGETARCH}-prod ./vmalert-prod
+ARG BINARY_SUFFIX=non-existing
+COPY vmalert-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmalert-prod

app/vmalert/notifier/alert.go

@@ -21,6 +21,8 @@ type Alert struct {
 	GroupID uint64
 	// Name represents Alert name
 	Name string
+	// Type defines the datasource type of the Alert
+	Type string
 	// Labels is the list of label-value pairs attached to the Alert
 	Labels map[string]string
 	// Annotations is the list of annotations generated on Alert evaluation
@@ -78,6 +80,7 @@ func (as AlertState) String() string {
 
 // AlertTplData is used to execute templating
 type AlertTplData struct {
+	Type     string
 	Labels   map[string]string
 	Value    float64
 	Expr     string
@@ -89,6 +92,7 @@ type AlertTplData struct {
 
 var tplHeaders = []string{
 	"{{ $value := .Value }}",
+	"{{ $type := .Type }}",
 	"{{ $labels := .Labels }}",
 	"{{ $expr := .Expr }}",
 	"{{ $externalLabels := .ExternalLabels }}",
@@ -106,6 +110,7 @@ var tplHeaders = []string{
 func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) {
 	tplData := AlertTplData{
 		Value:    a.Value,
+		Type:     a.Type,
 		Labels:   labels,
 		Expr:     a.Expr,
 		AlertID:  a.ID,

app/vmalert/notifier/alert_test.go

@@ -61,7 +61,7 @@ func TestAlertExecTemplate(t *testing.T) {
 		for k := range tplExpected {
 			got, exp := tpl[k], tplExpected[k]
 			if got != exp {
-				t.Fatalf("unexpected template for key=%q; got %q; want %q", k, got, exp)
+				t.Fatalf("unexpected template for key=%q; \ngot %q; \nwant %q", k, got, exp)
 			}
 		}
 	}
@@ -200,6 +200,16 @@ func TestAlertExecTemplate(t *testing.T) {
 	}, map[string]string{
 		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
 	})
+
+	// Datasource type
+	f(&Alert{
+		Type: "vlogs",
+		Expr: "up",
+	}, map[string]string{
+		"grafana_url": `vm-grafana.com/explore?left={"datasource":"{{ if eq .Type "vlogs" }}VictoriaLogs{{ else }}VictoriaMetrics{{ end }}","queries":[{"expr":"{{ .Expr }}"}]}`,
+	}, map[string]string{
+		"grafana_url": `vm-grafana.com/explore?left={"datasource":"VictoriaLogs","queries":[{"expr":"up"}]}`,
+	})
 }
 
 func TestAlert_toPromLabels(t *testing.T) {

app/vmalert/notifier/config.go

@@ -1,8 +1,8 @@
 package notifier
 
 import (
-	"crypto/md5"
 	"fmt"
+	"hash/fnv"
 	"net/url"
 	"os"
 	"path"
@@ -99,7 +99,7 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(any) error) error {
 	if err != nil {
 		return fmt.Errorf("failed to marshal configuration for checksum: %w", err)
 	}
-	h := md5.New()
+	h := fnv.New64a()
 	h.Write(b)
 	cfg.Checksum = fmt.Sprintf("%x", h.Sum(nil))
 	return nil

app/vmalert/remotewrite/client.go

@@ -146,8 +146,13 @@ func (c *Client) Close() error {
 		return fmt.Errorf("client is already closed")
 	}
 	close(c.input)
+
+	start := time.Now()
+	logger.Infof("shutting down remote write client: flushing remained series")
 	close(c.doneCh)
 	c.wg.Wait()
+	logger.Infof("shutting down remote write client: finished in %v", time.Since(start))
+
 	return nil
 }
 
@@ -156,21 +161,16 @@ func (c *Client) run(ctx context.Context) {
 	wr := &prompbmarshal.WriteRequest{}
 	shutdown := func() {
 		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
-		logger.Infof("shutting down remote write client and flushing remained series")
 
-		shutdownFlushCnt := 0
 		for ts := range c.input {
 			wr.Timeseries = append(wr.Timeseries, ts)
 			if len(wr.Timeseries) >= c.maxBatchSize {
-				shutdownFlushCnt += len(wr.Timeseries)
 				c.flush(lastCtx, wr)
 			}
 		}
 		// flush the last batch. `flush` will re-check and avoid flushing empty batch.
-		shutdownFlushCnt += len(wr.Timeseries)
 		c.flush(lastCtx, wr)
 
-		logger.Infof("shutting down remote write client flushed %d series", shutdownFlushCnt)
 		cancel()
 	}
 	c.wg.Add(1)

app/vmalert/replay.go

@@ -24,7 +24,7 @@ var (
 			"and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule."+
 			"Keep it equal or bigger than -remoteWrite.flushInterval.")
 	replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
-		"Max number of data points expected in one request. It affects the max time range for every `/query_range` request during the replay. The higher the value, the less requests will be made during replay.")
+		"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
 		"Defines how many retries to make before giving up on rule if request for it returns an error.")
 	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+

app/vmalert/rule/alerting.go

@@ -124,6 +124,10 @@ func (arm *alertingRuleMetrics) close() {
 
 // NewAlertingRule creates a new AlertingRule
 func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
+	debug := group.Debug
+	if cfg.Debug != nil {
+		debug = *cfg.Debug
+	}
 	ar := &AlertingRule{
 		Type:          group.Type,
 		RuleID:        cfg.ID,
@@ -137,14 +141,14 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		GroupName:     group.Name,
 		File:          group.File,
 		EvalInterval:  group.Interval,
-		Debug:         cfg.Debug,
+		Debug:         debug,
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:            group.Type.String(),
 			ApplyIntervalAsTimeFilter: setIntervalAsTimeFilter(group.Type.String(), cfg.Expr),
 			EvaluationInterval:        group.Interval,
 			QueryParams:               group.Params,
 			Headers:                   group.Headers,
-			Debug:                     cfg.Debug,
+			Debug:                     debug,
 		}),
 		alerts: make(map[uint64]*notifier.Alert),
 	}
@@ -537,6 +541,7 @@ func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.Query
 
 	tplData := notifier.AlertTplData{
 		Value:    m.Values[0],
+		Type:     ar.Type.String(),
 		Labels:   ls.origin,
 		Expr:     ar.Expr,
 		AlertID:  hash(ls.processed),
@@ -597,6 +602,7 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, a
 	return &notifier.Alert{
 		GroupID:     ar.GroupID,
 		Name:        ar.Name,
+		Type:        ar.Type.String(),
 		Expr:        ar.Expr,
 		For:         ar.For,
 		ActiveAt:    start,

app/vmalert/rule/alerting_test.go

@@ -653,6 +653,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		hash(map[string]string{"alertname": "for-pending"}): {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-pending",
+			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-pending"},
 			Annotations: map[string]string{"activeAt": "5000"},
 			State:       notifier.StatePending,
@@ -672,6 +673,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		hash(map[string]string{"alertname": "for-firing"}): {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-firing",
+			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-firing"},
 			Annotations: map[string]string{"activeAt": "1000"},
 			State:       notifier.StateFiring,
@@ -692,6 +694,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		hash(map[string]string{"alertname": "for-hold-pending"}): {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "for-hold-pending",
+			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
 			Annotations: map[string]string{"activeAt": "5000"},
 			State:       notifier.StatePending,
@@ -747,6 +750,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		hash(map[string]string{"alertname": "multi-series"}): {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
+			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "multi-series"},
 			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
@@ -758,6 +762,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		hash(map[string]string{"alertname": "multi-series", "foo": "bar"}): {
 			GroupID:     fakeGroup.GetID(),
 			Name:        "multi-series",
+			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "multi-series", "foo": "bar"},
 			Annotations: map[string]string{},
 			State:       notifier.StatePending,

app/vmalert/rule/group.go

@@ -30,8 +30,8 @@ var (
 	resendDelay        = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier.")
 	maxResolveDuration = flag.Duration("rule.maxResolveDuration", 0, "Limits the maxiMum duration for automatic alert expiration, "+
 		"which by default is 4 times evaluationInterval of the parent group")
-	evalDelay = flag.Duration("rule.evalDelay", 30*time.Second, "Adjustment of the `time` parameter for rule evaluation requests to compensate intentional data delay from the datasource. "+
-		"Normally, should be equal to `-search.latencyOffset` (cmd-line flag configured for VictoriaMetrics single-node or vmselect). "+
+	evalDelay = flag.Duration("rule.evalDelay", 30*time.Second, "Adjustment of the 'time' parameter for rule evaluation requests to compensate intentional data delay from the datasource. "+
+		"Normally, should be equal to '-search.latencyOffset' (cmd-line flag configured for VictoriaMetrics single-node or vmselect). "+
 		"This doesn't apply to groups with eval_offset specified.")
 	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
 	remoteReadLookBack     = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries. "+
@@ -57,6 +57,7 @@ type Group struct {
 	// checksum stores the hash of yaml definition for this group.
 	checksum       string
 	LastEvaluation time.Time
+	Debug          bool
 
 	Labels          map[string]string
 	Params          url.Values
@@ -118,6 +119,7 @@ func NewGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		Headers:         make(map[string]string),
 		NotifierHeaders: make(map[string]string),
 		Labels:          cfg.Labels,
+		Debug:           cfg.Debug,
 		evalAlignment:   cfg.EvalAlignment,
 
 		doneCh:     make(chan struct{}),
@@ -271,6 +273,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 	g.Limit = newGroup.Limit
 	g.checksum = newGroup.checksum
 	g.Rules = newRules
+	g.Debug = newGroup.Debug
 	return nil
 }
 
@@ -471,6 +474,7 @@ func (g *Group) DeepCopy() *Group {
 	newG := Group{}
 	_ = json.Unmarshal(data, &newG)
 	newG.Rules = g.Rules
+	newG.id = g.id
 	return &newG
 }
 

app/vmalert/rule/group_test.go

@@ -36,144 +36,206 @@ func TestMain(m *testing.M) {
 }
 
 func TestUpdateWith(t *testing.T) {
-	f := func(currentRules, newRules []config.Rule) {
+	f := func(oldG, newG config.Group) {
 		t.Helper()
 
 		ns := metrics.NewSet()
-		g := &Group{
-			Name:    "test",
-			metrics: &groupMetrics{set: ns},
-		}
 		qb := &datasource.FakeQuerier{}
-		for _, r := range currentRules {
-			r.ID = config.HashRule(r)
-			g.Rules = append(g.Rules, g.newRule(qb, r))
+		for i := range oldG.Rules {
+			oldG.Rules[i].ID = config.HashRule(oldG.Rules[i])
+		}
+		for i := range newG.Rules {
+			newG.Rules[i].ID = config.HashRule(newG.Rules[i])
 		}
 
-		ng := &Group{
-			Name: "test",
-		}
-		for _, r := range newRules {
-			r.ID = config.HashRule(r)
-			ng.Rules = append(ng.Rules, ng.newRule(qb, r))
-		}
+		g := NewGroup(oldG, qb, 0, nil)
+		g.metrics = &groupMetrics{set: ns}
+		expect := NewGroup(newG, qb, 0, nil)
 
-		err := g.updateWith(ng)
+		err := g.updateWith(expect)
 		if err != nil {
 			t.Fatalf("cannot update rule: %s", err)
 		}
 
-		if len(g.Rules) != len(newRules) {
-			t.Fatalf("expected to have %d rules; got: %d", len(g.Rules), len(newRules))
+		if len(g.Rules) != len(expect.Rules) {
+			t.Fatalf("expected to have %d rules; got: %d", len(expect.Rules), len(g.Rules))
 		}
 		sort.Slice(g.Rules, func(i, j int) bool {
 			return g.Rules[i].ID() < g.Rules[j].ID()
 		})
-		sort.Slice(ng.Rules, func(i, j int) bool {
-			return ng.Rules[i].ID() < ng.Rules[j].ID()
+		sort.Slice(expect.Rules, func(i, j int) bool {
+			return expect.Rules[i].ID() < expect.Rules[j].ID()
 		})
 		for i, r := range g.Rules {
-			got, want := r, ng.Rules[i]
+			got, want := r, expect.Rules[i]
 			if got.ID() != want.ID() {
 				t.Fatalf("expected to have rule %q; got %q", want, got)
 			}
 			if err := CompareRules(t, got, want); err != nil {
-				t.Fatalf("comparison error: %s", err)
+				t.Fatalf("comparison1 error: %s", err)
 			}
 		}
+		if g.Debug != expect.Debug {
+			t.Fatalf("expected to have debug %v; got %v", expect.Debug, g.Debug)
+		}
 	}
 
 	// new rule
-	f(nil, []config.Rule{
-		{Alert: "bar"},
-	})
+	f(config.Group{}, config.Group{
+		Rules: []config.Rule{
+			{Alert: "bar"},
+		}})
 
 	// update alerting rule
-	f([]config.Rule{
-		{
-			Alert: "foo",
-			Expr:  "up > 0",
-			For:   promutil.NewDuration(time.Second),
+	f(config.Group{
+		Rules: []config.Rule{
+			{
+				Alert: "foo",
+				Expr:  "up > 0",
+				For:   promutil.NewDuration(time.Second),
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+				Annotations: map[string]string{
+					"summary":     "{{ $value|humanize }}",
+					"description": "{{$labels}}",
+				},
+			},
+			{
+				Alert: "bar",
+				Expr:  "up > 0",
+				For:   promutil.NewDuration(time.Second),
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+			},
+		}}, config.Group{
+		Rules: []config.Rule{
+			{
+				Alert: "foo",
+				Expr:  "up > 10",
+				For:   promutil.NewDuration(time.Second),
+				Labels: map[string]string{
+					"baz": "bar",
+				},
+				Annotations: map[string]string{
+					"summary": "none",
+				},
+			},
+			{
+				Alert:         "bar",
+				Expr:          "up > 0",
+				For:           promutil.NewDuration(2 * time.Second),
+				KeepFiringFor: promutil.NewDuration(time.Minute),
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+			},
+		}})
+
+	// update recording rule
+	debug := true
+	f(config.Group{
+		Rules: []config.Rule{{
+			Record: "foo",
+			Expr:   "max(up)",
 			Labels: map[string]string{
 				"bar": "baz",
 			},
-			Annotations: map[string]string{
-				"summary":     "{{ $value|humanize }}",
-				"description": "{{$labels}}",
-			},
-		},
-		{
-			Alert: "bar",
-			Expr:  "up > 0",
-			For:   promutil.NewDuration(time.Second),
-			Labels: map[string]string{
-				"bar": "baz",
-			},
-		},
-	}, []config.Rule{
-		{
-			Alert: "foo",
-			Expr:  "up > 10",
-			For:   promutil.NewDuration(time.Second),
+		}}}, config.Group{
+		Rules: []config.Rule{{
+			Record: "foo",
+			Expr:   "min(up)",
+			Debug:  &debug,
 			Labels: map[string]string{
 				"baz": "bar",
 			},
-			Annotations: map[string]string{
-				"summary": "none",
-			},
-		},
-		{
-			Alert:         "bar",
-			Expr:          "up > 0",
-			For:           promutil.NewDuration(2 * time.Second),
-			KeepFiringFor: promutil.NewDuration(time.Minute),
-			Labels: map[string]string{
-				"bar": "baz",
-			},
-		},
-	})
+		}}})
 
-	// update recording rule
-	f([]config.Rule{{
-		Record: "foo",
-		Expr:   "max(up)",
-		Labels: map[string]string{
-			"bar": "baz",
-		},
-	}}, []config.Rule{{
-		Record: "foo",
-		Expr:   "min(up)",
-		Debug:  true,
-		Labels: map[string]string{
-			"baz": "bar",
-		},
-	}})
+	// update debug
+	f(config.Group{
+		Rules: []config.Rule{
+			{
+				Record: "foo",
+				Expr:   "max(up)",
+			},
+			{
+				Alert: "foo",
+				Expr:  "up > 0",
+				Debug: &debug,
+				For:   promutil.NewDuration(time.Second),
+			},
+		}}, config.Group{
+		Rules: []config.Rule{
+			{
+				Record: "foo",
+				Expr:   "max(up)",
+				Debug:  &debug,
+			},
+			{
+				Alert: "foo",
+				Expr:  "up > 0",
+				For:   promutil.NewDuration(time.Second),
+			},
+		}})
 
 	// empty rule
-	f([]config.Rule{{Alert: "foo"}, {Record: "bar"}}, nil)
+	f(config.Group{
+		Rules: []config.Rule{{Alert: "foo"}, {Record: "bar"}}}, config.Group{})
 
 	// multiple rules
-	f([]config.Rule{
-		{Alert: "bar"},
-		{Alert: "baz"},
-		{Alert: "foo"},
-	}, []config.Rule{
-		{Alert: "baz"},
-		{Record: "foo"},
-	})
+	f(config.Group{
+		Rules: []config.Rule{
+			{Alert: "bar"},
+			{Alert: "baz"},
+			{Alert: "foo"},
+		}}, config.Group{
+		Rules: []config.Rule{
+			{Alert: "baz"},
+			{Record: "foo"},
+		}})
 
 	// replace rule
-	f([]config.Rule{{Alert: "foo1"}}, []config.Rule{{Alert: "foo2"}})
+	f(config.Group{
+		Rules: []config.Rule{{Alert: "foo1"}}}, config.Group{
+		Rules: []config.Rule{{Alert: "foo2"}}})
 
 	// replace multiple rules
-	f([]config.Rule{
-		{Alert: "foo1"},
-		{Record: "foo2"},
-		{Alert: "foo3"},
-	}, []config.Rule{
-		{Alert: "foo3"},
-		{Alert: "foo4"},
-		{Record: "foo5"},
+	f(config.Group{
+		Rules: []config.Rule{
+			{Alert: "foo1"},
+			{Record: "foo2"},
+			{Alert: "foo3"},
+		}}, config.Group{
+		Rules: []config.Rule{
+			{Alert: "foo3"},
+			{Alert: "foo4"},
+			{Record: "foo5"},
+		}})
+
+	f(config.Group{Debug: false}, config.Group{Debug: true})
+	f(config.Group{
+		Debug: false,
+		Rules: []config.Rule{
+			{Alert: "foo1"},
+		},
+	}, config.Group{
+		Debug: true,
+		Rules: []config.Rule{
+			{Alert: "foo1"},
+		},
+	})
+
+	f(config.Group{
+		Debug: false,
+		Rules: []config.Rule{
+			{Alert: "foo1"},
+		},
+	}, config.Group{
+		Debug: false,
+		Rules: []config.Rule{
+			{Alert: "foo1", Debug: &debug},
+		},
 	})
 }
 
@@ -276,6 +338,7 @@ func TestGroupStart(t *testing.T) {
           summary: "{{ $value }}"
 `
 	)
+
 	var groups []config.Group
 	err := yaml.Unmarshal([]byte(rules), &groups)
 	if err != nil {
@@ -476,6 +539,7 @@ func TestCloseWithEvalInterruption(t *testing.T) {
           summary: "{{ $value }}"
 `
 	)
+
 	var groups []config.Group
 	err := yaml.Unmarshal([]byte(rules), &groups)
 	if err != nil {

app/vmalert/rule/recording.go

@@ -83,6 +83,10 @@ func (rr *RecordingRule) ID() uint64 {
 
 // NewRecordingRule creates a new RecordingRule
 func NewRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *RecordingRule {
+	debug := group.Debug
+	if cfg.Debug != nil {
+		debug = *cfg.Debug
+	}
 	rr := &RecordingRule{
 		Type:      group.Type,
 		RuleID:    cfg.ID,
@@ -92,14 +96,14 @@ func NewRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		GroupID:   group.GetID(),
 		GroupName: group.Name,
 		File:      group.File,
-		Debug:     cfg.Debug,
+		Debug:     debug,
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:            group.Type.String(),
 			ApplyIntervalAsTimeFilter: setIntervalAsTimeFilter(group.Type.String(), cfg.Expr),
 			EvaluationInterval:        group.Interval,
 			QueryParams:               group.Params,
 			Headers:                   group.Headers,
-			Debug:                     cfg.Debug,
+			Debug:                     debug,
 		}),
 	}
 
@@ -254,12 +258,18 @@ func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompbmarshal.TimeSer
 			Value: rr.Name,
 		})
 	}
+	// add extra labels configured by user
 	for k := range rr.Labels {
-		prevLabel := promrelabel.GetLabelByName(m.Labels, k)
-		if prevLabel != nil && prevLabel.Value != rr.Labels[k] {
-			// Rename the prevLabel to "exported_" + label.Name
-			prevLabel.Name = fmt.Sprintf("exported_%s", prevLabel.Name)
+		existingLabel := promrelabel.GetLabelByName(m.Labels, k)
+		if existingLabel != nil { // there is a conflict between extra and existing label
+			if existingLabel.Value == rr.Labels[k] {
+				// extra and existing labels are identical - do nothing
+				continue
+			}
+			// preserve existing label by adding "exported_" prefix
+			existingLabel.Name = fmt.Sprintf("exported_%s", existingLabel.Name)
 		}
+		// add extra label
 		m.Labels = append(m.Labels, prompbmarshal.Label{
 			Name:  k,
 			Value: rr.Labels[k],
@@ -278,6 +288,7 @@ func (rr *RecordingRule) updateWith(r Rule) error {
 	rr.Expr = nr.Expr
 	rr.Labels = nr.Labels
 	rr.q = nr.q
+	rr.Debug = nr.Debug
 	return nil
 }
 

app/vmalert/rule/recording_test.go

@@ -168,6 +168,7 @@ func TestRecordingRule_Exec(t *testing.T) {
 	}, [][]datasource.Metric{{
 		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
+		metricWithValueAndLabels(t, 1, "__name__", "baz", "job", "baz", "source", "test"),
 	}}, [][]prompbmarshal.TimeSeries{{
 		newTimeSeries([]float64{2}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
 			{
@@ -202,6 +203,21 @@ func TestRecordingRule_Exec(t *testing.T) {
 					Value: "origin",
 				},
 			}),
+		newTimeSeries([]float64{1}, []int64{ts.UnixNano()},
+			[]prompbmarshal.Label{
+				{
+					Name:  "__name__",
+					Value: "job:foo",
+				},
+				{
+					Name:  "job",
+					Value: "baz",
+				},
+				{
+					Name:  "source",
+					Value: "test",
+				},
+			}),
 	}})
 }
 

app/vmalert/rule/rule.go

@@ -35,7 +35,7 @@ type Rule interface {
 	registerMetrics(set *metrics.Set)
 }
 
-var errDuplicate = errors.New("result contains metrics with the same labelset during evaluation. See https://docs.victoriametrics.com/vmalert/#series-with-the-same-labelset for details")
+var errDuplicate = errors.New("result contains metrics with the same labelset during evaluation. See https://docs.victoriametrics.com/victoriametrics/vmalert/#series-with-the-same-labelset for details")
 
 type ruleState struct {
 	sync.RWMutex

app/vmalert/static/css/custom.css

@@ -0,0 +1,101 @@
+html {
+  scroll-padding-top: 60px;
+}
+
+.nav-link svg {
+  fill: var(--bs-navbar-color);
+}
+
+.nav-link.active svg {
+  fill: var(--bs-navbar-active-color);
+}
+
+.nav-link:not(.active):hover svg {
+  fill: var(--bs-nav-link-hover-color);
+}
+
+.dropdown-menu {
+  --bs-dropdown-link-active-color: #000;
+  --bs-dropdown-link-active-bg: #e9e9e9;
+  & .dropdown-item.active {
+    font-weight: 700;
+  }
+}
+
+.dropdown-menu {
+  min-width: 0;
+}
+
+#search {
+  max-width: 400px;
+}
+
+body {
+  padding-top: 4.5rem;
+}
+
+.group-items {
+  cursor: pointer;
+  padding: 5px;
+  margin-top: 5px;
+  position: relative;
+}
+
+.btn svg, .dropdown-item svg {
+  fill: #000;
+  stroke: #000;
+}
+
+.btn:hover svg, .btn.show svg {
+  fill: #fff;
+  stroke: #fff;
+}
+
+.btn:has(svg) {
+  height: 38px;
+}
+
+.group-items:not(:has(.sub-item:not(.d-none))) {
+  display: none !important;
+}
+
+.group-items:hover {
+  background-color: #f8f9fa!important;
+}
+
+.table {
+  table-layout: fixed;
+}
+
+.table .error-cell {
+  word-break: break-word;
+  font-size: 14px;
+}
+
+pre {
+  overflow: scroll;
+  min-height: 30px;
+  max-width: 100%;
+}
+
+pre::-webkit-scrollbar {
+  -webkit-appearance: none;
+  width: 0px;
+  height: 5px;
+}
+
+pre::-webkit-scrollbar-thumb {
+  border-radius: 5px;
+  background-color: rgba(0,0,0,.5);
+  -webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
+}
+
+textarea.curl-area {
+  width: 100%;
+  line-height: 1;
+  font-size: 12px;
+  border: none;
+  margin: 0;
+  padding: 0;
+  overflow: scroll;
+}

app/vmalert/static/icons/icons.svg

@@ -0,0 +1,37 @@
+<svg xmlns="http://www.w3.org/2000/svg">
+  <symbol id="vm" viewBox="0 0 464.61 533.89">
+    <path d="M192.16 234.72c9 7.67 24.12 13.49 39.3 13.69h1.68c15.18-.2 30.31-6 39.3-13.69 47.43-40.45 184.65-166.24 184.65-166.24C493.93 34.21 391.45.2 233.14.01h-1.68C73.15.2-29.33 34.21 7.46 68.48c.05 0 137.27 125.79 184.7 166.24"/><path d="M272.4 302.83c-9 7.67-24.12 13.5-39.3 13.7h-1.6c-15.18-.2-30.31-6-39.3-13.7-32.81-28-148.56-132.93-192.16-172.7v60.74c0 6.67 2.55 15.52 7.09 19.68 29.64 27.18 143.94 131.8 185.07 166.88 9 7.67 24.12 13.49 39.3 13.69h1.6c15.18-.2 30.31-6 39.3-13.69 41.13-35.08 155.43-139.7 185.07-166.88 4.54-4.16 7.09-13 7.09-19.68v-60.74c-43.6 39.82-159.35 144.72-192.16 172.7"/><path d="M272.4 445.59c-9 7.67-24.12 13.49-39.3 13.69h-1.6c-15.18-.2-30.31-6-39.3-13.69-32.81-28-148.56-132.94-192.16-172.7v60.73c0 6.67 2.55 15.53 7.09 19.69 29.64 27.17 143.94 131.8 185.07 166.87 9 7.67 24.12 13.5 39.3 13.7h1.6c15.18-.2 30.31-6 39.3-13.7 41.13-35.07 155.43-139.7 185.07-166.87 4.54-4.16 7.09-13 7.09-19.69v-60.73c-43.6 39.76-159.35 144.72-192.16 172.7"/>
+  </symbol>
+
+  <symbol id="collapse" viewBox="3 2 42 44">
+    <path d="M22.6 15.4a1.9 1.9 0 0 0 2.8 0l6-5.9a2.1 2.1 0 0 0 .2-2.7 1.9 1.9 0 0 0-3-.2L26 9.2V4a2 2 0 0 0-4 0v5.2l-2.6-2.6a1.9 1.9 0 0 0-3 .2 2.1 2.1 0 0 0 .2 2.7Zm2.8 17.2a1.9 1.9 0 0 0-2.8 0l-6 5.9a2.1 2.1 0 0 0-.2 2.7 1.9 1.9 0 0 0 3 .2l2.6-2.6V44a2 2 0 0 0 4 0v-5.2l2.6 2.6a1.9 1.9 0 0 0 3-.2 2.1 2.1 0 0 0-.2-2.7ZM6 22h36a2 2 0 0 0 0-4H6a2 2 0 0 0 0 4m36 4H6a2 2 0 0 0 0 4h36a2 2 0 0 0 0-4"/>
+  </symbol>
+
+  <symbol id="expand" viewBox="205.701 131.84 40.006 44.045">
+    <path d="M224.163 175.27a1.9 1.9 0 0 0 2.8 0l6-5.9a2.1 2.1 0 0 0 .2-2.7 1.9 1.9 0 0 0-3-.2l-2.6 2.6v-5.2c0-1.54-1.667-2.502-3-1.732-.619.357-1 1.017-1 1.732v5.2l-2.6-2.6a1.9 1.9 0 0 0-3 .2 2.1 2.1 0 0 0 .2 2.7zm-16.459-23.297h36c1.54 0 2.502-1.667 1.732-3a2 2 0 0 0-1.732-1h-36c-1.54 0-2.502 1.667-1.732 3 .357.619 1.017 1 1.732 1m36 4h-36c-1.54 0-2.502 1.667-1.732 3 .357.619 1.017 1 1.732 1h36c1.54 0 2.502-1.667 1.732-3a2 2 0 0 0-1.732-1m-16.59-23.517a1.9 1.9 0 0 0-2.8 0l-6 5.9a2.1 2.1 0 0 0-.2 2.7 1.9 1.9 0 0 0 3 .2l2.6-2.6v5.2c0 1.54 1.667 2.502 3 1.732.619-.357 1-1.017 1-1.732v-5.2l2.6 2.6a1.9 1.9 0 0 0 3-.2 2.1 2.1 0 0 0-.2-2.7z"/>
+  </symbol>
+
+  <symbol id="filter" viewBox="-10 -10 320 310">
+    <path d="M288.953 0h-277c-5.522 0-10 4.478-10 10v49.531c0 5.522 4.478 10 10 10h12.372l91.378 107.397v113.978a10 10 0 0 0 15.547 8.32l49.5-33a10 10 0 0 0 4.453-8.32v-80.978l91.378-107.397h12.372c5.522 0 10-4.478 10-10V10c0-5.522-4.477-10-10-10M167.587 166.77a10 10 0 0 0-2.384 6.48v79.305l-29.5 19.666V173.25a10 10 0 0 0-2.384-6.48L50.585 69.531h199.736zM278.953 49.531h-257V20h257z"/>
+  </symbol>
+
+  <symbol id="search" viewBox="0 0 490.4 490.4">
+    <path d="m484.1 454.796-110.5-110.6c29.8-36.3 47.6-82.8 47.6-133.4 0-116.3-94.3-210.6-210.6-210.6S0 94.496 0 210.796s94.3 210.6 210.6 210.6c50.8 0 97.4-18 133.8-48l110.5 110.5c12.9 11.8 25 4.2 29.2 0 8.4-8.3 8.4-20.8 0-29.1m-443-244c0-93.6 75.9-169.5 169.5-169.5s169.6 75.9 169.6 169.5-75.9 169.5-169.5 169.5-169.6-75.9-169.6-169.5"/>
+  </symbol>
+
+  <symbol id="all" viewBox="-10 -10 320 310">
+    <path d="M288.953 0h-277c-5.522 0-10 4.478-10 10v49.531c0 5.522 4.478 10 10 10h12.372l91.378 107.397v113.978a10 10 0 0 0 15.547 8.32l49.5-33a10 10 0 0 0 4.453-8.32v-80.978l91.378-107.397h12.372c5.522 0 10-4.478 10-10V10c0-5.522-4.477-10-10-10M167.587 166.77a10 10 0 0 0-2.384 6.48v79.305l-29.5 19.666V173.25a10 10 0 0 0-2.384-6.48L50.585 69.531h199.736zM278.953 49.531h-257V20h257z"/><path stroke-width="20" stroke="#000" d="m-10-10 310 310"/>
+  </symbol>
+
+  <symbol id="nomatch" viewBox="0 0 116.623 116.623">
+    <path d="M99.114 89.251H57.783l19.181-41.797h22.149a5.971 5.971 0 1 0 0-11.942H82.444L94.591 9.049a5.97 5.97 0 0 0-2.938-7.917 5.97 5.97 0 0 0-7.918 2.936L69.304 35.513H41.662c-2.289-.458-17.685-4.48-17.685-29.542a5.97 5.97 0 1 0-11.942 0c0 18.536 7.157 29.533 14.93 35.514-7.774 5.981-14.93 16.978-14.93 35.514a5.971 5.971 0 1 0 11.942 0c0-25.007 14.886-29.013 17.677-29.542h22.17L44.643 89.253H17.509a5.972 5.972 0 0 0 0 11.943h21.653l-3.196 6.965a5.972 5.972 0 1 0 10.856 4.981l5.479-11.943h46.812a5.975 5.975 0 0 0 .001-11.948"/>
+  </symbol>
+
+  <symbol id="exclamation" viewBox="0 0 16 16">
+    <path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
+  </symbol>
+
+  <symbol id="unhealthy" viewBox="0 0 14 14">
+    <circle cx="7" cy="7" r="6.5" stroke="#000" fill="none"/><path stroke-linecap="round" stroke="#000" stroke-width="1.5" d="m5 5 4 4M5 9l4-4"/>
+  </symbol>
+</svg>

app/vmalert/static/js/custom.js

@@ -1,38 +1,32 @@
-function expandAll() {
-    $('.group-heading').each(function () {
-        let style = $(this).attr("style")
-        // display only elements that are currently visible
-        if (style === "display: none;") {
-            return
+function actionAll(isCollapse) {
+    document.querySelectorAll('.collapse').forEach((collapse) => {
+        if (isCollapse) {
+            collapse.classList.remove('show');
+        } else {
+            collapse.classList.add('show');
         }
-        $(this).next().addClass('show')
     });
 }
 
-function collapseAll() {
-    $('.collapse').removeClass('show');
-}
-
-function showByID(id) {
-    if (!id) {
-        return
-    }
-    let parent = $("#" + id).parent();
-    if (!parent) {
-        return
-    }
-    let target = $("#" + parent.attr("data-bs-target"));
-    if (target.length > 0) {
-        target.addClass('show');
+function groupFilter(key) {
+    if (key) {
+        location.href = `?filter=${key}`;
+    } else {
+        window.location = window.location.pathname;
     }
 }
 
-function toggleByID(id) {
-    if (id) {
-        let el = $("#" + id);
-        if (el.length > 0) {
-            el.click();
-        }
+function showBySelector(selector) {
+    if (!selector) {
+        return
+    }
+    const control = document.querySelector(`${selector} [data-bs-target]`);
+    if (!control) {
+        return
+    }
+    let target = document.getElementById(control.getAttribute('data-bs-target').slice(1));
+    if (target) {
+        target.classList.add('show');
     }
 }
 
@@ -46,28 +40,16 @@ function debounce(func, delay) {
     };
 }
 
-$('#search').on("keyup", debounce(search, 500));
-
 // search shows or hides groups&rules that satisfy the search phrase.
 // case-insensitive, respects GET param `search`.
 function search() {
-    $(".rule").show();
-
-    let groupHeader = $(".group-heading")
-    let searchPhrase = $("#search").val().toLowerCase()
-    if (searchPhrase.length === 0) {
-        groupHeader.show()
-        setParamURL('search', '')
-        return
+    let searchBox = document.getElementById('search');
+    if (!searchBox) {
+        return;
     }
+    const searchPhrase = searchBox.value.toLowerCase();
 
-    $(".rule-table").removeClass('show');
-    groupHeader.hide()
-
-    searchPhrase = searchPhrase.toLowerCase()
-    filterRuleByName(searchPhrase);
-    filterRuleByLabels(searchPhrase);
-    filterGroupsByName(searchPhrase);
+    filterRules(searchPhrase);
 
     setParamURL('search', searchPhrase)
 }
@@ -83,81 +65,51 @@ function getParamURL(key) {
     return url.searchParams.get(key)
 }
 
-function filterGroupsByName(searchPhrase) {
-    $(".group-heading").each(function () {
-        const groupName = $(this).attr('data-group-name').toLowerCase();
-        const hasValue = groupName.indexOf(searchPhrase) >= 0
-
-        if (!hasValue) {
-            return
-        }
-
-        const target = $(this).attr("data-bs-target");
-        $(`div[id="${target}"] .rule`).show();
-        $(this).show();
-    });
-}
-
-function filterRuleByName(searchPhrase) {
-    $(".rule").each(function () {
-        const ruleName = $(this).attr("data-rule-name").toLowerCase();
-        const hasValue = ruleName.indexOf(searchPhrase) >= 0
-        if (!hasValue) {
-            $(this).hide();
-            return
-        }
-
-        const target = $(this).attr('data-bs-target')
-        $(`#rules-${target}`).addClass('show');
-        $(`div[data-bs-target='rules-${target}']`).show();
-        $(this).show();
-    });
-}
-
-function filterRuleByLabels(searchPhrase) {
-    $(".rule").each(function () {
-        const matches = $(".label", this).filter(function () {
-            const label = $(this).text().toLowerCase();
-            return label.indexOf(searchPhrase) >= 0;
-        }).length;
-
-        if (matches > 0) {
-            const target = $(this).attr('data-bs-target')
-            $(`#rules-${target}`).addClass('show');
-            $(`div[data-bs-target='rules-${target}']`).show();
-            $(this).show();
-        }
-    });
-}
-
-$(document).ready(function () {
-    $(".group-heading a").click(function (e) {
-        e.stopPropagation(); // prevent collapse logic on link click
-        let target = $(this).attr('href');
-        if (target.length > 0) {
-            toggleByID(target.substr(1));
-        }
-    });
-
-    $(".group-heading").click(function (e) {
-        let target = $(this).attr('data-bs-target');
-        let el = $("#" + target);
-        new bootstrap.Collapse(el, {
-            toggle: true
+function filterRules(searchPhrase) {
+    document.querySelectorAll('.sub-items').forEach((rules) => {
+        let found = false;
+        rules.querySelectorAll('.sub-item').forEach((rule) => {
+            if (searchPhrase) {
+                const ruleName = rule.innerText.toLowerCase();
+                const matches = []
+                const hasValue = ruleName.indexOf(searchPhrase) >= 0;
+                rule.querySelectorAll('.label').forEach((label) => {
+                    const text = label.innerText.toLowerCase();
+                    if (text.indexOf(searchPhrase) >= 0) {
+                        matches.push(text);
+                    }
+                });
+                if (!matches.length && !hasValue) {
+                    rule.classList.add('d-none');
+                    return;
+                }
+            }
+            rule.classList.remove('d-none');
+            found = true;
         });
+        if (found && searchPhrase || !searchPhrase) {
+            rules.classList.remove('d-none');
+        } else {
+            rules.classList.add('d-none');
+        }
     });
+}
 
+document.addEventListener('DOMContentLoaded', () => {
     // update search element with value from URL, if any
-    let searchPhrase = getParamURL('search')
-    $("#search").val(searchPhrase)
+    const searchPhrase = getParamURL('search')
+    const searchBox = document.getElementById('search');
+    if (searchBox) {
+        searchBox.addEventListener('keyup', debounce(search, 500));
+        searchBox.value = searchPhrase;
+    }
 
     // apply filtering by search phrase
     search()
 
-    let hash = window.location.hash.substr(1);
-    showByID(hash);
-});
+    showBySelector(window.location.hash);
 
-$(document).ready(function () {
-    $('[data-bs-toggle="tooltip"]').tooltip();
+    document.querySelectorAll('[data-bs-toggle="tooltip"]').forEach((tooltip) => {
+        new bootstrap.Tooltip(tooltip);
+    });
 });

app/vmalert/tpl/footer.qtpl

@@ -1,16 +1,10 @@
 {% import (
     "net/http"
-
-    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 ) %}
 
 
 {% func Footer(r *http.Request) %}
-    {%code prefix := vmalertutil.Prefix(r.URL.Path) %}
         </main>
-        <script src="{%s prefix %}static/js/jquery-3.6.0.min.js" type="text/javascript"></script>
-        <script src="{%s prefix %}static/js/bootstrap.bundle.min.js" type="text/javascript"></script>
-        <script src="{%s prefix %}static/js/custom.js" type="text/javascript"></script>
     </body>
 </html>
 {% endfunc %}

app/vmalert/tpl/footer.qtpl.go

@@ -7,77 +7,54 @@ package tpl
 //line app/vmalert/tpl/footer.qtpl:1
 import (
 	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 )
 
-//line app/vmalert/tpl/footer.qtpl:8
+//line app/vmalert/tpl/footer.qtpl:6
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmalert/tpl/footer.qtpl:8
+//line app/vmalert/tpl/footer.qtpl:6
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmalert/tpl/footer.qtpl:8
+//line app/vmalert/tpl/footer.qtpl:6
 func StreamFooter(qw422016 *qt422016.Writer, r *http.Request) {
-//line app/vmalert/tpl/footer.qtpl:8
-	qw422016.N().S(`
-    `)
-//line app/vmalert/tpl/footer.qtpl:9
-	prefix := vmalertutil.Prefix(r.URL.Path)
-
-//line app/vmalert/tpl/footer.qtpl:9
+//line app/vmalert/tpl/footer.qtpl:6
 	qw422016.N().S(`
         </main>
-        <script src="`)
-//line app/vmalert/tpl/footer.qtpl:11
-	qw422016.E().S(prefix)
-//line app/vmalert/tpl/footer.qtpl:11
-	qw422016.N().S(`static/js/jquery-3.6.0.min.js" type="text/javascript"></script>
-        <script src="`)
-//line app/vmalert/tpl/footer.qtpl:12
-	qw422016.E().S(prefix)
-//line app/vmalert/tpl/footer.qtpl:12
-	qw422016.N().S(`static/js/bootstrap.bundle.min.js" type="text/javascript"></script>
-        <script src="`)
-//line app/vmalert/tpl/footer.qtpl:13
-	qw422016.E().S(prefix)
-//line app/vmalert/tpl/footer.qtpl:13
-	qw422016.N().S(`static/js/custom.js" type="text/javascript"></script>
     </body>
 </html>
 `)
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 }
 
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 func WriteFooter(qq422016 qtio422016.Writer, r *http.Request) {
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 	StreamFooter(qw422016, r)
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 }
 
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 func Footer(r *http.Request) string {
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 	WriteFooter(qb422016, r)
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 	return qs422016
-//line app/vmalert/tpl/footer.qtpl:16
+//line app/vmalert/tpl/footer.qtpl:10
 }

app/vmalert/tpl/header.qtpl

@@ -1,7 +1,7 @@
 {% import (
     "net/http"
-    "path"
     "net/url"
+    "path"
 
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 ) %}
@@ -13,65 +13,13 @@
 <head>
     <title>vmalert{% if title != "" %} - {%s title %}{% endif %}</title>
     <link href="{%s prefix %}static/css/bootstrap.min.css" rel="stylesheet" />
-    <style>
-        body{
-          min-height: 75rem;
-          padding-top: 4.5rem;
-        }
-        .group-heading {
-            cursor: pointer;
-            padding: 5px;
-            margin-top: 5px;
-            position: relative;
-        }
-        .group-heading .anchor {
-            position:absolute;
-            top:-60px;
-        }
-        .group-heading span {
-            float: right;
-            margin-left: 5px;
-            margin-right: 5px;
-        }
-         .group-heading:hover {
-            background-color: #f8f9fa!important;
-        }
-        .table {
-            table-layout: fixed;
-        }
-        .table .error-cell{
-            word-break: break-word;
-            font-size: 14px;
-        }
-        pre {
-            overflow: scroll;
-            min-height: 30px;
-            max-width: 100%;
-        }
-        pre::-webkit-scrollbar {
-          -webkit-appearance: none;
-          width: 0px;
-          height: 5px;
-        }
-        pre::-webkit-scrollbar-thumb {
-          border-radius: 5px;
-          background-color: rgba(0,0,0,.5);
-          -webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
-        }
-        textarea.curl-area{
-            width: 100%;
-            line-height: 1;
-            font-size: 12px;
-            border: none;
-            margin: 0;
-            padding: 0;
-            overflow: scroll;
-        }
-    </style>
+    <link href="{%s prefix %}static/css/custom.css" rel="stylesheet" />
+    <script src="{%s prefix %}static/js/bootstrap.bundle.min.js" type="text/javascript" defer></script>
+    <script src="{%s prefix %}static/js/custom.js" type="text/javascript" defer></script>
 </head>
 <body>
     {%= printNavItems(r, title, navItems, userErr) %}
-    <main class="px-2">
+    <main class="px-4 py-2">
     {%= errorBody(userErr) %}
 {% endfunc %}
 
@@ -79,25 +27,27 @@
 {% code
 type NavItem struct {
 	Name string
-	Url  string
+	URL  string
+	Icon string
 }
 %}
 
 {% func printNavItems(r *http.Request, current string, items []NavItem, userErr error) %}
-{%code
-    prefix := vmalertutil.Prefix(r.URL.Path)
-%}
-<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+{%code prefix := vmalertutil.Prefix(r.URL.Path) %}
+<nav class="navbar navbar-expand navbar-dark fixed-top bg-dark">
   <div class="container-fluid">
     <div class="collapse navbar-collapse" id="navbarCollapse">
-        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+        <ul class="navbar-nav me-auto mb-0 align-items-center">
             {% for _, item := range items  %}
                 <li class="nav-item">
                     {% code
-                    u, _ := url.Parse(item.Url)
+                    u, _ := url.Parse(item.URL)
                     %}
-                    <a class="nav-link{% if current == item.Name %} active{% endif %}"
-                       href="{% if u.IsAbs() %}{%s item.Url %}{% else %}{%s path.Join(prefix,item.Url) %}{% endif %}">
+                    <a class="d-flex gap-2 align-items-center nav-link{% if current == item.Name %} active{% endif %}"
+                       href="{% if u.IsAbs() %}{%s item.URL %}{% else %}{%s path.Join(prefix, item.URL) %}{% endif %}">
+                        {% if item.Icon != "" %}
+                        <svg width="40" height="40"><use href="{%s prefix %}static/icons/icons.svg#{%s item.Icon %}"></svg>
+                        {% endif %}
                         {%s item.Name %}
                     </a>
                 </li>

app/vmalert/tpl/header.qtpl.go

@@ -55,216 +55,196 @@ func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem
 	qw422016.E().S(prefix)
 //line app/vmalert/tpl/header.qtpl:15
 	qw422016.N().S(`static/css/bootstrap.min.css" rel="stylesheet" />
-    <style>
-        body{
-          min-height: 75rem;
-          padding-top: 4.5rem;
-        }
-        .group-heading {
-            cursor: pointer;
-            padding: 5px;
-            margin-top: 5px;
-            position: relative;
-        }
-        .group-heading .anchor {
-            position:absolute;
-            top:-60px;
-        }
-        .group-heading span {
-            float: right;
-            margin-left: 5px;
-            margin-right: 5px;
-        }
-         .group-heading:hover {
-            background-color: #f8f9fa!important;
-        }
-        .table {
-            table-layout: fixed;
-        }
-        .table .error-cell{
-            word-break: break-word;
-            font-size: 14px;
-        }
-        pre {
-            overflow: scroll;
-            min-height: 30px;
-            max-width: 100%;
-        }
-        pre::-webkit-scrollbar {
-          -webkit-appearance: none;
-          width: 0px;
-          height: 5px;
-        }
-        pre::-webkit-scrollbar-thumb {
-          border-radius: 5px;
-          background-color: rgba(0,0,0,.5);
-          -webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
-        }
-        textarea.curl-area{
-            width: 100%;
-            line-height: 1;
-            font-size: 12px;
-            border: none;
-            margin: 0;
-            padding: 0;
-            overflow: scroll;
-        }
-    </style>
+    <link href="`)
+//line app/vmalert/tpl/header.qtpl:16
+	qw422016.E().S(prefix)
+//line app/vmalert/tpl/header.qtpl:16
+	qw422016.N().S(`static/css/custom.css" rel="stylesheet" />
+    <script src="`)
+//line app/vmalert/tpl/header.qtpl:17
+	qw422016.E().S(prefix)
+//line app/vmalert/tpl/header.qtpl:17
+	qw422016.N().S(`static/js/bootstrap.bundle.min.js" type="text/javascript" defer></script>
+    <script src="`)
+//line app/vmalert/tpl/header.qtpl:18
+	qw422016.E().S(prefix)
+//line app/vmalert/tpl/header.qtpl:18
+	qw422016.N().S(`static/js/custom.js" type="text/javascript" defer></script>
 </head>
 <body>
     `)
-//line app/vmalert/tpl/header.qtpl:73
+//line app/vmalert/tpl/header.qtpl:21
 	streamprintNavItems(qw422016, r, title, navItems, userErr)
-//line app/vmalert/tpl/header.qtpl:73
+//line app/vmalert/tpl/header.qtpl:21
 	qw422016.N().S(`
-    <main class="px-2">
+    <main class="px-4 py-2">
     `)
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:23
 	streamerrorBody(qw422016, userErr)
-//line app/vmalert/tpl/header.qtpl:75
+//line app/vmalert/tpl/header.qtpl:23
 	qw422016.N().S(`
 `)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 }
 
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 func WriteHeader(qq422016 qtio422016.Writer, r *http.Request, navItems []NavItem, title string, userErr error) {
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 	StreamHeader(qw422016, r, navItems, title, userErr)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 }
 
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 func Header(r *http.Request, navItems []NavItem, title string, userErr error) string {
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 	WriteHeader(qb422016, r, navItems, title, userErr)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:24
 }
 
-//line app/vmalert/tpl/header.qtpl:80
+//line app/vmalert/tpl/header.qtpl:28
 type NavItem struct {
 	Name string
-	Url  string
+	URL  string
+	Icon string
 }
 
-//line app/vmalert/tpl/header.qtpl:86
+//line app/vmalert/tpl/header.qtpl:35
 func streamprintNavItems(qw422016 *qt422016.Writer, r *http.Request, current string, items []NavItem, userErr error) {
-//line app/vmalert/tpl/header.qtpl:86
+//line app/vmalert/tpl/header.qtpl:35
 	qw422016.N().S(`
 `)
-//line app/vmalert/tpl/header.qtpl:88
+//line app/vmalert/tpl/header.qtpl:36
 	prefix := vmalertutil.Prefix(r.URL.Path)
 
-//line app/vmalert/tpl/header.qtpl:89
+//line app/vmalert/tpl/header.qtpl:36
 	qw422016.N().S(`
-<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
+<nav class="navbar navbar-expand navbar-dark fixed-top bg-dark">
   <div class="container-fluid">
     <div class="collapse navbar-collapse" id="navbarCollapse">
-        <ul class="navbar-nav me-auto mb-2 mb-md-0">
+        <ul class="navbar-nav me-auto mb-0 align-items-center">
             `)
-//line app/vmalert/tpl/header.qtpl:94
+//line app/vmalert/tpl/header.qtpl:41
 	for _, item := range items {
-//line app/vmalert/tpl/header.qtpl:94
+//line app/vmalert/tpl/header.qtpl:41
 		qw422016.N().S(`
                 <li class="nav-item">
                     `)
-//line app/vmalert/tpl/header.qtpl:97
-		u, _ := url.Parse(item.Url)
+//line app/vmalert/tpl/header.qtpl:44
+		u, _ := url.Parse(item.URL)
 
-//line app/vmalert/tpl/header.qtpl:98
+//line app/vmalert/tpl/header.qtpl:45
 		qw422016.N().S(`
-                    <a class="nav-link`)
-//line app/vmalert/tpl/header.qtpl:99
+                    <a class="d-flex gap-2 align-items-center nav-link`)
+//line app/vmalert/tpl/header.qtpl:46
 		if current == item.Name {
-//line app/vmalert/tpl/header.qtpl:99
+//line app/vmalert/tpl/header.qtpl:46
 			qw422016.N().S(` active`)
-//line app/vmalert/tpl/header.qtpl:99
+//line app/vmalert/tpl/header.qtpl:46
 		}
-//line app/vmalert/tpl/header.qtpl:99
+//line app/vmalert/tpl/header.qtpl:46
 		qw422016.N().S(`"
                        href="`)
-//line app/vmalert/tpl/header.qtpl:100
+//line app/vmalert/tpl/header.qtpl:47
 		if u.IsAbs() {
-//line app/vmalert/tpl/header.qtpl:100
-			qw422016.E().S(item.Url)
-//line app/vmalert/tpl/header.qtpl:100
+//line app/vmalert/tpl/header.qtpl:47
+			qw422016.E().S(item.URL)
+//line app/vmalert/tpl/header.qtpl:47
 		} else {
-//line app/vmalert/tpl/header.qtpl:100
-			qw422016.E().S(path.Join(prefix, item.Url))
-//line app/vmalert/tpl/header.qtpl:100
+//line app/vmalert/tpl/header.qtpl:47
+			qw422016.E().S(path.Join(prefix, item.URL))
+//line app/vmalert/tpl/header.qtpl:47
 		}
-//line app/vmalert/tpl/header.qtpl:100
+//line app/vmalert/tpl/header.qtpl:47
 		qw422016.N().S(`">
                         `)
-//line app/vmalert/tpl/header.qtpl:101
+//line app/vmalert/tpl/header.qtpl:48
+		if item.Icon != "" {
+//line app/vmalert/tpl/header.qtpl:48
+			qw422016.N().S(`
+                        <svg width="40" height="40"><use href="`)
+//line app/vmalert/tpl/header.qtpl:49
+			qw422016.E().S(prefix)
+//line app/vmalert/tpl/header.qtpl:49
+			qw422016.N().S(`static/icons/icons.svg#`)
+//line app/vmalert/tpl/header.qtpl:49
+			qw422016.E().S(item.Icon)
+//line app/vmalert/tpl/header.qtpl:49
+			qw422016.N().S(`"></svg>
+                        `)
+//line app/vmalert/tpl/header.qtpl:50
+		}
+//line app/vmalert/tpl/header.qtpl:50
+		qw422016.N().S(`
+                        `)
+//line app/vmalert/tpl/header.qtpl:51
 		qw422016.E().S(item.Name)
-//line app/vmalert/tpl/header.qtpl:101
+//line app/vmalert/tpl/header.qtpl:51
 		qw422016.N().S(`
                     </a>
                 </li>
             `)
-//line app/vmalert/tpl/header.qtpl:104
+//line app/vmalert/tpl/header.qtpl:54
 	}
-//line app/vmalert/tpl/header.qtpl:104
+//line app/vmalert/tpl/header.qtpl:54
 	qw422016.N().S(`
         </ul>
   </div>
   `)
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:57
 	streamerrorIcon(qw422016, userErr)
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:57
 	qw422016.N().S(`
 </nav>
 `)
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 }
 
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 func writeprintNavItems(qq422016 qtio422016.Writer, r *http.Request, current string, items []NavItem, userErr error) {
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 	streamprintNavItems(qw422016, r, current, items, userErr)
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 }
 
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 func printNavItems(r *http.Request, current string, items []NavItem, userErr error) string {
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 	writeprintNavItems(qb422016, r, current, items, userErr)
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:109
+//line app/vmalert/tpl/header.qtpl:59
 }
 
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:61
 func streamerrorIcon(qw422016 *qt422016.Writer, err error) {
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:61
 	qw422016.N().S(`
 `)
-//line app/vmalert/tpl/header.qtpl:112
+//line app/vmalert/tpl/header.qtpl:62
 	if err != nil {
-//line app/vmalert/tpl/header.qtpl:112
+//line app/vmalert/tpl/header.qtpl:62
 		qw422016.N().S(`
 <div class="d-flex" data-bs-toggle="tooltip" data-bs-placement="left" title="Configuration file failed to reload! Click to see more details.">
   <a type="button" data-bs-toggle="collapse" href="#reload-groups-error">
@@ -276,89 +256,89 @@ func streamerrorIcon(qw422016 *qt422016.Writer, err error) {
   </a>
 </div>
 `)
-//line app/vmalert/tpl/header.qtpl:122
+//line app/vmalert/tpl/header.qtpl:72
 	}
-//line app/vmalert/tpl/header.qtpl:122
+//line app/vmalert/tpl/header.qtpl:72
 	qw422016.N().S(`
 `)
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 }
 
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 func writeerrorIcon(qq422016 qtio422016.Writer, err error) {
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 	streamerrorIcon(qw422016, err)
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 }
 
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 func errorIcon(err error) string {
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 	writeerrorIcon(qb422016, err)
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:123
+//line app/vmalert/tpl/header.qtpl:73
 }
 
-//line app/vmalert/tpl/header.qtpl:125
+//line app/vmalert/tpl/header.qtpl:75
 func streamerrorBody(qw422016 *qt422016.Writer, err error) {
-//line app/vmalert/tpl/header.qtpl:125
+//line app/vmalert/tpl/header.qtpl:75
 	qw422016.N().S(`
 `)
-//line app/vmalert/tpl/header.qtpl:126
+//line app/vmalert/tpl/header.qtpl:76
 	if err != nil {
-//line app/vmalert/tpl/header.qtpl:126
+//line app/vmalert/tpl/header.qtpl:76
 		qw422016.N().S(`
 <div class="collapse mt-2 mb-2" id="reload-groups-error">
   <div class="card card-body">
     `)
-//line app/vmalert/tpl/header.qtpl:129
+//line app/vmalert/tpl/header.qtpl:79
 		qw422016.E().S(err.Error())
-//line app/vmalert/tpl/header.qtpl:129
+//line app/vmalert/tpl/header.qtpl:79
 		qw422016.N().S(`
   </div>
 </div>
 `)
-//line app/vmalert/tpl/header.qtpl:132
+//line app/vmalert/tpl/header.qtpl:82
 	}
-//line app/vmalert/tpl/header.qtpl:132
+//line app/vmalert/tpl/header.qtpl:82
 	qw422016.N().S(`
 `)
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 }
 
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 func writeerrorBody(qq422016 qtio422016.Writer, err error) {
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 	streamerrorBody(qw422016, err)
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 }
 
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 func errorBody(err error) string {
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 	writeerrorBody(qb422016, err)
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:133
+//line app/vmalert/tpl/header.qtpl:83
 }

app/vmalert/vmalertutil/links.go

@@ -11,7 +11,7 @@ const prefix = "/vmalert/"
 
 // Prefix returns "/vmalert/" prefix if it is missing in the path.
 func Prefix(path string) string {
-	pp := httpserver.GetPathPrefix()
+	pp := strings.TrimRight(httpserver.GetPathPrefix(), "/")
 	path = strings.TrimLeft(path, pp)
 	if strings.HasPrefix(path, prefix) {
 		return pp

app/vmalert/web.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"sort"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -30,16 +30,17 @@ var (
 		{fmt.Sprintf("api/v1/alert?%s=<int>&%s=<int>", paramGroupID, paramAlertID), "get alert status by group and alert ID"},
 	}
 	systemLinks = [][2]string{
+		{"vmalert/groups", "UI"},
 		{"flags", "command-line flags"},
 		{"metrics", "list of application metrics"},
 		{"-/reload", "reload configuration"},
 	}
 	navItems = []tpl.NavItem{
-		{Name: "vmalert", Url: "../vmalert"},
-		{Name: "Groups", Url: "groups"},
-		{Name: "Alerts", Url: "alerts"},
-		{Name: "Notifiers", Url: "notifiers"},
-		{Name: "Docs", Url: "https://docs.victoriametrics.com/vmalert/"},
+		{Name: "vmalert", URL: "../vmalert", Icon: "vm"},
+		{Name: "Groups", URL: "groups"},
+		{Name: "Alerts", URL: "alerts"},
+		{Name: "Notifiers", URL: "notifiers"},
+		{Name: "Docs", URL: "https://docs.victoriametrics.com/victoriametrics/vmalert/"},
 	}
 )
 
@@ -88,10 +89,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		WriteRuleDetails(w, r, rule)
 		return true
 	case "/vmalert/groups":
-		var data []apiGroup
-		rf := extractRulesFilter(r)
-		data = rh.groups(rf)
-		WriteListGroups(w, r, data)
+		filter := r.URL.Query().Get("filter")
+		rf := extractRulesFilter(r, filter)
+		data := rh.groups(rf)
+		WriteListGroups(w, r, data, filter)
 		return true
 	case "/vmalert/notifiers":
 		WriteListTargets(w, r, notifier.GetTargets())
@@ -103,9 +104,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		// Grafana makes an extra request to `/rules`
 		// handler in addition to `/api/v1/rules` calls in alerts UI,
 		var data []apiGroup
-		rf := extractRulesFilter(r)
+		filter := r.URL.Query().Get("filter")
+		rf := extractRulesFilter(r, filter)
 		data = rh.groups(rf)
-		WriteListGroups(w, r, data)
+		WriteListGroups(w, r, data, filter)
 		return true
 
 	case "/vmalert/api/v1/rules", "/api/v1/rules":
@@ -113,7 +115,8 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		var data []byte
 		var err error
 
-		rf := extractRulesFilter(r)
+		filter := r.URL.Query().Get("filter")
+		rf := extractRulesFilter(r, filter)
 		data, err = rh.listGroups(rf)
 
 		if err != nil {
@@ -226,9 +229,11 @@ type rulesFilter struct {
 	ruleNames     []string
 	ruleType      string
 	excludeAlerts bool
+	onlyUnhealthy bool
+	onlyNoMatch   bool
 }
 
-func extractRulesFilter(r *http.Request) rulesFilter {
+func extractRulesFilter(r *http.Request, filter string) rulesFilter {
 	rf := rulesFilter{}
 
 	var ruleType string
@@ -246,6 +251,12 @@ func extractRulesFilter(r *http.Request) rulesFilter {
 	rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
 	rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
 	rf.files = append([]string{}, r.Form["file[]"]...)
+	switch filter {
+	case "unhealthy":
+		rf.onlyUnhealthy = true
+	case "noMatch":
+		rf.onlyNoMatch = true
+	}
 	return rf
 }
 
@@ -253,24 +264,12 @@ func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	isInList := func(list []string, needle string) bool {
-		if len(list) < 1 {
-			return true
-		}
-		for _, i := range list {
-			if i == needle {
-				return true
-			}
-		}
-		return false
-	}
-
 	groups := make([]apiGroup, 0)
 	for _, group := range rh.m.groups {
-		if !isInList(rf.groupNames, group.Name) {
+		if len(rf.groupNames) > 0 && !slices.Contains(rf.groupNames, group.Name) {
 			continue
 		}
-		if !isInList(rf.files, group.File) {
+		if len(rf.files) > 0 && !slices.Contains(rf.files, group.File) {
 			continue
 		}
 
@@ -282,24 +281,34 @@ func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
 			if rf.ruleType != "" && rf.ruleType != r.Type {
 				continue
 			}
-			if !isInList(rf.ruleNames, r.Name) {
+			if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, r.Name) {
 				continue
 			}
 			if rf.excludeAlerts {
 				r.Alerts = nil
 			}
+			if (r.LastError == "" && rf.onlyUnhealthy) || (!isNoMatch(r) && rf.onlyNoMatch) {
+				continue
+			}
+			if r.LastError != "" {
+				g.Unhealthy++
+			} else {
+				g.Healthy++
+			}
+			if isNoMatch(r) {
+				g.NoMatch++
+			}
 			filteredRules = append(filteredRules, r)
 		}
 		g.Rules = filteredRules
 		groups = append(groups, g)
 	}
 	// sort list of groups for deterministic output
-	sort.Slice(groups, func(i, j int) bool {
-		a, b := groups[i], groups[j]
+	slices.SortFunc(groups, func(a, b apiGroup) int {
 		if a.Name != b.Name {
-			return a.Name < b.Name
+			return strings.Compare(a.Name, b.Name)
 		}
-		return a.File < b.File
+		return strings.Compare(a.File, b.File)
 	})
 	return groups
 }
@@ -345,8 +354,8 @@ func (rh *requestHandler) groupAlerts() []groupAlerts {
 			})
 		}
 	}
-	sort.Slice(gAlerts, func(i, j int) bool {
-		return gAlerts[i].Group.Name < gAlerts[j].Group.Name
+	slices.SortFunc(gAlerts, func(a, b groupAlerts) int {
+		return strings.Compare(a.Group.Name, b.Group.Name)
 	})
 	return gAlerts
 }
@@ -368,8 +377,8 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 	}
 
 	// sort list of alerts for deterministic output
-	sort.Slice(lr.Data.Alerts, func(i, j int) bool {
-		return lr.Data.Alerts[i].ID < lr.Data.Alerts[j].ID
+	slices.SortFunc(lr.Data.Alerts, func(a, b *apiAlert) int {
+		return strings.Compare(a.ID, b.ID)
 	})
 
 	b, err := json.Marshal(lr)

app/vmalert/web.qtpl

@@ -10,6 +10,69 @@
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 ) %}
 
+{% func Controls(prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) %}
+    <div class="btn-toolbar mb-3" role="toolbar">
+        <div class="d-flex gap-2 justify-content-between w-100">
+            <div class="d-flex gap-2 align-items-center">
+                <a class="btn btn-outline-dark d-flex align-items-center" role="button" onclick="actionAll(true)">
+                    <span class="d-none d-md-block">Collapse All</span>
+                    <svg class="d-md-none" height="20" width="20">
+                        <use href="{%s prefix %}static/icons/icons.svg#collapse"/>
+                    </svg>
+                </a>
+                <a class="btn btn-outline-dark d-flex align-items-center" role="button" onclick="actionAll(false)">
+                    <span class="d-none d-md-block">Expand All</span>
+                    <svg class="d-md-none" width="20" height="20">
+                        <use href="{%s prefix %}static/icons/icons.svg#expand"/>
+                    </svg>
+                </a>
+                {% if len(filters) > 0 %}
+                    <span class="d-none d-md-inline-block">Filter by status:</span>
+                    <svg class="d-md-none" width="20" height="20">
+                        <use href="{%s prefix %}static/icons/icons.svg#filter">
+                    </svg>
+                    <div class="dropdown">
+                        <button
+                            class="btn btn-outline-dark dropdown-toggle d-flex justify-content-between align-items-center"
+                            type="button"
+                            data-bs-toggle="dropdown"
+                            aria-expanded="false"
+                        >
+                            <span class="d-none d-md-inline-block">{%s currentText %}</span>
+                            <svg class="d-md-none" width="22" height="22">
+                                <use href="{%s prefix %}static/icons/icons.svg#{%s currentIcon %}"/>
+                            </svg>
+                        </button>
+                        <ul class="dropdown-menu">
+                            {% for key, title := range filters %}
+                                {% if title != currentText %}
+                                    <li>
+                                        <a class="dropdown-item" onclick="groupFilter('{%s key %}')">
+                                            <span class="d-none d-md-inline-block">{%s title %}</span>
+                                            <svg class="d-md-none" width="22" height="22">
+                                                <use href="{%s prefix %}static/icons/icons.svg#{%s icons[key] %}"/>
+                                            </svg>
+                                        </a>
+                                    </li>
+                                {% endif %}
+                            {% endfor %}
+                        </ul>
+                    </div>
+                {% endif %}
+            </div>
+            {% if search %}
+                <div class="input-group flex-grow-1 justify-content-end">
+                    <span class="input-group-text">
+                        <svg height="25" width="20">
+                            <use href="{%s prefix %}static/icons/icons.svg#search">
+                        </svg>
+                    </span>
+                    <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
+                </div>
+            {% endif %}
+        </div>
+    </div>
+{% endfunc %}
 
 {% func Welcome(r *http.Request) %}
     {%= tpl.Header(r, navItems, "vmalert", getLastConfigError()) %}
@@ -30,143 +93,124 @@
     {%= tpl.Footer(r) %}
 {% endfunc %}
 
-{% func buttonActive(filter, expValue string) %}
-    {% if filter != expValue %}
-btn-secondary
-    {% else %}
-btn-primary
-    {% endif %}
-{% endfunc %}
-
-{% func ListGroups(r *http.Request, originGroups []apiGroup) %}
-    {%code prefix := vmalertutil.Prefix(r.URL.Path) %}
+{% func ListGroups(r *http.Request, groups []apiGroup, filter string) %}
+    {%code
+        prefix := vmalertutil.Prefix(r.URL.Path)
+        filters := map[string]string{
+            "":          "All",
+            "unhealthy": "Unhealthy",
+            "noMatch":   "No Match",
+        }
+        icons := map[string]string{
+            "":          "all",
+            "unhealthy": "unhealthy",
+            "noMatch":   "nomatch",
+        }
+        currentText := filters[filter]
+        currentIcon := icons[filter]
+    %}
     {%= tpl.Header(r, navItems, "Groups", getLastConfigError()) %}
-        {%code
-            filter := r.URL.Query().Get("filter")
-            rOk := make(map[string]int)
-            rNotOk := make(map[string]int)
-            rNoMatch := make(map[string]int)
-            var groups []apiGroup
-            for _, g := range originGroups {
-                var rules []apiRule
-                for _, r := range g.Rules {
-                    if r.LastError != "" {
-                        rNotOk[g.ID]++
-                    } else {
-                        rOk[g.ID]++
-                    }
-                    if isNoMatch(r) {
-                        rNoMatch[g.ID]++
-                    }
-                    if (filter == "unhealthy" && r.LastError == "") ||
-                       (filter == "noMatch" && !isNoMatch(r)) {
-                        continue
-                    }
-                    rules = append(rules, r)
-                }
-                if len(rules) > 0 {
-                    g.Rules = rules
-                    groups = append(groups, g)
-                }
-            }
-        %}
-        <div class="btn-toolbar mb-3" role="toolbar">
-          <div>
-            <a class="btn {%= buttonActive(filter, "") %}" role="button" onclick="window.location = window.location.pathname">All</a>
-            <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-            <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
-            <a class="btn {%= buttonActive(filter, "unhealthy") %}" role="button" onclick="location.href='?filter=unhealthy'" title="Show only rules with errors">Unhealthy</a>
-            <a class="btn {%= buttonActive(filter, "noMatch") %}" role="button" onclick="location.href='?filter=noMatch'" title="Show only rules matching no time series during last evaluation">NoMatch</a>
-          </div>
-          <div class="col-md-4 col-lg-5">
-            <div class="px-3 input-group">
-              <div class="input-group-prepend">
-                <span class="input-group-text">
-                  <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
-                </span>
-              </div>
-              <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
-            </div>
-          </div>
-        </div>
+        {%= Controls(prefix, currentIcon, currentText, icons, filters, true) %}
         {%  if len(groups) > 0 %}
-            {% for _, g := range groups  %}
-                  <div
-                    class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{%endif%}" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
-                    <span class="anchor" id="group-{%s g.ID %}"></span>
-                    <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
-                     {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
-                     {% if rNoMatch[g.ID] > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d rNoMatch[g.ID] %}</span> {% endif %}
-                    <span class="badge bg-success" title="Number of rules with status Ok">{%d rOk[g.ID] %}</span>
-                    <p class="fs-6 fw-lighter">{%s g.File %}</p>
-                    {% if len(g.Params) > 0 %}
-                        <div class="fs-6 fw-lighter">Extra params
-                        {% for _, param := range g.Params %}
-                                <span class="float-left badge bg-primary">{%s param %}</span>
-                        {% endfor %}
-                        </div>
-                    {% endif %}
-                    {% if len(g.Headers) > 0 %}
-                        <div class="fs-6 fw-lighter">Extra headers
-                        {% for _, header := range g.Headers %}
-                                <span class="float-left badge bg-primary">{%s header %}</span>
-                        {% endfor %}
-                        </div>
-                    {% endif %}
-                </div>
-                <div class="collapse rule-table" id="rules-{%s g.ID %}">
-                    <table class="table table-striped table-hover table-sm">
-                        <thead>
-                            <tr>
-                                <th scope="col" style="width: 60%">Rule</th>
-                                <th scope="col" style="width: 20%" class="text-center" title="How many samples were produced by the rule">Samples</th>
-                                <th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
-                            </tr>
-                        </thead>
-                        <tbody>
-                        {% for _, r := range g.Rules %}
-                            <tr class="rule{% if r.LastError != "" %} alert-danger{% endif %}" data-rule-name="{%s r.Name %}" data-bs-target="{%s g.ID %}">
-                                <td>
-                                    <div class="row">
-                                        <div class="col-12 mb-2">
-                                            {% if r.Type == "alerting" %}
-                                            {% if r.KeepFiringFor > 0 %}
-                                            <b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds, keep_firing_for: {%v r.KeepFiringFor %} seconds)
-                                            {% else %}
-                                            <b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds)
-                                            {% endif %}
-                                            {% else %}
-                                            <b>record:</b> {%s r.Name %}
-                                            {% endif %}
-                                            |
-                                            {%= seriesFetchedWarn(r) %}
-                                            <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
-                                        </div>
-                                        <div class="col-12">
-                                            <code><pre>{%s r.Query %}</pre></code>
-                                        </div>
-                                        <div class="col-12 mb-2">
-                                            {% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
-                                            {% for k, v := range r.Labels %}
-                                                    <span class="ms-1 badge bg-primary label">{%s k %}={%s v %}</span>
-                                            {% endfor %}
-                                        </div>
-                                        {% if r.LastError != "" %}
-                                        <div class="col-12">
-                                            <b>Error:</b>
-                                            <div class="error-cell">
-                                            {%s r.LastError %}
+            {% for _, g := range groups %}
+                <div id="group-{%s g.ID %}" class="d-flex w-100 border-0 flex-column group-items{% if g.Unhealthy > 0 %} alert-danger{% endif %}">
+                    <span class="d-flex justify-content-between">
+                        <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
+                        <span
+                            class="flex-grow-1 d-flex justify-content-end"
+                            role="button"
+                            data-bs-toggle="collapse"
+                            data-bs-target="#sub-{%s g.ID %}"
+                        >
+                            <span class="d-flex gap-2">
+                                {% if g.Unhealthy > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d g.Unhealthy %}</span> {% endif %}
+                                {% if g.NoMatch > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d g.NoMatch %}</span> {% endif %}
+                                <span class="badge bg-success" title="Number of rules with status Ok">{%d g.Healthy %}</span>
+                            </span>
+                        </span>
+                    </span>
+                    <span
+                        class="d-flex flex-column row-gap-2 mb-2"
+                        role="button"
+                        data-bs-toggle="collapse"
+                        data-bs-target="#sub-{%s g.ID %}"
+                    >
+                        <span class="fs-6 text-start w-100 fw-lighter">{%s g.File %}</span>
+                        {% if len(g.Params) > 0 %}
+                            <span class="fs-6 text-start w-100 d-flex justify-content-between fw-lighter">
+                                <span>Extra params</span>
+                                <span class="d-flex align-items-center gap-2">
+                                    {% for _, param := range g.Params %}
+                                        <span class="badge bg-primary">{%s param %}</span>
+                                    {% endfor %}
+                                </span>
+                            </span>
+                        {% endif %}
+                        {% if len(g.Headers) > 0 %}
+                            <span class="fs-6 text-start w-100 d-flex justify-content-between fw-lighter">
+                                <span>Extra headers</span>
+                                <span class="d-flex align-items-center gap-2">
+                                    {% for _, header := range g.Headers %}
+                                        <span class="badge bg-primary label">{%s header %}</span>
+                                    {% endfor %}
+                                </span>
+                            </span>
+                        {% endif %}
+                    </span>
+                    <div class="collapse sub-items" id="sub-{%s g.ID %}">
+                        <table class="table table-striped table-hover table-sm">
+                            <thead>
+                                <tr>
+                                    <th scope="col" style="width: 60%">Rule</th>
+                                    <th scope="col" style="width: 20%" class="text-center" title="How many samples were produced by the rule">Samples</th>
+                                    <th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {% for _, r := range g.Rules %}
+                                    <tr class="sub-item{% if r.LastError != "" %} alert-danger{% endif %}">
+                                        <td>
+                                            <div class="row">
+                                                <div class="col-12 mb-2">
+                                                    {% if r.Type == "alerting" %}
+                                                        {% if r.KeepFiringFor > 0 %}
+                                                            <b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds, keep_firing_for: {%v r.KeepFiringFor %} seconds)
+                                                        {% else %}
+                                                            <b>alert:</b> {%s r.Name %} (for: {%v r.Duration %} seconds)
+                                                        {% endif %}
+                                                    {% else %}
+                                                        <b>record:</b> {%s r.Name %}
+                                                    {% endif %}
+                                                    |
+                                                    {%= seriesFetchedWarn(prefix, r) %}
+                                                    <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
+                                                </div>
+                                                <div class="col-12">
+                                                    <code><pre>{%s r.Query %}</pre></code>
+                                                </div>
+                                                <div class="col-12 mb-2">
+                                                    {% if len(r.Labels) > 0 %} <b>Labels:</b>{% endif %}
+                                                    {% for k, v := range r.Labels %}
+                                                        <span class="ms-1 badge bg-primary label">{%s k %}={%s v %}</span>
+                                                    {% endfor %}
+                                                </div>
+                                                {% if r.LastError != "" %}
+                                                    <div class="col-12">
+                                                        <b>Error:</b>
+                                                        <div class="error-cell">
+                                                            {%s r.LastError %}
+                                                        </div>
+                                                    </div>
+                                                {% endif %}
                                             </div>
-                                        </div>
-                                        {% endif %}
-                                    </div>
-                                </td>
-                                <td class="text-center">{%d r.LastSamples %}</td>
-                                <td class="text-center">{%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago</td>
-                            </tr>
-                        {% endfor %}
-                     </tbody>
-                    </table>
+                                        </td>
+                                        <td class="text-center">{%d r.LastSamples %}</td>
+                                        <td class="text-center">{%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago</td>
+                                    </tr>
+                                {% endfor %}
+                            </tbody>
+                        </table>
+                    </div>
                 </div>
             {% endfor %}
         {% else %}
@@ -174,169 +218,164 @@ btn-primary
                 <p>No groups...</p>
             </div>
         {% endif %}
-
     {%= tpl.Footer(r) %}
-
 {% endfunc %}
 
 
 {% func ListAlerts(r *http.Request, groupAlerts []groupAlerts) %}
     {%code prefix := vmalertutil.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "Alerts", getLastConfigError()) %}
+    {%= Controls(prefix, "", "", nil, nil, true) %}
     {% if len(groupAlerts) > 0 %}
-         <div class="btn-toolbar mb-3" role="toolbar">
-              <div>
-                <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-                 <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
-              </div>
-              <div class="col-md-4 col-lg-5">
-                <div class="px-3 input-group">
-                  <div class="input-group-prepend">
-                    <span class="input-group-text">
-                      <svg fill="#000000" height="25px" width="20px" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 490.4 490.4" xml:space="preserve"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path d="M484.1,454.796l-110.5-110.6c29.8-36.3,47.6-82.8,47.6-133.4c0-116.3-94.3-210.6-210.6-210.6S0,94.496,0,210.796 s94.3,210.6,210.6,210.6c50.8,0,97.4-18,133.8-48l110.5,110.5c12.9,11.8,25,4.2,29.2,0C492.5,475.596,492.5,463.096,484.1,454.796z M41.1,210.796c0-93.6,75.9-169.5,169.5-169.5s169.6,75.9,169.6,169.5s-75.9,169.5-169.5,169.5S41.1,304.396,41.1,210.796z"></path> </g> </g></svg>
-                    </span>
-                  </div>
-                  <input id="search" placeholder="Filter by group, rule or labels" type="text" class="form-control"/>
-                </div>
-              </div>
-          </div>
          {% for _, ga := range groupAlerts %}
-            {%code g := ga.Group %}
-            <div class="group-heading alert-danger" data-bs-target="rules-{%s g.ID %}" data-group-name="{%s g.Name %}">
-                <span class="anchor" id="group-{%s g.ID %}"></span>
-                <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
-                <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
-                <br>
-                <p class="fs-6 fw-lighter">{%s g.File %}</p>
-            </div>
-            {%code
-                var keys []string
-                alertsByRule := make(map[string][]*apiAlert)
-                for _, alert := range ga.Alerts {
-                    if len(alertsByRule[alert.RuleID]) < 1 {
-                        keys = append(keys, alert.RuleID)
-                    }
-                    alertsByRule[alert.RuleID] = append(alertsByRule[alert.RuleID], alert)
-                }
-                sort.Strings(keys)
-            %}
-            <div class="collapse rule-table" id="rules-{%s g.ID %}">
-                {% for _, ruleID := range keys %}
-                    {%code
-                        defaultAR := alertsByRule[ruleID][0]
-                        var labelKeys []string
-                        for k := range defaultAR.Labels {
-                            labelKeys = append(labelKeys, k)
-                        }
-                        sort.Strings(labelKeys)
-                    %}
-                    <br>
-                    <div class="rule" data-rule-name="{%s defaultAR.Name %}" data-bs-target="{%s g.ID %}">
-                      <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
-                       | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
-                      <br>
-                      <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
-                      <table class="table table-striped table-hover table-sm">
-                          <thead>
-                              <tr>
-                                  <th scope="col">Labels</th>
-                                  <th scope="col">State</th>
-                                  <th scope="col">Active at</th>
-                                  <th scope="col">Value</th>
-                                  <th scope="col">Link</th>
-                              </tr>
-                          </thead>
-                          <tbody>
-                          {% for _, ar := range alertsByRule[ruleID] %}
-                              <tr>
-                                  <td>
-                                      {% for _, k := range labelKeys %}
-                                          <span class="ms-1 badge bg-primary label">{%s k %}={%s ar.Labels[k] %}</span>
-                                      {% endfor %}
-                                  </td>
-                                  <td>{%= badgeState(ar.State) %}</td>
-                                  <td>
-                                      {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
-                                      {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
-                                      {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
-                                  </td>
-                                  <td>{%s ar.Value %}</td>
-                                  <td>
-                                      <a href="{%s prefix+ar.WebLink() %}">Details</a>
-                                  </td>
-                              </tr>
-                          {% endfor %}
-                       </tbody>
-                      </table>
-                    </div>
-                {% endfor %}
-            </div>
-        {% endfor %}
-
-    {% else %}
-        <div>
-            <p>No active alerts...</p>
-        </div>
-    {% endif %}
-
-    {%= tpl.Footer(r) %}
-
+             {%code
+                 g := ga.Group
+                 var keys []string
+                 alertsByRule := make(map[string][]*apiAlert)
+                 for _, alert := range ga.Alerts {
+                     if len(alertsByRule[alert.RuleID]) < 1 {
+                         keys = append(keys, alert.RuleID)
+                     }
+                     alertsByRule[alert.RuleID] = append(alertsByRule[alert.RuleID], alert)
+                 }
+                 sort.Strings(keys)
+             %}
+             <div class="d-flex w-100 flex-column group-items alert-danger">
+                 <span id="group-{%s g.ID %}" class="d-flex justify-content-between">
+                     <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
+                     <span
+                         class="flex-grow-1 d-flex justify-content-end"
+                         role="button"
+                         data-bs-toggle="collapse"
+                         data-bs-target="#sub-{%s g.ID %}"
+                     >
+                         <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
+                     </span>
+                 </span>
+                 <span>
+                     <span
+                         class="fs-6 text-start w-100 fw-lighter"
+                         role="button" 
+                         data-bs-toggle="collapse"
+                         data-bs-target="#sub-{%s g.ID %}"
+                     >{%s g.File %}</span>
+                 </span>
+                 <div class="collapse sub-items" id="sub-{%s g.ID %}">
+                     {% for _, ruleID := range keys %}
+                         {%code
+                             defaultAR := alertsByRule[ruleID][0]
+                             var labelKeys []string
+                             for k := range defaultAR.Labels {
+                                 labelKeys = append(labelKeys, k)
+                             }
+                             sort.Strings(labelKeys)
+                         %}
+                         <br>
+                         <div class="sub-item">
+                             <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
+                             | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
+                             <br>
+                             <b>expr:</b><code><pre>{%s defaultAR.Expression %}</pre></code>
+                             <table class="table table-striped table-hover table-sm">
+                                 <thead>
+                                     <tr>
+                                         <th scope="col">Labels</th>
+                                         <th scope="col">State</th>
+                                         <th scope="col">Active at</th>
+                                         <th scope="col">Value</th>
+                                         <th scope="col">Link</th>
+                                     </tr>
+                                 </thead>
+                                 <tbody>
+                                     {% for _, ar := range alertsByRule[ruleID] %}
+                                         <tr>
+                                             <td>
+                                                 {% for _, k := range labelKeys %}
+                                                     <span class="ms-1 badge bg-primary label">{%s k %}={%s ar.Labels[k] %}</span>
+                                                 {% endfor %}
+                                             </td>
+                                             <td>{%= badgeState(ar.State) %}</td>
+                                             <td>
+                                                 {%s ar.ActiveAt.Format("2006-01-02T15:04:05Z07:00") %}
+                                                 {% if ar.Restored %}{%= badgeRestored() %}{% endif %}
+                                                 {% if ar.Stabilizing %}{%= badgeStabilizing() %}{% endif %}
+                                             </td>
+                                             <td>{%s ar.Value %}</td>
+                                             <td><a href="{%s prefix+ar.WebLink() %}">Details</a></td>
+                                         </tr>
+                                     {% endfor %}
+                                 </tbody>
+                             </table>
+                         </div>
+                     {% endfor %}
+                 </div>
+             </div>
+         {% endfor %}
+     {% else %}
+         <div>
+             <p>No active alerts...</p>
+         </div>
+     {% endif %}
+     {%= tpl.Footer(r) %}
 {% endfunc %}
 
 {% func ListTargets(r *http.Request, targets map[notifier.TargetType][]notifier.Target) %}
+    {%code prefix := vmalertutil.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "Notifiers", getLastConfigError()) %}
+    {%= Controls(prefix, "", "", nil, nil, false) %}
     {% if len(targets) > 0 %}
-         <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
-         <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
-
-         {%code
-             var keys []string
-             for key := range targets {
-                 keys = append(keys, string(key))
-             }
-             sort.Strings(keys)
-         %}
-
-         {% for i := range keys  %}
-           {%code typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
+        {%code
+            var keys []string
+            for key := range targets {
+                keys = append(keys, string(key))
+            }
+            sort.Strings(keys)
+        %}
+        {% for i := range keys %}
+            {%code
+                typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
                 count := len(ns)
             %}
-           <div class="group-heading" data-bs-target="notifiers-{%s typeK %}">
-             <span class="anchor" id="group-{%s typeK %}"></span>
-             <a href="#group-{%s typeK %}">{%s typeK %} ({%d count %})</a>
-         </div>
-         <div class="collapse show" id="notifiers-{%s typeK %}">
-             <table class="table table-striped table-hover table-sm">
-                 <thead>
-                     <tr>
-                         <th scope="col">Labels</th>
-                         <th scope="col">Address</th>
-                     </tr>
-                 </thead>
-                 <tbody>
-                 {% for _, n := range ns %}
-                     <tr>
-                         <td>
-                              {% for _, l := range n.Labels.GetLabels() %}
-                                      <span class="ms-1 badge bg-primary">{%s l.Name %}={%s l.Value %}</span>
-                              {% endfor %}
-                          </td>
-                         <td>{%s n.Notifier.Addr() %}</td>
-                     </tr>
-                 {% endfor %}
-              </tbody>
-             </table>
-         </div>
-     {% endfor %}
-
+            <div class="d-flex w-100 flex-column group-items">
+                <span class="d-flex justify-content-between" id="group-{%s typeK %}">
+                    <a href="#group-{%s typeK %}">{%s typeK %} ({%d count %})</a>
+                    <span
+                        class="flex-grow-1"
+                        role="button"
+                        data-bs-toggle="collapse"
+                        data-bs-target="#sub-{%s typeK %}"
+                    ></span>
+                </span>
+                <div id="sub-{%s typeK %}" class="collapse show sub-items">
+                    <table class="table table-striped table-hover table-sm">
+                        <thead>
+                            <tr class="sub-item">
+                                <th scope="col">Labels</th>
+                                <th scope="col">Address</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for _, n := range ns %}
+                                <tr>
+                                    <td>
+                                        {% for _, l := range n.Labels.GetLabels() %}
+                                            <span class="ms-1 badge bg-primary">{%s l.Name %}={%s l.Value %}</span>
+                                        {% endfor %}
+                                    </td>
+                                    <td>{%s n.Notifier.Addr() %}</td>
+                                </tr>
+                            {% endfor %}
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        {% endfor %}
     {% else %}
         <div>
             <p>No targets...</p>
         </div>
     {% endif %}
-
     {%= tpl.Footer(r) %}
-
 {% endfunc %}
 
 {% func Alert(r *http.Request, alert *apiAlert) %}
@@ -348,7 +387,6 @@ btn-primary
             labelKeys = append(labelKeys, k)
         }
         sort.Strings(labelKeys)
-
         var annotationKeys []string
         for k := range alert.Annotations {
             annotationKeys = append(annotationKeys, k)
@@ -611,15 +649,15 @@ btn-primary
 <span class="badge bg-warning text-dark" title="This firing state is kept because of `keep_firing_for`">stabilizing</span>
 {% endfunc %}
 
-{% func seriesFetchedWarn(r apiRule) %}
+{% func seriesFetchedWarn(prefix string, r apiRule) %}
 {% if isNoMatch(r) %}
-<svg xmlns="http://www.w3.org/2000/svg"
+<svg
     data-bs-toggle="tooltip"
     title="No match! This rule's last evaluation hasn't selected any time series from the datasource.
     It might be that either this data is missing in the datasource or there is a typo in rule's expression.
     See more in Details."
-    width="18" height="18" fill="currentColor" class="bi bi-exclamation-triangle-fill flex-shrink-0 me-2" viewBox="0 0 16 16" role="img" aria-label="Warning:">
-       <path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
+    width="18" height="18" fill="currentColor" class="bi bi-exclamation-triangle-fill flex-shrink-0 me-2" role="img" aria-label="Warning:">
+       <use href="{%s prefix %}static/icons/icons.svg#exclamation"/>
 </svg>
 {% endif %}
 {% endfunc %}

app/vmalert/web_types.go

@@ -98,6 +98,12 @@ type apiGroup struct {
 	EvalOffset float64 `json:"eval_offset,omitempty"`
 	// EvalDelay will adjust the `time` parameter of rule evaluation requests to compensate intentional query delay from datasource.
 	EvalDelay float64 `json:"eval_delay,omitempty"`
+	// Unhealthy unhealthy rules count
+	Unhealthy int
+	// Healthy passing rules count
+	Healthy int
+	// NoMatch not matching rules count
+	NoMatch int
 }
 
 // groupAlerts represents a group of alerts for WEB view
@@ -325,8 +331,7 @@ func groupToAPI(g *rule.Group) apiGroup {
 	g = g.DeepCopy()
 	ag := apiGroup{
 		// encode as string to avoid rounding
-		ID: fmt.Sprintf("%d", g.GetID()),
-
+		ID:              strconv.FormatUint(g.GetID(), 10),
 		Name:            g.Name,
 		Type:            g.Type.String(),
 		File:            g.File,
@@ -336,8 +341,7 @@ func groupToAPI(g *rule.Group) apiGroup {
 		Params:          urlValuesToStrings(g.Params),
 		Headers:         headersToStrings(g.Headers),
 		NotifierHeaders: headersToStrings(g.NotifierHeaders),
-
-		Labels: g.Labels,
+		Labels:          g.Labels,
 	}
 	if g.EvalOffset != nil {
 		ag.EvalOffset = g.EvalOffset.Seconds()

app/vmalert/web_types_test.go

@@ -55,7 +55,7 @@ func TestRecordingToApi(t *testing.T) {
 	}
 }
 
-func TestUrlValuesToStrings(t *testing.T) {
+func TestURLValuesToStrings(t *testing.T) {
 	mapQueryParams := map[string][]string{
 		"param1": {"param1"},
 		"param2": {"anotherparam"},

app/vmauth/README.md

@@ -1,3 +1,3 @@
-See vmauth docs [here](https://docs.victoriametrics.com/vmauth/).
+See vmauth docs [here](https://docs.victoriametrics.com/victoriametrics/vmauth/).
 
-vmauth docs can be edited at [docs/vmauth.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmauth.md).
+vmauth docs can be edited at [docs/vmauth.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmauth.md).

app/vmauth/auth_config.go

@@ -34,15 +34,15 @@ import (
 
 var (
 	authConfigPath = flag.String("auth.config", "", "Path to auth config. It can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmauth/ for details on the format of this auth config")
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/ for details on the format of this auth config")
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "interval for config file re-read. "+
 		"Zero value disables config re-reading. By default, refreshing is disabled, send SIGHUP for config refresh.")
 	defaultRetryStatusCodes = flagutil.NewArrayInt("retryStatusCodes", 0, "Comma-separated list of default HTTP response status codes when vmauth re-tries the request on other backends. "+
-		"See https://docs.victoriametrics.com/vmauth/#load-balancing for details")
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#load-balancing for details")
 	defaultLoadBalancingPolicy = flag.String("loadBalancingPolicy", "least_loaded", "The default load balancing policy to use for backend urls specified inside url_prefix section. "+
-		"Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/vmauth/#load-balancing")
+		"Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/victoriametrics/vmauth/#load-balancing")
 	discoverBackendIPsGlobal = flag.Bool("discoverBackendIPs", false, "Whether to discover backend IPs via periodic DNS queries to hostnames specified in url_prefix. "+
-		"This may be useful when url_prefix points to a hostname with dynamically scaled instances behind it. See https://docs.victoriametrics.com/vmauth/#discovering-backend-ips")
+		"This may be useful when url_prefix points to a hostname with dynamically scaled instances behind it. See https://docs.victoriametrics.com/victoriametrics/vmauth/#discovering-backend-ips")
 	discoverBackendIPsInterval = flag.Duration("discoverBackendIPsInterval", 10*time.Second, "The interval for re-discovering backend IPs if -discoverBackendIPs command-line flag is set. "+
 		"Too low value may lead to DNS errors")
 	httpAuthHeader = flagutil.NewArrayString("httpAuthHeader", "HTTP request header to use for obtaining authorization tokens. By default auth tokens are read from Authorization request header")

app/vmauth/main.go

@@ -57,15 +57,15 @@ var (
 	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
 		"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
 	backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
-		"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
-		"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	backendTLSCertFile = flag.String("backend.TLSCertFile", "", "Optional path to TLS client certificate file, which must be sent to HTTPS backend. "+
-		"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	backendTLSKeyFile = flag.String("backend.TLSKeyFile", "", "Optional path to TLS client key file, which must be sent to HTTPS backend. "+
-		"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	backendTLSServerName = flag.String("backend.TLSServerName", "", "Optional TLS ServerName, which must be sent to HTTPS backend. "+
-		"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	dryRun                   = flag.Bool("dryRun", false, "Whether to check only config files without running vmauth. The auth configuration file is validated. The -auth.config flag must be specified.")
 	removeXFFHTTPHeaderValue = flag.Bool(`removeXFFHTTPHeaderValue`, false, "Whether to remove the X-Forwarded-For HTTP header value from client requests before forwarding them to the backend. "+
 		"Recommended when vmauth is exposed to the internet.")
@@ -96,20 +96,23 @@ func main() {
 	logger.Infof("starting vmauth at %q...", listenAddrs)
 	startTime := time.Now()
 	initAuthConfig()
+
 	disableInternalRoutes := len(*httpInternalListenAddr) > 0
 	rh := requestHandlerWithInternalRoutes
 	if disableInternalRoutes {
 		rh = requestHandler
 	}
 
-	serveOpts := httpserver.ServeOptions{
-		UseProxyProtocol:     useProxyProtocol,
+	go httpserver.Serve(listenAddrs, rh, httpserver.ServeOptions{
+		UseProxyProtocol: useProxyProtocol,
+		// built-in routes will be exposed at *httpInternalListenAddr
 		DisableBuiltinRoutes: disableInternalRoutes,
-	}
-	go httpserver.ServeWithOpts(listenAddrs, rh, serveOpts)
+	})
 
 	if len(*httpInternalListenAddr) > 0 {
-		go httpserver.Serve(*httpInternalListenAddr, nil, internalRequestHandler)
+		go httpserver.Serve(*httpInternalListenAddr, internalRequestHandler, httpserver.ServeOptions{
+			UseProxyProtocol: useProxyProtocol,
+		})
 	}
 	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
 
@@ -537,7 +540,7 @@ func usage() {
 	const s = `
 vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
 
-See the docs at https://docs.victoriametrics.com/vmauth/ .
+See the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/ .
 `
 	flagutil.Usage(s)
 }

app/vmauth/multiarch/Dockerfile

@@ -9,4 +9,5 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8427
 ENTRYPOINT ["/vmauth-prod"]
 ARG TARGETARCH
-COPY vmauth-linux-${TARGETARCH}-prod ./vmauth-prod
+ARG BINARY_SUFFIX=non-existing
+COPY vmauth-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmauth-prod

app/vmbackup/README.md

@@ -1,3 +1,3 @@
-See vmbackup docs [here](https://docs.victoriametrics.com/vmbackup/).
+See vmbackup docs [here](https://docs.victoriametrics.com/victoriametrics/vmbackup/).
 
-vmbackup docs can be edited at [docs/vmbackup.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmbackup.md).
+vmbackup docs can be edited at [docs/vmbackup.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmbackup.md).

app/vmbackup/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"net/url"
@@ -18,6 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot/snapshotutil"
@@ -26,7 +28,7 @@ import (
 var (
 	httpListenAddr    = flag.String("httpListenAddr", ":8420", "TCP address for exporting metrics at /metrics page")
 	storageDataPath   = flag.String("storageDataPath", "victoria-metrics-data", "Path to VictoriaMetrics data. Must match -storageDataPath from VictoriaMetrics or vmstorage")
-	snapshotName      = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set")
+	snapshotName      = flag.String("snapshotName", "", "Name for the snapshot to backup. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots. There is no need in setting -snapshotName if -snapshot.createURL is set")
 	snapshotCreateURL = flag.String("snapshot.createURL", "", "VictoriaMetrics create snapshot url. When this is given a snapshot will automatically be created during backup. "+
 		"Example: http://victoriametrics:8428/snapshot/create . There is no need in setting -snapshotName if -snapshot.createURL is set")
 	snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. "+
@@ -95,10 +97,17 @@ func main() {
 	}
 
 	listenAddrs := []string{*httpListenAddr}
-	go httpserver.Serve(listenAddrs, nil, nil)
+	go httpserver.Serve(listenAddrs, nil, httpserver.ServeOptions{})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		procutil.WaitForSigterm()
+		logger.Infof("received stop signal, canceling backup operation")
+		cancel()
+	}()
 
 	pushmetrics.Init()
-	err := makeBackup()
+	err := makeBackup(ctx)
 	deleteSnapshot()
 	if err != nil {
 		logger.Fatalf("cannot create backup: %s", err)
@@ -113,14 +122,14 @@ func main() {
 	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
 }
 
-func makeBackup() error {
-	dstFS, err := newDstFS()
+func makeBackup(ctx context.Context) error {
+	dstFS, err := newDstFS(ctx)
 	if err != nil {
 		return err
 	}
 	if *snapshotName == "" {
 		// Make server-side copy from -origin to -dst
-		originFS, err := newRemoteOriginFS()
+		originFS, err := newRemoteOriginFS(ctx)
 		if err != nil {
 			return err
 		}
@@ -139,7 +148,7 @@ func makeBackup() error {
 		if err != nil {
 			return err
 		}
-		originFS, err := newOriginFS()
+		originFS, err := newOriginFS(ctx)
 		if err != nil {
 			return err
 		}
@@ -164,7 +173,7 @@ func usage() {
 vmbackup performs backups for VictoriaMetrics data from instant snapshots to gcs, s3, azblob
 or local filesystem. Backed up data can be restored with vmrestore.
 
-See the docs at https://docs.victoriametrics.com/vmbackup/ .
+See the docs at https://docs.victoriametrics.com/victoriametrics/vmbackup/ .
 `
 	flagutil.Usage(s)
 }
@@ -199,8 +208,8 @@ func newSrcFS() (*fslocal.FS, error) {
 	return fs, nil
 }
 
-func newDstFS() (common.RemoteFS, error) {
-	fs, err := actions.NewRemoteFS(*dst)
+func newDstFS(ctx context.Context) (common.RemoteFS, error) {
+	fs, err := actions.NewRemoteFS(ctx, *dst)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-dst`=%q: %w", *dst, err)
 	}
@@ -239,22 +248,22 @@ func hasFilepathPrefix(path, prefix string) bool {
 	return true
 }
 
-func newOriginFS() (common.OriginFS, error) {
+func newOriginFS(ctx context.Context) (common.OriginFS, error) {
 	if len(*origin) == 0 {
 		return &fsnil.FS{}, nil
 	}
-	fs, err := actions.NewRemoteFS(*origin)
+	fs, err := actions.NewRemoteFS(ctx, *origin)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
 	}
 	return fs, nil
 }
 
-func newRemoteOriginFS() (common.RemoteFS, error) {
+func newRemoteOriginFS(ctx context.Context) (common.RemoteFS, error) {
 	if len(*origin) == 0 {
 		return nil, fmt.Errorf("-origin cannot be empty when -snapshotName and -snapshot.createURL aren't set")
 	}
-	fs, err := actions.NewRemoteFS(*origin)
+	fs, err := actions.NewRemoteFS(ctx, *origin)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
 	}

app/vmbackup/multiarch/Dockerfile

@@ -8,4 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENTRYPOINT ["/vmbackup-prod"]
 ARG TARGETARCH
-COPY vmbackup-linux-${TARGETARCH}-prod ./vmbackup-prod
+ARG BINARY_SUFFIX=non-existing
+COPY vmbackup-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmbackup-prod

app/vmbackupmanager/README.md

@@ -1,3 +1,3 @@
-See vmbackupmanager docs [here](https://docs.victoriametrics.com/vmbackupmanager/).
+See vmbackupmanager docs [here](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/).
 
-vmbackupmanager docs can be edited at [docs/vmbackupmanager.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmbackupmanager.md).
+vmbackupmanager docs can be edited at [docs/vmbackupmanager.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmbackupmanager.md).

app/vmctl/README.md

@@ -1,3 +1,3 @@
-See vmctl docs [here](https://docs.victoriametrics.com/vmctl/).
+See vmctl docs [here](https://docs.victoriametrics.com/victoriametrics/vmctl/).
 
-vmctl docs can be edited at [docs/vmctl.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmctl.md).
+vmctl docs can be edited at [docs/vmctl.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmctl.md).