app/vminsert: moves replication at datapointWrite level

* It allows to tolerate slow storage node for replicationFactor > 1 and disableRerouting=false
* It allows to have partial replication with replicationFactor > and dropSamplesOnOverload=true
* It increases cpu and memory usage by 30-50% per replica

Signed-off-by: f41gh7 <nik@victoriametrics.com>

.gitignore

@@ -7,6 +7,7 @@
 .vscode
 *.test
 *.swp
+/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data

Makefile

@@ -234,7 +234,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.1
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`
@@ -253,33 +253,3 @@ install-wwhrd:
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
-
-copy-docs:
-# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
-# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
-	echo "---" > ${DST}
-	@if [ ${ORDER} -ne 0 ]; then \
-		echo "sort: ${ORDER}" >> ${DST}; \
-		echo "weight: ${ORDER}" >> ${DST}; \
-		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
-	fi
-
-	echo "title: ${TITLE}" >> ${DST}
-	@if [ ${OLD_URL} ]; then \
-		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
-	fi
-	echo "---" >> ${DST}
-	cat ${SRC} >> ${DST}
-	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
-	sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"/' ${DST}
-	rm -rf docs/*.tmp
-
-# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
-# For ORDER=0 it adds no order tag/weight tag.
-# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
-# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
-# Images starting with <img src="docs/ are replaced with <img src="
-# Cluster docs are supposed to be ordered as 2nd.
-# The rest of docs is ordered manually.
-docs-sync:
-	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md OLD_URL='/Cluster-VictoriaMetrics.html' ORDER=2 TITLE='Cluster version' $(MAKE) copy-docs

SECURITY.md

@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
+| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to security@victoriametrics.com
+Please report any security issues to <security@victoriametrics.com>

app/vlinsert/elasticsearch/elasticsearch.go

@@ -57,6 +57,12 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
+	if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
+		// Return fake response for Logstash APIs requests.
+		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
 	switch path {
 	case "/":
 		switch r.Method {

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
     "main.css": "./static/css/main.1041c3d4.css",
-    "main.js": "./static/js/main.62a82f4a.js",
+    "main.js": "./static/js/main.8451d9b9.js",
     "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.d46c42c8e891f06298c4.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
     "static/css/main.1041c3d4.css",
-    "static/js/main.62a82f4a.js"
+    "static/js/main.8451d9b9.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.62a82f4a.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8451d9b9.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/media/MetricsQL.d46c42c8e891f06298c4.md

@@ -10,9 +10,6 @@ aliases:
 - /ExtendedPromQL.html
 - /MetricsQL.html
 ---
-
-# MetricsQL
-
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -220,9 +217,11 @@ See also [descent_over_time](#descent_over_time).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [median_over_time](#median_over_time).
+See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
 
 #### changes
 
@@ -262,6 +261,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
@@ -272,6 +273,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
@@ -282,6 +285,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
@@ -292,7 +297,9 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### count_over_time
 
@@ -313,6 +320,8 @@ The results are calculated independently per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
@@ -432,6 +441,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### histogram_over_time
 
 `histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -443,11 +454,15 @@ For example, the following query calculates median temperature by country over t
 
 `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### hoeffding_bound_lower
 
 `hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 
 #### hoeffding_bound_upper
@@ -455,6 +470,8 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 `hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 
 #### holt_winters
@@ -462,8 +479,9 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
 (aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
-Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
-returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
+Both `sf` and `tf` must be in the range `[0...1]`.
+
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
 
 This function is supported by PromQL.
 
@@ -495,13 +513,14 @@ See also [deriv](#deriv).
 
 `increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -510,12 +529,13 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
 
 `increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 See also [increase_pure](#increase_pure) and [increase](#increase).
 
 #### increase_pure
@@ -524,6 +544,10 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
 of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
 while [increase](#increase) ignores the first value in a series if it is too big.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [increase](#increas) and [increase_prometheus](#increase_prometheus).
+
 #### increases_over_time
 
 `increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -540,15 +564,18 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### irate
 
 `irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
 the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -587,6 +614,8 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 #### max_over_time
@@ -594,9 +623,11 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmax_over_time](#tmax_over_time).
+See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
 
 #### median_over_time
 
@@ -604,6 +635,8 @@ See also [tmax_over_time](#tmax_over_time).
 on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [avg_over_time](#avg_over_time).
 
 #### min_over_time
@@ -611,9 +644,11 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmin_over_time](#tmin_over_time).
+See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
 
 #### mode_over_time
 
@@ -622,6 +657,8 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values are discrete.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### outlier_iqr_over_time
 
 `outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -632,6 +669,8 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
 For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [outliers_iqr](#outliers_iqr).
 
 #### predict_linear
@@ -659,6 +698,8 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The `phi` value must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [quantiles_over_time](#quantiles_over_time).
@@ -670,6 +711,8 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [quantile_over_time](#quantile_over_time).
 
 #### range_over_time
@@ -680,11 +723,12 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rate
 
 `rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -703,6 +747,8 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
 on the given lookbehind window `d`. The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### resets
@@ -710,10 +756,11 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 `resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
 of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 #### rollup
@@ -725,6 +772,10 @@ These values are calculated individually per each time series returned from the
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [rollup_rate](#rollup_rate).
+
 #### rollup_candlestick
 
 `rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -736,6 +787,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rollup_delta
 
 `rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -762,6 +815,8 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_increase
 
 `rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -774,12 +829,17 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_rate
 
 `rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
 for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
 and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
+The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
 when to use `rollup_rate()`.
@@ -787,10 +847,12 @@ when to use `rollup_rate()`.
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
+
 #### rollup_scrape_interval
 
 `rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -824,6 +886,8 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
@@ -838,6 +902,8 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
@@ -849,6 +915,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time).
 
 #### stale_samples_over_time
@@ -866,6 +934,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stdvar_over_time](#stdvar_over_time).
@@ -877,6 +947,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stddev_over_time](#stddev_over_time).
@@ -888,6 +960,8 @@ values equal to `eq` on the given lookbehind window `d` per each time series ret
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### sum_gt_over_time
@@ -897,6 +971,8 @@ values bigger than `gt` on the given lookbehind window `d` per each time series
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### sum_le_over_time
@@ -906,6 +982,8 @@ values smaller or equal to `le` on the given lookbehind window `d` per each time
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
@@ -915,6 +993,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 #### sum2_over_time
@@ -924,6 +1004,8 @@ values on the given lookbehind window `d` per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### timestamp
 
 `timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1001,6 +1083,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 
@@ -2230,7 +2314,8 @@ Any [rollup function](#rollup-functions) for something other than [series select
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
 For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
-This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
 
 VictoriaMetrics performs subqueries in the following way:
 
@@ -2263,5 +2348,6 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`. 
-  This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
+  This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).

app/vmagent/main.go

@@ -318,6 +318,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
@@ -564,6 +568,10 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -674,7 +682,8 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
 
 	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

app/vmagent/remotewrite/remotewrite.go

@@ -441,7 +441,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 {
+	if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
 		rctx = getRelabelCtx()
 		defer putRelabelCtx(rctx)
 	}

app/vmagent/remotewrite/streamaggr.go

@@ -50,7 +50,7 @@ var (
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
 		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
+	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
 		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
 		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
@@ -133,7 +133,7 @@ func initStreamAggrConfigGlobal() {
 	} else {
 		dedupInterval := streamAggrGlobalDedupInterval.Duration()
 		if dedupInterval > 0 {
-			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrDropInputLabels, "dedup-global")
+			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 		}
 	}
 }
@@ -202,6 +202,7 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 		DropInputLabels:      *streamAggrGlobalDropInputLabels,
 		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
 		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
+		KeepInput:            *streamAggrGlobalKeepInput,
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
@@ -229,7 +230,8 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag
 		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
 		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
+		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
+		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)

app/vmalert/datasource/init.go

@@ -17,7 +17,7 @@ import (
 
 var (
 	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
-		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"See also -remoteRead.disablePathAppend and -datasource.showURL")
 	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
 	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
@@ -99,7 +99,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
 	}
 	tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
 	tr.DisableKeepAlives = *disableKeepAlive

app/vmalert/datasource/vm_prom_api_timing_test.go

@@ -13,7 +13,7 @@ func BenchmarkMetrics(b *testing.B) {
 
 	var pi promInstant
 	if err := pi.Unmarshal(payload); err != nil {
-		b.Fatalf(err.Error())
+		b.Fatal(err.Error())
 	}
 	b.Run("Instant", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {

app/vmalert/notifier/alertmanager.go

@@ -76,9 +76,6 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	for key, value := range headers {
-		req.Header.Set(key, value)
-	}
 
 	if am.timeout > 0 {
 		var cancel context.CancelFunc
@@ -94,6 +91,11 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 			return err
 		}
 	}
+	// external headers have higher priority
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
 	resp, err := am.client.Do(req)
 	if err != nil {
 		return err
@@ -130,7 +132,8 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}
 	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for alertmanager URL=%q: %w", alertManagerURL, err)
+
 	}
 
 	ba := new(promauth.BasicAuthConfig)
@@ -145,7 +148,9 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	aCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
-		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
+		utils.WithHeaders(strings.Join(authCfg.Headers, "^^")),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/notifier/alertmanager_test.go

@@ -3,6 +3,7 @@ package notifier
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -48,6 +49,9 @@ func TestAlertManager_Send(t *testing.T) {
 			conn, _, _ := w.(http.Hijacker).Hijack()
 			_ = conn.Close()
 		case 1:
+			if r.Header.Get(headerKey) != headerValue {
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+			}
 			w.WriteHeader(500)
 		case 2:
 			var a []struct {
@@ -72,6 +76,9 @@ func TestAlertManager_Send(t *testing.T) {
 			if a[0].EndAt.IsZero() {
 				t.Fatalf("expected non-zero end time")
 			}
+			if r.Header.Get(headerKey) != "bar" {
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+			}
 		case 3:
 			if r.Header.Get(headerKey) != headerValue {
 				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
@@ -86,6 +93,7 @@ func TestAlertManager_Send(t *testing.T) {
 			Username: baUser,
 			Password: promauth.NewSecret(baPass),
 		},
+		Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
@@ -105,7 +113,7 @@ func TestAlertManager_Send(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, nil); err != nil {
+	}}, map[string]string{headerKey: "bar"}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 	if c != 2 {

app/vmalert/notifier/init.go

@@ -25,6 +25,9 @@ var (
 		"Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). "+
 		"-notifier.url, -notifier.config and -notifier.blackhole are mutually exclusive.")
 
+	headers = flagutil.NewArrayString("notifier.headers", "Optional HTTP headers to send with each request to the corresponding -notifier.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
+		"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
@@ -171,6 +174,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
 				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
 			},
+			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
 		addr = strings.TrimSuffix(addr, "/")

app/vmalert/remoteread/init.go

@@ -17,7 +17,7 @@ var (
 	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
 		"Remote read is used to restore alerts state."+
 		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
 
 	showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
@@ -68,7 +68,7 @@ func Init() (datasource.QuerierBuilder, error) {
 	}
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
 	tr.DialContext = netutil.NewStatDialFunc("vmalert_remoteread")

app/vmalert/remotewrite/debug_client.go

@@ -32,7 +32,7 @@ func NewDebugClient() (*DebugClient, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
 	}
 	c := &DebugClient{
 		c: &http.Client{

app/vmalert/remotewrite/init.go

@@ -15,7 +15,7 @@ import (
 var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. "+
-		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -72,7 +72,7 @@ func Init(ctx context.Context) (*Client, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
 	}
 	t.IdleConnTimeout = *idleConnectionTimeout
 	t.DialContext = netutil.NewStatDialFunc("vmalert_remotewrite")

app/vmalert/rule/alerting.go

@@ -441,9 +441,6 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			}
 			a.Value = m.Values[0]
 			a.Annotations = annotations
-			if err != nil {
-				return nil, err
-			}
 			a.KeepFiringSince = time.Time{}
 			continue
 		}

app/vmauth/auth_config.go

@@ -84,9 +84,6 @@ type UserInfo struct {
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
-	// Whether to use backend host header in requests to backend.
-	useBackendHostHeader bool
-
 	rt http.RoundTripper
 
 	requests         *metrics.Counter
@@ -96,8 +93,9 @@ type UserInfo struct {
 
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
-	RequestHeaders  []*Header `yaml:"headers,omitempty"`
-	ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
+	RequestHeaders   []*Header `yaml:"headers,omitempty"`
+	ResponseHeaders  []*Header `yaml:"response_headers,omitempty"`
+	KeepOriginalHost *bool     `yaml:"keep_original_host,omitempty"`
 }
 
 func (ui *UserInfo) beginConcurrencyLimit() error {
@@ -152,15 +150,6 @@ func (h *Header) MarshalYAML() (any, error) {
 	return h.sOriginal, nil
 }
 
-func hasEmptyHostHeader(headers []*Header) bool {
-	for _, h := range headers {
-		if h.Name == "Host" && h.Value == "" {
-			return true
-		}
-	}
-	return false
-}
-
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
 	// SrcPaths is an optional list of regular expressions, which must match the request path.
@@ -472,7 +461,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	}
 
 	// Slow path - select other backend urls.
-	n := atomicCounter.Add(1)
+	n := atomicCounter.Add(1) - 1
 
 	for i := uint32(0); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
@@ -608,7 +597,7 @@ func initAuthConfig() {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
 	sighupCh := procutil.NewSighupChan()
 
-	_, err := loadAuthConfig()
+	_, err := reloadAuthConfig()
 	if err != nil {
 		logger.Fatalf("cannot load auth config: %s", err)
 	}
@@ -640,7 +629,7 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 
 	updateFn := func() {
 		configReloads.Inc()
-		updated, err := loadAuthConfig()
+		updated, err := reloadAuthConfig()
 		if err != nil {
 			logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
 			configSuccess.Set(0)
@@ -666,27 +655,45 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 	}
 }
 
-// authConfigData stores the yaml definition for this config.
-// authConfigData needs to be updated each time authConfig is updated.
-var authConfigData atomic.Pointer[[]byte]
-
 var (
-	authConfig   atomic.Pointer[AuthConfig]
-	authUsers    atomic.Pointer[map[string]*UserInfo]
+	// authConfigData stores the yaml definition for this config.
+	// authConfigData needs to be updated each time authConfig is updated.
+	authConfigData atomic.Pointer[[]byte]
+
+	// authConfig contains the currently loaded auth config
+	authConfig atomic.Pointer[AuthConfig]
+
+	// authUsers contains the currently loaded auth users
+	authUsers atomic.Pointer[map[string]*UserInfo]
+
 	authConfigWG sync.WaitGroup
 	stopCh       chan struct{}
 )
 
-// loadAuthConfig loads and applies the config from *authConfigPath.
+// reloadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
 // The config can be not applied if there is a parsing error
 // or if there are no changes to the current authConfig.
-func loadAuthConfig() (bool, error) {
+func reloadAuthConfig() (bool, error) {
 	data, err := fscore.ReadFileOrHTTP(*authConfigPath)
 	if err != nil {
 		return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
 	}
 
+	ok, err := reloadAuthConfigData(data)
+	if err != nil {
+		return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
+	}
+	if !ok {
+		return false, nil
+	}
+
+	mp := authUsers.Load()
+	logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
+	return true, nil
+}
+
+func reloadAuthConfigData(data []byte) (bool, error) {
 	oldData := authConfigData.Load()
 	if oldData != nil && bytes.Equal(data, *oldData) {
 		// there are no updates in the config - skip reloading.
@@ -695,14 +702,13 @@ func loadAuthConfig() (bool, error) {
 
 	ac, err := parseAuthConfig(data)
 	if err != nil {
-		return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
+		return false, fmt.Errorf("failed to parse auth config: %w", err)
 	}
 
 	m, err := parseAuthConfigUsers(ac)
 	if err != nil {
-		return false, fmt.Errorf("failed to parse users from -auth.config=%q: %w", *authConfigPath, err)
+		return false, fmt.Errorf("failed to parse users from auth config: %w", err)
 	}
-	logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
 
 	acPrev := authConfig.Load()
 	if acPrev != nil {
@@ -749,7 +755,6 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
-		ui.useBackendHostHeader = hasEmptyHostHeader(ui.HeadersConf.RequestHeaders)
 
 		metricLabels, err := ui.getMetricLabels()
 		if err != nil {
@@ -814,7 +819,6 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
-		ui.useBackendHostHeader = hasEmptyHostHeader(ui.HeadersConf.RequestHeaders)
 
 		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
 		if err != nil {
@@ -1005,6 +1009,14 @@ func getAuthTokensFromRequest(r *http.Request) []string {
 		}
 	}
 
+	// Authorization via http://user:pass@hosname/path
+	if u := r.URL.User; u != nil && u.Username() != "" {
+		username := u.Username()
+		password, _ := u.Password()
+		at := getHTTPAuthBasicToken(username, password)
+		ats = append(ats, at)
+	}
+
 	return ats
 }
 
@@ -1030,10 +1042,6 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
 }
 
 func sanitizeURLPrefix(urlPrefix *url.URL) (*url.URL, error) {
-	// Remove trailing '/' from urlPrefix
-	for strings.HasSuffix(urlPrefix.Path, "/") {
-		urlPrefix.Path = urlPrefix.Path[:len(urlPrefix.Path)-1]
-	}
 	// Validate urlPrefix
 	if urlPrefix.Scheme != "http" && urlPrefix.Scheme != "https" {
 		return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, urlPrefix.Scheme)

app/vmauth/auth_config_test.go

@@ -82,6 +82,14 @@ users:
   headers: foobar
 `)
 
+	// Invalid keep_original_host value
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  keep_original_host: foobar
+`)
+
 	// empty url_prefix
 	f(`
 users:
@@ -410,7 +418,7 @@ users:
 				HeadersConf: HeadersConf{
 					RequestHeaders: []*Header{
 						mustNewHeader("'foo: bar'"),
-						mustNewHeader("'xxx: y'"),
+						mustNewHeader("'xxx:'"),
 					},
 				},
 			},
@@ -429,7 +437,7 @@ users:
     url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
     headers:
     - "foo: bar"
-    - "xxx: y"
+    - "xxx:"
 `, map[string]*UserInfo{
 		getHTTPAuthBearerToken("foo"):    sharedUserInfo,
 		getHTTPAuthBasicToken("foo", ""): sharedUserInfo,
@@ -458,6 +466,7 @@ users:
 	})
 
 	// with default url
+	keepOriginalHost := true
 	f(`
 users:
 - bearer_token: foo
@@ -469,6 +478,7 @@ users:
     headers:
     - "foo: bar"
     - "xxx: y"
+    keep_original_host: true
   default_url:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
@@ -491,6 +501,7 @@ users:
 							mustNewHeader("'foo: bar'"),
 							mustNewHeader("'xxx: y'"),
 						},
+						KeepOriginalHost: &keepOriginalHost,
 					},
 				},
 			},
@@ -517,6 +528,7 @@ users:
 							mustNewHeader("'foo: bar'"),
 							mustNewHeader("'xxx: y'"),
 						},
+						KeepOriginalHost: &keepOriginalHost,
 					},
 				},
 			},
@@ -536,12 +548,17 @@ users:
   metric_labels:
     dc: eu
     team: dev
+  keep_original_host: true
 - username: foo-same
   password: bar
   url_prefix: https://bar/x
   metric_labels:
     backend_env: test
     team: accounting
+  headers:
+  - "foo: bar"
+  response_headers:
+  - "Abc: def"
 `, map[string]*UserInfo{
 		getHTTPAuthBasicToken("foo-same", "baz"): {
 			Username:  "foo-same",
@@ -551,6 +568,9 @@ users:
 				"dc":   "eu",
 				"team": "dev",
 			},
+			HeadersConf: HeadersConf{
+				KeepOriginalHost: &keepOriginalHost,
+			},
 		},
 		getHTTPAuthBasicToken("foo-same", "bar"): {
 			Username:  "foo-same",
@@ -560,6 +580,14 @@ users:
 				"backend_env": "test",
 				"team":        "accounting",
 			},
+			HeadersConf: HeadersConf{
+				RequestHeaders: []*Header{
+					mustNewHeader("'foo: bar'"),
+				},
+				ResponseHeaders: []*Header{
+					mustNewHeader("'Abc: def'"),
+				},
+			},
 		},
 	})
 }
@@ -695,9 +723,9 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 
 	fn := func(ns ...int) {
 		t.Helper()
-		bus := up.bus.Load()
-		pbus := *bus
-		for i, b := range pbus {
+		pbus := up.bus.Load()
+		bus := *pbus
+		for i, b := range bus {
 			got := int(b.concurrentRequests.Load())
 			exp := ns[i]
 			if got != exp {
@@ -707,39 +735,39 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	}
 
 	up.getBackendURL()
-	fn(0, 1, 0)
+	fn(1, 0, 0)
 	up.getBackendURL()
-	fn(0, 1, 1)
+	fn(1, 1, 0)
 	up.getBackendURL()
 	fn(1, 1, 1)
 
 	up.getBackendURL()
 	up.getBackendURL()
-	fn(1, 2, 2)
+	fn(2, 2, 1)
 
 	bus := up.bus.Load()
 	pbus := *bus
 	pbus[0].concurrentRequests.Add(2)
 	pbus[2].concurrentRequests.Add(5)
-	fn(3, 2, 7)
+	fn(4, 2, 6)
 
 	up.getBackendURL()
-	fn(3, 3, 7)
+	fn(4, 3, 6)
 
 	up.getBackendURL()
-	fn(3, 4, 7)
+	fn(4, 4, 6)
 
 	up.getBackendURL()
-	fn(4, 4, 7)
+	fn(4, 5, 6)
 
 	up.getBackendURL()
-	fn(5, 4, 7)
+	fn(5, 5, 6)
 
 	up.getBackendURL()
-	fn(5, 5, 7)
+	fn(6, 5, 6)
 
 	up.getBackendURL()
-	fn(6, 5, 7)
+	fn(6, 6, 6)
 
 	up.getBackendURL()
 	fn(6, 6, 7)

app/vmauth/main.go

@@ -117,8 +117,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 
-		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
-		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
+		handleMissingAuthorizationError(w)
 		return true
 	}
 
@@ -189,12 +188,11 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			// to a route that is not in the configuration for unauthorized user.
 			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5236
 			if ui.BearerToken == "" && ui.Username == "" && len(*authUsers.Load()) > 0 {
-				w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
-				http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
+				handleMissingAuthorizationError(w)
 				return
 			}
 			missingRouteRequests.Inc()
-			httpserver.Errorf(w, r, "missing route for %q", u.String())
+			httpserver.Errorf(w, r, "missing route for %s", u.String())
 			return
 		}
 		up, hc = ui.DefaultURL, ui.HeadersConf
@@ -220,7 +218,15 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
+
+		wasLocalRetry := false
+	again:
+		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
+		if needLocalRetry && !wasLocalRetry {
+			wasLocalRetry = true
+			goto again
+		}
+
 		bu.put()
 		if ok {
 			return
@@ -228,29 +234,28 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		bu.setBroken()
 	}
 	err := &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("all the backends for the user %q are unavailable", ui.name()),
-		StatusCode: http.StatusServiceUnavailable,
+		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
+		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)
 	ui.backendErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
-	// This code has been copied from net/http/httputil/reverseproxy.go
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
 	req := sanitizeRequestHeaders(r)
-	req.URL = targetURL
 
-	if req.URL.Scheme == "https" || ui.useBackendHostHeader {
-		// Override req.Host only for https requests, since https server verifies hostnames during TLS handshake,
-		// so it expects the targetURL.Host in the request.
-		// There is no need in overriding the req.Host for http requests, since it is expected that backend server
-		// may properly process queries with the original req.Host.
-		req.Host = targetURL.Host
-	}
+	req.URL = targetURL
+	req.Header.Set("User-Agent", "vmauth")
 	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	var trivialRetries int
+	if hc.KeepOriginalHost == nil || !*hc.KeepOriginalHost {
+		if host := getHostHeader(hc.RequestHeaders); host != "" {
+			req.Host = host
+		} else {
+			req.Host = targetURL.Host
+		}
+	}
+
 	rtb, rtbOK := req.Body.(*readTrackingBody)
-again:
 	res, err := ui.rt.RoundTrip(req)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -262,7 +267,7 @@ again:
 				// Timed out request must be counted as errors, since this usually means that the backend is slow.
 				ui.backendErrors.Inc()
 			}
-			return true
+			return true, false
 		}
 		if !rtbOK || !rtb.canRetry() {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
@@ -272,19 +277,19 @@ again:
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
-			return true
+			return true, false
 		}
-		// Retry request on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
-		if (netutil.IsTrivialNetworkError(err) || errors.Is(err, io.EOF)) && trivialRetries < 1 {
-			trivialRetries++
-			goto again
+		if netutil.IsTrivialNetworkError(err) {
+			// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
+			return false, true
 		}
+
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
 		// it explicitly reads request body, which may fail retries.
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
-		return false
+		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
 		_ = res.Body.Close()
@@ -298,7 +303,7 @@ again:
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
-			return true
+			return true, false
 		}
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
@@ -307,7 +312,7 @@ again:
 		// it explicitly reads request body, which may fail retries.
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
 			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
-		return false
+		return false, false
 	}
 	removeHopHeaders(res.Header)
 	copyHeader(w.Header(), res.Header)
@@ -323,9 +328,9 @@ again:
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
-		return true
+		return true, false
 	}
-	return true
+	return true, false
 }
 
 var copyBufPool bytesutil.ByteBufferPool
@@ -338,12 +343,21 @@ func copyHeader(dst, src http.Header) {
 	}
 }
 
-func updateHeadersByConfig(headers http.Header, config []*Header) {
-	for _, h := range config {
+func getHostHeader(headers []*Header) string {
+	for _, h := range headers {
+		if h.Name == "Host" {
+			return h.Value
+		}
+	}
+	return ""
+}
+
+func updateHeadersByConfig(dst http.Header, src []*Header) {
+	for _, h := range src {
 		if h.Value == "" {
-			headers.Del(h.Name)
+			dst.Del(h.Name)
 		} else {
-			headers.Set(h.Name, h.Value)
+			dst.Set(h.Name, h.Value)
 		}
 	}
 }
@@ -483,6 +497,11 @@ See the docs at https://docs.victoriametrics.com/vmauth/ .
 	flagutil.Usage(s)
 }
 
+func handleMissingAuthorizationError(w http.ResponseWriter) {
+	w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+	http.Error(w, "missing 'Authorization' request header", http.StatusUnauthorized)
+}
+
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
 	w.Header().Add("Retry-After", "10")
 	err = &httpserver.ErrorWithStatusCode{
@@ -537,10 +556,24 @@ func getReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
 		maxBodySize = 0
 	}
 	rtb.maxBodySize = maxBodySize
+
+	if r == nil {
+		// This is GET request without request body
+		r = (*zeroReader)(nil)
+	}
 	rtb.r = r
 	return rtb
 }
 
+type zeroReader struct{}
+
+func (r *zeroReader) Read(_ []byte) (int, error) {
+	return 0, io.EOF
+}
+func (r *zeroReader) Close() error {
+	return nil
+}
+
 func putReadTrackingBody(rtb *readTrackingBody) {
 	rtb.reset()
 	readTrackingBodyPool.Put(rtb)
@@ -560,7 +593,7 @@ func (rtb *readTrackingBody) Read(p []byte) (int, error) {
 		if rtb.bufComplete {
 			return 0, io.EOF
 		}
-		return 0, fmt.Errorf("cannot read data after closing the reader")
+		return 0, fmt.Errorf("cannot read client request body after closing client reader")
 	}
 
 	n, err := rtb.r.Read(p)

app/vmauth/main_test.go

@@ -2,10 +2,511 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
+func TestRequestHandler(t *testing.T) {
+	f := func(cfgStr, requestURL string, backendHandler http.HandlerFunc, responseExpected string) {
+		t.Helper()
+
+		ts := httptest.NewServer(backendHandler)
+		defer ts.Close()
+
+		cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
+		responseExpected = strings.ReplaceAll(responseExpected, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodGet, requestURL, nil)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		r.RequestURI = r.URL.RequestURI()
+		r.RemoteAddr = "42.2.3.84:6789"
+		r.Header.Set("X-Forwarded-For", "12.34.56.78")
+		r.Header.Set("Connection", "Some-Header,Other-Header")
+		r.Header.Set("Some-Header", "foobar")
+		r.Header.Set("Pass-Header", "abc")
+
+		w := &fakeResponseWriter{}
+		if !requestHandler(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+		responseExpected = strings.TrimSpace(responseExpected)
+		if response != responseExpected {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, responseExpected)
+		}
+	}
+
+	// regular url_prefix
+	cfgStr := `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo?bar=baz`
+	requestURL := "http://some-host.com/abc/def?some_arg=some_value"
+	backendHandler := func(w http.ResponseWriter, r *http.Request) {
+		h := w.Header()
+		h.Set("Connection", "close")
+		h.Set("Foo", "bar")
+
+		var bb bytes.Buffer
+		if err := r.Header.Write(&bb); err != nil {
+			panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
+		}
+		fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
+	}
+	responseExpected := `
+statusCode=200
+Foo: bar
+requested_url={BACKEND}/foo/abc/def?bar=baz&some_arg=some_value
+Pass-Header: abc
+User-Agent: vmauth
+X-Forwarded-For: 12.34.56.78, 42.2.3.84`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// keep_original_host
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  keep_original_host: true`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url=http://some-host.com/foo/abc/def?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// override user-agent header
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  headers:
+  - "User-Agent: foobar"`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/abc/def?bar=baz
+User-Agent=foobar`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// delete user-agent header
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  headers:
+  - "User-Agent:"`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/abc/def?bar=baz
+User-Agent=Go-http-client/1.1`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// override request host with non-empty host
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  headers:
+  - "Host: other-host:12345"
+  - "abc:"`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url=http://other-host:12345/foo/abc/def?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// override request host with empty host
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  headers:
+  - "Host:"`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/abc/def?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// /-/reload handler failure
+	origAuthKey := reloadAuthKey.Get()
+	if err := reloadAuthKey.Set("secret"); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo"`
+	requestURL = "http://some-host.com/-/reload"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+The provided authKey doesn't match -reloadAuthKey`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	if err := reloadAuthKey.Set(origAuthKey); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	// missing authorization
+	cfgStr = `
+users:
+- username: foo
+  url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://some-host.com/a/b"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+Www-Authenticate: Basic realm="Restricted"
+missing 'Authorization' request header`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// incorrect authorization
+	cfgStr = `
+users:
+- username: foo
+  password: secret
+  url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://foo:invalid-secret@some-host.com/a/b"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+Unauthorized`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// incorrect authorization with logging invalid auth tokens
+	origLogInvalidAuthTokens := *logInvalidAuthTokens
+	*logInvalidAuthTokens = true
+	cfgStr = `
+users:
+- username: foo
+  password: secret
+  url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://foo:invalid-secret@some-host.com/a/b?c=d"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /a/b?c=d; cannot authorize request with auth tokens ["http_auth:Basic Zm9vOmludmFsaWQtc2VjcmV0"]`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	*logInvalidAuthTokens = origLogInvalidAuthTokens
+
+	// correct authorization
+	cfgStr = `
+users:
+- username: foo
+  password: secret
+  url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://foo:secret@some-host.com/a/b"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/bar/a/b`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify how path cleanup works
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo?bar=baz`
+	requestURL = "http://some-host.com/../../a//.///bar/"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/a/bar/?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify how path cleanup works for url without path
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo?bar=baz`
+	requestURL = "http://some-host.com/"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify how path cleanup works for url without path if url_prefix path ends with /
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo/?bar=baz`
+	requestURL = "http://some-host.com/"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify how path cleanup works for url without path and the url_prefix without path prefix
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/?bar=baz`
+	requestURL = "http://some-host.com/"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify routing to default_url
+	cfgStr = `
+unauthorized_user:
+  url_map:
+  - src_paths: ["/foo/.+"]
+    url_prefix: {BACKEND}/x-foo/
+  default_url: {BACKEND}/404.html`
+	requestURL = "http://some-host.com/abc?de=fg"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/404.html?request_path=http%3A%2F%2Fsome-host.com%2Fabc%3Fde%3Dfg`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify routing to default url_prefix
+	cfgStr = `
+unauthorized_user:
+  url_map:
+  - src_paths: ["/foo/.+"]
+    url_prefix: {BACKEND}/x-foo/
+  url_prefix: {BACKEND}/default`
+	requestURL = "http://some-host.com/abc?de=fg"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/default/abc?de=fg`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// missing default_url and default url_prefix for unauthorized user
+	cfgStr = `
+unauthorized_user:
+  url_map:
+  - src_paths: ["/foo/.+"]
+    url_prefix: {BACKEND}/x-foo/`
+	requestURL = "http://some-host.com/abc?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=400
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /abc?de=fg; missing route for http://some-host.com/abc?de=fg`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// missing default_url and default url_prefix for unauthorized user when there are configs for authorized users
+	cfgStr = `
+users:
+- username: some-user
+  url_map:
+  - src_paths: ["/foo/.+"]
+    url_prefix: {BACKEND}/x-foo/
+unauthorized_user:
+  url_map:
+  - src_paths: ["/abc/.*"]
+    url_prefix: {BACKEND}/x-bar`
+	requestURL = "http://some-host.com/abc?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+Www-Authenticate: Basic realm="Restricted"
+missing 'Authorization' request header`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// all the backend_urls are unavailable for unauthorized user
+	cfgStr = `
+unauthorized_user:
+  url_map:
+  - src_paths: ["/foo/.*"]
+    url_prefix:
+    - http://127.0.0.1:1/
+    - http://127.0.0.1:2/`
+	requestURL = "http://some-host.com/foo/?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=502
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// all the backend_urls are unavailable for authorized user
+	cfgStr = `
+users:
+- username: some-user
+  url_map:
+  - src_paths: ["/foo/.*"]
+    url_prefix:
+    - http://127.0.0.1:1/
+    - http://127.0.0.1:2/`
+	requestURL = "http://some-user@some-host.com/foo/?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=502
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "some-user" are unavailable`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// zero discovered backend IPs
+	customResolver := &fakeResolver{
+		Resolver: &net.Resolver{},
+		lookupIPAddrResults: map[string][]net.IPAddr{
+			"some-addr": {},
+		},
+	}
+	origResolver := netutil.Resolver
+	netutil.Resolver = customResolver
+	cfgStr = `
+unauthorized_user:
+  url_prefix: ['http://some-addr:1234/foo/bar']
+  discover_backend_ips: true`
+	requestURL = "http://abc.com/def/?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=502
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /def/?de=fg; all the 0 backends for the user "" are unavailable`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	netutil.Resolver = origResolver
+
+	// retry_status_codes failure
+	var retries atomic.Int64
+	cfgStr = `
+unauthorized_user:
+  url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
+  retry_status_codes: [500, 502]`
+	requestURL = "http://some-host.com/foo/?de=fg"
+	backendHandler = func(w http.ResponseWriter, _ *http.Request) {
+		retries.Add(1)
+		w.WriteHeader(500)
+	}
+	responseExpected = `
+statusCode=502
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	if n := retries.Load(); n != 2 {
+		t.Fatalf("unexpected number of retries; got %d; want 2", n)
+	}
+
+	// retry_status_codes success
+	retries.Store(0)
+	cfgStr = `
+unauthorized_user:
+  url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
+  retry_status_codes: [500, 502]`
+	requestURL = "http://some-host.com/foo/?de=fg"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		if n := retries.Add(1); n < 2 {
+			w.WriteHeader(500)
+			return
+		}
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/path2/foo/?de=fg`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	if n := retries.Load(); n != 2 {
+		t.Fatalf("unexpected number of retries; got %d; want 2", n)
+	}
+}
+
+type fakeResponseWriter struct {
+	h http.Header
+
+	bb bytes.Buffer
+}
+
+func (w *fakeResponseWriter) getResponse() string {
+	return w.bb.String()
+}
+
+func (w *fakeResponseWriter) Header() http.Header {
+	if w.h == nil {
+		w.h = http.Header{}
+	}
+	return w.h
+}
+
+func (w *fakeResponseWriter) Write(p []byte) (int, error) {
+	return w.bb.Write(p)
+}
+
+func (w *fakeResponseWriter) WriteHeader(statusCode int) {
+	fmt.Fprintf(&w.bb, "statusCode=%d\n", statusCode)
+	if w.h == nil {
+		return
+	}
+	err := w.h.WriteSubset(&w.bb, map[string]bool{
+		"Content-Length":         true,
+		"Content-Type":           true,
+		"Date":                   true,
+		"X-Content-Type-Options": true,
+	})
+	if err != nil {
+		panic(fmt.Errorf("cannot marshal headers: %s", err))
+	}
+}
+
 func TestReadTrackingBody_RetrySuccess(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()

app/vmauth/target_url_test.go

@@ -335,8 +335,8 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
 		t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
 	}
 
-	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up")
 	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.50:8481/select/0/prometheus/api/v1/query?query=up")
+	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up")
 	f(ui, `/insert/0/prometheus/api/v1/write`, "http://10.6.142.52:8480/insert/0/prometheus/api/v1/write")
 	// unsuccessful dns resolve
 	f(ui, `/test`, "http://non-exist-dns-addr/test")

app/vmctl/backoff/backoff.go

@@ -10,12 +10,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-const (
-	backoffRetries     = 10
-	backoffFactor      = 1.8
-	backoffMinDuration = time.Second * 2
-)
-
 // retryableFunc describes call back which will repeat on errors
 type retryableFunc func() error
 
@@ -30,12 +24,22 @@ type Backoff struct {
 }
 
 // New initialize backoff object
-func New() *Backoff {
-	return &Backoff{
-		retries:     backoffRetries,
-		factor:      backoffFactor,
-		minDuration: backoffMinDuration,
+func New(retries int, factor float64, minDuration time.Duration) (*Backoff, error) {
+	if retries <= 0 {
+		return nil, fmt.Errorf("number of backoff retries must be greater than 0")
 	}
+	if factor <= 1 {
+		return nil, fmt.Errorf("backoff retry factor must be greater than 1")
+	}
+	if minDuration <= 0 {
+		return nil, fmt.Errorf("backoff retry minimum duration must be greater than 0")
+	}
+
+	return &Backoff{
+		retries:     retries,
+		factor:      factor,
+		minDuration: minDuration,
+	}, nil
 }
 
 // Retry process retries until all attempts are completed

app/vmctl/backoff/backoff_test.go

@@ -3,6 +3,7 @@ package backoff
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 )
@@ -110,3 +111,32 @@ func TestBackoffRetry_Success(t *testing.T) {
 	resultExpected := 1
 	f(retryFunc, resultExpected)
 }
+
+func TestBackoff_New(t *testing.T) {
+	f := func(retries int, factor float64, minDuration time.Duration, errExpected string) {
+		t.Helper()
+
+		_, err := New(retries, factor, minDuration)
+		if err == nil {
+			if errExpected != "" {
+				t.Fatalf("expecting non-nil error")
+			}
+			return
+		}
+		if !strings.Contains(err.Error(), errExpected) {
+			t.Fatalf("unexpected error: got %q; want %q", err.Error(), errExpected)
+		}
+	}
+
+	// empty retries
+	f(0, 1.1, time.Millisecond*10, "retries must be greater than 0")
+
+	// empty factor
+	f(1, 0, time.Millisecond*10, "factor must be greater than 1")
+
+	// empty minDuration
+	f(1, 1.1, 0, "minimum duration must be greater than 0")
+
+	// no errors
+	f(1, 1.1, time.Millisecond*10, "")
+}

app/vmctl/flags.go

@@ -56,6 +56,10 @@ const (
 	vmRateLimit  = "vm-rate-limit"
 
 	vmInterCluster = "vm-intercluster"
+
+	vmBackoffRetries     = "vm-backoff-retries"
+	vmBackoffFactor      = "vm-backoff-factor"
+	vmBackoffMinDuration = "vm-backoff-min-duration"
 )
 
 var (
@@ -146,6 +150,21 @@ var (
 			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
 			Value: false,
 		},
+		&cli.IntFlag{
+			Name:  vmBackoffRetries,
+			Value: 10,
+			Usage: "How many import retries to perform before giving up.",
+		},
+		&cli.Float64Flag{
+			Name:  vmBackoffFactor,
+			Value: 1.8,
+			Usage: "Factor to multiply the base duration after each failed import retry. Must be greater than 1.0",
+		},
+		&cli.DurationFlag{
+			Name:  vmBackoffMinDuration,
+			Value: time.Second * 2,
+			Usage: "Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'.",
+		},
 	}
 )
 
@@ -430,6 +449,10 @@ const (
 	vmNativeDstCAFile             = "vm-native-dst-ca-file"
 	vmNativeDstServerName         = "vm-native-dst-server-name"
 	vmNativeDstInsecureSkipVerify = "vm-native-dst-insecure-skip-verify"
+
+	vmNativeBackoffRetries     = "vm-native-backoff-retries"
+	vmNativeBackoffFactor      = "vm-native-backoff-factor"
+	vmNativeBackoffMinDuration = "vm-native-backoff-min-duration"
 )
 
 var (
@@ -599,6 +622,21 @@ var (
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
 			Value: false,
 		},
+		&cli.IntFlag{
+			Name:  vmNativeBackoffRetries,
+			Value: 10,
+			Usage: "How many export/import retries to perform before giving up.",
+		},
+		&cli.Float64Flag{
+			Name:  vmNativeBackoffFactor,
+			Value: 1.8,
+			Usage: "Factor to multiply the base duration after each failed export/import retry. Must be greater than 1.0",
+		},
+		&cli.DurationFlag{
+			Name:  vmNativeBackoffMinDuration,
+			Value: time.Second * 2,
+			Usage: "Minimum duration to wait before the first export/import retry. Each subsequent export/import retry will be multiplied by the '--vm-native-backoff-factor'.",
+		},
 	}
 )
 

app/vmctl/main.go

@@ -68,7 +68,7 @@ func main() {
 
 					tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
-						return fmt.Errorf("failed to create Transport: %s", err)
+						return fmt.Errorf("failed to create transport for -%s=%q: %s", otsdbAddr, addr, err)
 					}
 					oCfg := opentsdb.Config{
 						Addr:       addr,
@@ -90,6 +90,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -143,6 +144,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -178,7 +180,7 @@ func main() {
 
 					tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
-						return fmt.Errorf("failed to create transport: %s", err)
+						return fmt.Errorf("failed to create transport for -%s=%q: %s", remoteReadSrcAddr, addr, err)
 					}
 
 					rr, err := remoteread.NewClient(remoteread.Config{
@@ -201,6 +203,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -233,6 +236,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -272,6 +276,14 @@ func main() {
 						return fmt.Errorf("flag %q can't be empty", vmNativeFilterMatch)
 					}
 
+					bfRetries := c.Int(vmNativeBackoffRetries)
+					bfFactor := c.Float64(vmNativeBackoffFactor)
+					bfMinDuration := c.Duration(vmNativeBackoffMinDuration)
+					bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
+					if err != nil {
+						return fmt.Errorf("failed to create backoff object: %s", err)
+					}
+
 					disableKeepAlive := c.Bool(vmNativeDisableHTTPKeepAlive)
 
 					var srcExtraLabels []string
@@ -350,7 +362,7 @@ func main() {
 							ExtraLabels: dstExtraLabels,
 							HTTPClient:  dstHTTPClient,
 						},
-						backoff:                  backoff.New(),
+						backoff:                  bf,
 						cc:                       c.Int(vmConcurrency),
 						disablePerMetricRequests: c.Bool(vmNativeDisablePerMetricMigration),
 						isNative:                 !c.Bool(vmNativeDisableBinaryProtocol),
@@ -426,7 +438,15 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 
 	tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 	if err != nil {
-		return vm.Config{}, fmt.Errorf("failed to create Transport: %s", err)
+		return vm.Config{}, fmt.Errorf("failed to create transport for -%s=%q: %s", vmAddr, addr, err)
+	}
+
+	bfRetries := c.Int(vmBackoffRetries)
+	bfFactor := c.Float64(vmBackoffFactor)
+	bfMinDuration := c.Duration(vmBackoffMinDuration)
+	bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
+	if err != nil {
+		return vm.Config{}, fmt.Errorf("failed to create backoff object: %s", err)
 	}
 
 	return vm.Config{
@@ -442,5 +462,6 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 		RoundDigits:        c.Int(vmRoundDigits),
 		ExtraLabels:        c.StringSlice(vmExtraLabel),
 		RateLimit:          c.Int64(vmRateLimit),
+		Backoff:            bf,
 	}, nil
 }

app/vmctl/vm/vm.go

@@ -54,6 +54,8 @@ type Config struct {
 	// RateLimit defines a data transfer speed in bytes per second.
 	// Is applied to each worker (see Concurrency) independently.
 	RateLimit int64
+	// Backoff defines backoff policy for retries
+	Backoff *backoff.Backoff
 }
 
 // Importer performs insertion of timeseries
@@ -144,7 +146,7 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		close:      make(chan struct{}),
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
-		backoff:    backoff.New(),
+		backoff:    cfg.Backoff,
 	}
 	if err := im.Ping(); err != nil {
 		return nil, fmt.Errorf("ping to %q failed: %s", addr, err)

app/vminsert/main.go

@@ -299,6 +299,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		addInfluxResponseHeaders(w)
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -423,7 +427,8 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/query", protocol="influx"}`)
+	influxQueryRequests  = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/query", protocol="influx"}`)
+	influxHealthRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/health", protocol="influx"}`)
 
 	opentelemetryPushRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/opentelemetry/v1/metrics", protocol="opentelemetry"}`)

app/vminsert/netstorage/insert_ctx.go

@@ -122,11 +122,48 @@ func (ctx *InsertCtx) ApplyRelabeling() {
 func (ctx *InsertCtx) WriteDataPoint(at *auth.Token, labels []prompb.Label, timestamp int64, value float64) error {
 	ctx.MetricNameBuf = storage.MarshalMetricNameRaw(ctx.MetricNameBuf[:0], at.AccountID, at.ProjectID, labels)
 	storageNodeIdx := ctx.GetStorageNodeIdx(at, labels)
-	return ctx.WriteDataPointExt(storageNodeIdx, ctx.MetricNameBuf, timestamp, value)
+	return ctx.writeDataPointToReplicas(storageNodeIdx, ctx.MetricNameBuf, timestamp, value)
 }
 
 // WriteDataPointExt writes the given metricNameRaw with (timestmap, value) to ctx buffer with the given storageNodeIdx.
 func (ctx *InsertCtx) WriteDataPointExt(storageNodeIdx int, metricNameRaw []byte, timestamp int64, value float64) error {
+	return ctx.writeDataPointToReplicas(storageNodeIdx, metricNameRaw, timestamp, value)
+}
+
+func (ctx *InsertCtx) writeDataPointToReplicas(storageNodeIdx int, metricNameRaw []byte, timestamp int64, value float64) error {
+	var firstErr error
+	var failsCount int
+	for i := 0; i < replicas; i++ {
+		snIdx := storageNodeIdx + i
+		if snIdx >= len(ctx.snb.sns) {
+			snIdx %= len(ctx.snb.sns)
+		}
+
+		if err := ctx.writeDataPointExt(snIdx, metricNameRaw, timestamp, value); err != nil {
+			if replicas == 1 {
+				return fmt.Errorf("cannot write datapoint: %w", err)
+			}
+			if firstErr == nil {
+				firstErr = err
+			}
+			failsCount++
+			// The data is partially replicated, so just emit a warning and return true.
+			// We could retry sending the data again, but this may result in uncontrolled duplicate data.
+			// So it is better returning true.
+			br := &ctx.bufRowss[snIdx]
+			rowsIncompletelyReplicatedTotal.Add(br.rows)
+			incompleteReplicationLogger.Warnf("cannot make a copy #%d out of %d copies according to -replicationFactor=%d, used_nodes=%d for %d bytes with %d rows, "+
+				"since a part of storage nodes is temporarily unavailable", i+1, replicas, *replicationFactor, len(br.buf), br.rows)
+			continue
+		}
+	}
+	if failsCount == replicas {
+		return fmt.Errorf("cannot write datapoint to any replicas: %w", firstErr)
+	}
+	return nil
+}
+
+func (ctx *InsertCtx) writeDataPointExt(storageNodeIdx int, metricNameRaw []byte, timestamp int64, value float64) error {
 	br := &ctx.bufRowss[storageNodeIdx]
 	snb := ctx.snb
 	sn := snb.sns[storageNodeIdx]

app/vminsert/netstorage/insert_ctx_pool.go

@@ -2,16 +2,23 @@ package netstorage
 
 import (
 	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 )
 
 // GetInsertCtx returns InsertCtx from the pool.
 //
 // Call PutInsertCtx for returning it to the pool.
 func GetInsertCtx() *InsertCtx {
-	if v := insertCtxPool.Get(); v != nil {
-		return v.(*InsertCtx)
+	select {
+	case ctx := <-insertCtxPoolCh:
+		return ctx
+	default:
+		if v := insertCtxPool.Get(); v != nil {
+			return v.(*InsertCtx)
+		}
+		return &InsertCtx{}
 	}
-	return &InsertCtx{}
 }
 
 // PutInsertCtx returns ctx to the pool.
@@ -19,7 +26,14 @@ func GetInsertCtx() *InsertCtx {
 // ctx cannot be used after the call.
 func PutInsertCtx(ctx *InsertCtx) {
 	ctx.Reset()
-	insertCtxPool.Put(ctx)
+	select {
+	case insertCtxPoolCh <- ctx:
+	default:
+		insertCtxPool.Put(ctx)
+	}
 }
 
-var insertCtxPool sync.Pool
+var (
+	insertCtxPool   sync.Pool
+	insertCtxPoolCh = make(chan *InsertCtx, cgroup.AvailableCPUs())
+)

app/vminsert/netstorage/netstorage.go

@@ -45,6 +45,8 @@ var (
 		"See also -disableRerouting")
 )
 
+var replicas int
+
 var errStorageReadOnly = errors.New("storage node is read only")
 
 func (sn *storageNode) isReady() bool {
@@ -140,15 +142,6 @@ again:
 }
 
 func (sn *storageNode) run(snb *storageNodesBucket, snIdx int) {
-	replicas := *replicationFactor
-	if replicas <= 0 {
-		replicas = 1
-	}
-	sns := snb.sns
-	if replicas > len(sns) {
-		replicas = len(sns)
-	}
-
 	sn.readOnlyCheckerWG.Add(1)
 	go func() {
 		defer sn.readOnlyCheckerWG.Done()
@@ -193,7 +186,7 @@ func (sn *storageNode) run(snb *storageNodesBucket, snIdx int) {
 			continue
 		}
 		// Send br to replicas storage nodes starting from snIdx.
-		for !sendBufToReplicasNonblocking(snb, &br, snIdx, replicas) {
+		for !sendBufToSnNonblocking(snb, &br, snIdx) {
 			d := timeutil.AddJitterToDuration(time.Millisecond * 200)
 			t := timerpool.Get(d)
 			select {
@@ -206,50 +199,38 @@ func (sn *storageNode) run(snb *storageNodesBucket, snIdx int) {
 				sn.checkHealth()
 			}
 		}
+		if sn.isBufferFull.CompareAndSwap(true, false) {
+			logger.Infof("transited node=%s to non-full", sn.dialer.Addr())
+		}
+
 		br.reset()
 	}
 }
 
-func sendBufToReplicasNonblocking(snb *storageNodesBucket, br *bufRows, snIdx, replicas int) bool {
-	usedStorageNodes := make(map[*storageNode]struct{}, replicas)
+func sendBufToSnNonblocking(snb *storageNodesBucket, br *bufRows, snIdx int) bool {
 	sns := snb.sns
-	for i := 0; i < replicas; i++ {
-		idx := snIdx + i
-		attempts := 0
-		for {
-			attempts++
-			if attempts > len(sns) {
-				if i == 0 {
-					// The data wasn't replicated at all.
-					cannotReplicateLogger.Warnf("cannot push %d bytes with %d rows to storage nodes, since all the nodes are temporarily unavailable; "+
-						"re-trying to send the data soon", len(br.buf), br.rows)
-					return false
-				}
-				// The data is partially replicated, so just emit a warning and return true.
-				// We could retry sending the data again, but this may result in uncontrolled duplicate data.
-				// So it is better returning true.
-				rowsIncompletelyReplicatedTotal.Add(br.rows)
-				incompleteReplicationLogger.Warnf("cannot make a copy #%d out of %d copies according to -replicationFactor=%d for %d bytes with %d rows, "+
-					"since a part of storage nodes is temporarily unavailable", i+1, replicas, *replicationFactor, len(br.buf), br.rows)
-				return true
-			}
-			if idx >= len(sns) {
-				idx %= len(sns)
-			}
-			sn := sns[idx]
-			idx++
-			if _, ok := usedStorageNodes[sn]; ok {
-				// The br has been already replicated to sn. Skip it.
-				continue
-			}
-			if !sn.sendBufRowsNonblocking(br) {
-				// Cannot send data to sn. Go to the next sn.
-				continue
-			}
-			// Successfully sent data to sn.
-			usedStorageNodes[sn] = struct{}{}
-			break
+	idx := snIdx
+	attempts := 0
+	for {
+		attempts++
+		if attempts > len(sns) {
+			// The data wasn't replicated at all.
+			cannotReplicateLogger.Warnf("cannot push %d bytes with %d rows to storage nodes, since all the nodes are temporarily unavailable; "+
+				"re-trying to send the data soon", len(br.buf), br.rows)
+			return false
 		}
+		if idx >= len(sns) {
+			idx %= len(sns)
+		}
+		sn := sns[idx]
+		idx++
+
+		if !sn.sendBufRowsNonblocking(br) {
+			// Cannot send data to sn. Go to the next sn.
+			continue
+		}
+		// Successfully sent data to sn.
+		break
 	}
 	return true
 }
@@ -413,6 +394,8 @@ type storageNode struct {
 	// In this case the data is re-routed to the remaining healthy vmstorage nodes.
 	isBroken atomic.Bool
 
+	isBufferFull atomic.Bool
+
 	// isReadOnly is set to true if the given vmstorage node is read only
 	// In this case the data is re-routed to the remaining healthy vmstorage nodes.
 	isReadOnly atomic.Bool
@@ -505,6 +488,13 @@ func setStorageNodesBucket(snb *storageNodesBucket) {
 //
 // Call MustStop when the initialized vmstorage connections are no longer needed.
 func Init(addrs []string, hashSeed uint64) {
+	replicas = *replicationFactor
+	if replicas <= 0 {
+		replicas = 1
+	}
+	if replicas > len(addrs) {
+		replicas = len(addrs)
+	}
 	snb := initStorageNodes(addrs, hashSeed)
 	setStorageNodesBucket(snb)
 }
@@ -772,7 +762,7 @@ var noStorageNodesLogger = logger.WithThrottler("storageNodesUnavailable", 5*tim
 func getNotReadyStorageNodeIdxs(snb *storageNodesBucket, dst []int, snExtra *storageNode) []int {
 	dst = dst[:0]
 	for i, sn := range snb.sns {
-		if sn == snExtra || !sn.isReady() {
+		if sn == snExtra || !sn.isReady() || sn.isBufferFull.Load() {
 			dst = append(dst, i)
 		}
 	}
@@ -787,10 +777,17 @@ func (sn *storageNode) trySendBuf(buf []byte, rows int) bool {
 
 	sent := false
 	sn.brLock.Lock()
-	if sn.isReady() && len(sn.br.buf)+len(buf) <= maxBufSizePerStorageNode {
+	if !sn.isReady() {
+		return sent
+	}
+	if len(sn.br.buf)+len(buf) <= maxBufSizePerStorageNode {
 		sn.br.buf = append(sn.br.buf, buf...)
 		sn.br.rows += rows
 		sent = true
+	} else {
+		if sn.isBufferFull.CompareAndSwap(false, true) {
+			logger.Infof("node: %s transited to full", sn.dialer.Addr())
+		}
 	}
 	sn.brLock.Unlock()
 	return sent

app/vmselect/graphite/eval.go

@@ -19,11 +19,12 @@ var maxGraphiteSeries = flag.Int("search.maxGraphiteSeries", 300e3, "The maximum
 	"See https://docs.victoriametrics.com/#graphite-render-api-usage")
 
 type evalConfig struct {
-	at          *auth.Token
-	startTime   int64
-	endTime     int64
-	storageStep int64
-	deadline    searchutils.Deadline
+	at                  *auth.Token
+	startTime           int64
+	endTime             int64
+	storageStep         int64
+	denyPartialResponse bool
+	deadline            searchutils.Deadline
 
 	currentTime time.Time
 
@@ -155,8 +156,7 @@ func evalMetricExpr(ec *evalConfig, me *graphiteql.MetricExpr) (nextSeriesFunc,
 }
 
 func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr graphiteql.Expr) (nextSeriesFunc, error) {
-	denyPartialResponse := true
-	rss, _, err := netstorage.ProcessSearchQuery(nil, denyPartialResponse, sq, ec.deadline)
+	rss, _, err := netstorage.ProcessSearchQuery(nil, ec.denyPartialResponse, sq, ec.deadline)
 	if err != nil {
 		return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}

app/vmselect/graphite/render_api.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -94,19 +95,21 @@ func RenderHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
 	if err != nil {
 		return fmt.Errorf("cannot setup tag filters: %w", err)
 	}
+	denyPartialResponse := httputils.GetDenyPartialResponse(r)
 	var nextSeriess []nextSeriesFunc
 	targets := r.Form["target"]
 	for _, target := range targets {
 		ec := &evalConfig{
-			at:            at,
-			startTime:     fromTime,
-			endTime:       untilTime,
-			storageStep:   storageStep,
-			deadline:      deadline,
-			currentTime:   startTime,
-			xFilesFactor:  xFilesFactor,
-			etfs:          etfs,
-			originalQuery: target,
+			at:                  at,
+			startTime:           fromTime,
+			endTime:             untilTime,
+			storageStep:         storageStep,
+			denyPartialResponse: denyPartialResponse,
+			deadline:            deadline,
+			currentTime:         startTime,
+			xFilesFactor:        xFilesFactor,
+			etfs:                etfs,
+			originalQuery:       target,
 		}
 		nextSeries, err := execExpr(ec, target)
 		if err != nil {

app/vmselect/main.go

@@ -43,7 +43,8 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	cacheDataPath         = flag.String("cacheDataPath", "", "Path to directory for cache files. By default, the cache is not persisted.")
+	cacheDataPath = flag.String("cacheDataPath", "", "Path to directory for cache files and temporary query results. "+
+		"By default, the cache won't be persisted, and temporary query results will be placed under /tmp/searchResults. If set, the cache will be persisted under cacheDataPath/rollupResult, and temporary query results will be placed under cacheDataPath/tmp/searchResults.")
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
 		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
@@ -193,7 +194,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	startTime := time.Now()
 	defer requestDuration.UpdateDuration(startTime)
 	tracerEnabled := httputils.GetBool(r, "trace")
-	qt := querytracer.New(tracerEnabled, r.URL.Path)
+	qt := querytracer.New(tracerEnabled, "%s", r.URL.Path)
 
 	// Limit the number of concurrent queries.
 	select {

app/vmselect/netstorage/tmp_blocks_file.go

@@ -140,7 +140,7 @@ func (tbf *tmpBlocksFile) Finalize() error {
 	tbf.buf = tbf.buf[:0]
 	r := fs.NewReaderAt(tbf.f)
 
-	// Hint the OS that the file is read almost sequentiallly.
+	// Hint the OS that the file is read almost sequentially.
 	// This should reduce the number of disk seeks, which is important
 	// for HDDs.
 	r.MustFadviseSequentialRead(true)

app/vmselect/promql/transform.go

@@ -1052,7 +1052,7 @@ func fixBrokenBuckets(i int, xss []leTimeseries) {
 
 	// Substitute upper bucket values with lower bucket values if the upper values are NaN
 	// or are bigger than the lower bucket values.
-	vNext := xss[0].ts.Values[0]
+	vNext := xss[0].ts.Values[i]
 	for j := 1; j < len(xss); j++ {
 		v := xss[j].ts.Values[i]
 		if math.IsNaN(v) || vNext > v {

app/vmselect/promql/transform_test.go

@@ -39,6 +39,30 @@ func TestFixBrokenBuckets(t *testing.T) {
 	f([]float64{5, 10, 4, 3}, []float64{5, 10, 10, 10})
 }
 
+func TestFixBrokenBucketsMultipleValues(t *testing.T) {
+	f := func(values, expectedResult [][]float64) {
+		t.Helper()
+		xss := make([]leTimeseries, len(values))
+		for i, v := range values {
+
+			xss[i].ts = &timeseries{
+				Values: v,
+			}
+		}
+		for i := range len(values) - 1 {
+			fixBrokenBuckets(i, xss)
+		}
+		result := make([][]float64, len(values))
+		for i, xs := range xss {
+			result[i] = xs.ts.Values
+		}
+		if !reflect.DeepEqual(result, expectedResult) {
+			t.Fatalf("unexpected result for values=%v\ngot\n%v\nwant\n%v", values, result, expectedResult)
+		}
+	}
+	f([][]float64{{10, 1}, {11, 2}, {13, 3}}, [][]float64{{10, 1}, {11, 2}, {13, 3}})
+}
+
 func TestVmrangeBucketsToLE(t *testing.T) {
 	f := func(buckets, bucketsExpected string) {
 		t.Helper()

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
     "main.css": "./static/css/main.fce049bf.css",
-    "main.js": "./static/js/main.4b213512.js",
+    "main.js": "./static/js/main.36501ae8.js",
     "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.d46c42c8e891f06298c4.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
     "static/css/main.fce049bf.css",
-    "static/js/main.4b213512.js"
+    "static/js/main.36501ae8.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.4b213512.js"></script><link href="./static/css/main.fce049bf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.36501ae8.js"></script><link href="./static/css/main.fce049bf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmselect/vmui/static/js/main.36501ae8.js.LICENSE.txt

@@ -5,7 +5,7 @@
 */
 
 /**
- * @remix-run/router v1.15.1
+ * @remix-run/router v1.17.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -16,7 +16,7 @@
  */
 
 /**
- * React Router DOM v6.22.1
+ * React Router DOM v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -27,7 +27,7 @@
  */
 
 /**
- * React Router v6.22.1
+ * React Router v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *

app/vmselect/vmui/static/media/MetricsQL.d46c42c8e891f06298c4.md

@@ -10,9 +10,6 @@ aliases:
 - /ExtendedPromQL.html
 - /MetricsQL.html
 ---
-
-# MetricsQL
-
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -220,9 +217,11 @@ See also [descent_over_time](#descent_over_time).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [median_over_time](#median_over_time).
+See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
 
 #### changes
 
@@ -262,6 +261,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
@@ -272,6 +273,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
@@ -282,6 +285,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
@@ -292,7 +297,9 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### count_over_time
 
@@ -313,6 +320,8 @@ The results are calculated independently per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
@@ -432,6 +441,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### histogram_over_time
 
 `histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -443,11 +454,15 @@ For example, the following query calculates median temperature by country over t
 
 `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### hoeffding_bound_lower
 
 `hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 
 #### hoeffding_bound_upper
@@ -455,6 +470,8 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 `hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 
 #### holt_winters
@@ -462,8 +479,9 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
 (aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
-Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
-returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
+Both `sf` and `tf` must be in the range `[0...1]`.
+
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
 
 This function is supported by PromQL.
 
@@ -495,13 +513,14 @@ See also [deriv](#deriv).
 
 `increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -510,12 +529,13 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
 
 `increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 See also [increase_pure](#increase_pure) and [increase](#increase).
 
 #### increase_pure
@@ -524,6 +544,10 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
 of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
 while [increase](#increase) ignores the first value in a series if it is too big.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [increase](#increas) and [increase_prometheus](#increase_prometheus).
+
 #### increases_over_time
 
 `increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -540,15 +564,18 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### irate
 
 `irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
 the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -587,6 +614,8 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 #### max_over_time
@@ -594,9 +623,11 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmax_over_time](#tmax_over_time).
+See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
 
 #### median_over_time
 
@@ -604,6 +635,8 @@ See also [tmax_over_time](#tmax_over_time).
 on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [avg_over_time](#avg_over_time).
 
 #### min_over_time
@@ -611,9 +644,11 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmin_over_time](#tmin_over_time).
+See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
 
 #### mode_over_time
 
@@ -622,6 +657,8 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values are discrete.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### outlier_iqr_over_time
 
 `outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -632,6 +669,8 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
 For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [outliers_iqr](#outliers_iqr).
 
 #### predict_linear
@@ -659,6 +698,8 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The `phi` value must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [quantiles_over_time](#quantiles_over_time).
@@ -670,6 +711,8 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [quantile_over_time](#quantile_over_time).
 
 #### range_over_time
@@ -680,11 +723,12 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rate
 
 `rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -703,6 +747,8 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
 on the given lookbehind window `d`. The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### resets
@@ -710,10 +756,11 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 `resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
 of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 #### rollup
@@ -725,6 +772,10 @@ These values are calculated individually per each time series returned from the
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [rollup_rate](#rollup_rate).
+
 #### rollup_candlestick
 
 `rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -736,6 +787,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rollup_delta
 
 `rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -762,6 +815,8 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_increase
 
 `rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -774,12 +829,17 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_rate
 
 `rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
 for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
 and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
+The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
 when to use `rollup_rate()`.
@@ -787,10 +847,12 @@ when to use `rollup_rate()`.
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
+
 #### rollup_scrape_interval
 
 `rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -824,6 +886,8 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
@@ -838,6 +902,8 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
@@ -849,6 +915,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time).
 
 #### stale_samples_over_time
@@ -866,6 +934,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stdvar_over_time](#stdvar_over_time).
@@ -877,6 +947,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stddev_over_time](#stddev_over_time).
@@ -888,6 +960,8 @@ values equal to `eq` on the given lookbehind window `d` per each time series ret
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### sum_gt_over_time
@@ -897,6 +971,8 @@ values bigger than `gt` on the given lookbehind window `d` per each time series
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### sum_le_over_time
@@ -906,6 +982,8 @@ values smaller or equal to `le` on the given lookbehind window `d` per each time
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
@@ -915,6 +993,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 #### sum2_over_time
@@ -924,6 +1004,8 @@ values on the given lookbehind window `d` per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### timestamp
 
 `timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1001,6 +1083,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 
@@ -2230,7 +2314,8 @@ Any [rollup function](#rollup-functions) for something other than [series select
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
 For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
-This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
 
 VictoriaMetrics performs subqueries in the following way:
 
@@ -2263,5 +2348,6 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`. 
-  This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
+  This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).

app/vmui/Dockerfile-web

@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.20.1
+FROM alpine:3.20.2
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/src/api/types.ts

@@ -38,6 +38,7 @@ export interface Logs {
 export interface LogHits {
   timestamps: string[];
   values: number[];
+  total?: number;
   fields: {
     [key: string]: string;
   };

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -10,9 +10,6 @@ aliases:
 - /ExtendedPromQL.html
 - /MetricsQL.html
 ---
-
-# MetricsQL
-
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -220,9 +217,11 @@ See also [descent_over_time](#descent_over_time).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [median_over_time](#median_over_time).
+See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
 
 #### changes
 
@@ -262,6 +261,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
@@ -272,6 +273,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
@@ -282,6 +285,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
@@ -292,7 +297,9 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### count_over_time
 
@@ -313,6 +320,8 @@ The results are calculated independently per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
@@ -432,6 +441,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### histogram_over_time
 
 `histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -443,11 +454,15 @@ For example, the following query calculates median temperature by country over t
 
 `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### hoeffding_bound_lower
 
 `hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 
 #### hoeffding_bound_upper
@@ -455,6 +470,8 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 `hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 
 #### holt_winters
@@ -462,8 +479,9 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
 (aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
-Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
-returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
+Both `sf` and `tf` must be in the range `[0...1]`.
+
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
 
 This function is supported by PromQL.
 
@@ -495,13 +513,14 @@ See also [deriv](#deriv).
 
 `increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -510,12 +529,13 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
 
 `increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 See also [increase_pure](#increase_pure) and [increase](#increase).
 
 #### increase_pure
@@ -524,6 +544,10 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
 of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
 while [increase](#increase) ignores the first value in a series if it is too big.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [increase](#increas) and [increase_prometheus](#increase_prometheus).
+
 #### increases_over_time
 
 `increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -540,15 +564,18 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### irate
 
 `irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
 the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -587,6 +614,8 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 #### max_over_time
@@ -594,9 +623,11 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmax_over_time](#tmax_over_time).
+See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
 
 #### median_over_time
 
@@ -604,6 +635,8 @@ See also [tmax_over_time](#tmax_over_time).
 on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [avg_over_time](#avg_over_time).
 
 #### min_over_time
@@ -611,9 +644,11 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmin_over_time](#tmin_over_time).
+See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
 
 #### mode_over_time
 
@@ -622,6 +657,8 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values are discrete.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### outlier_iqr_over_time
 
 `outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -632,6 +669,8 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
 For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [outliers_iqr](#outliers_iqr).
 
 #### predict_linear
@@ -659,6 +698,8 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The `phi` value must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [quantiles_over_time](#quantiles_over_time).
@@ -670,6 +711,8 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [quantile_over_time](#quantile_over_time).
 
 #### range_over_time
@@ -680,11 +723,12 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rate
 
 `rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -703,6 +747,8 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
 on the given lookbehind window `d`. The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### resets
@@ -710,10 +756,11 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 `resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
 of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 #### rollup
@@ -725,6 +772,10 @@ These values are calculated individually per each time series returned from the
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [rollup_rate](#rollup_rate).
+
 #### rollup_candlestick
 
 `rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -736,6 +787,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rollup_delta
 
 `rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -762,6 +815,8 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_increase
 
 `rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -774,12 +829,17 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_rate
 
 `rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
 for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
 and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
+The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
 when to use `rollup_rate()`.
@@ -787,10 +847,12 @@ when to use `rollup_rate()`.
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
+
 #### rollup_scrape_interval
 
 `rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -824,6 +886,8 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
@@ -838,6 +902,8 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
@@ -849,6 +915,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time).
 
 #### stale_samples_over_time
@@ -866,6 +934,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stdvar_over_time](#stdvar_over_time).
@@ -877,6 +947,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stddev_over_time](#stddev_over_time).
@@ -888,6 +960,8 @@ values equal to `eq` on the given lookbehind window `d` per each time series ret
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### sum_gt_over_time
@@ -897,6 +971,8 @@ values bigger than `gt` on the given lookbehind window `d` per each time series
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### sum_le_over_time
@@ -906,6 +982,8 @@ values smaller or equal to `le` on the given lookbehind window `d` per each time
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
@@ -915,6 +993,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 #### sum2_over_time
@@ -924,6 +1004,8 @@ values on the given lookbehind window `d` per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### timestamp
 
 `timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1001,6 +1083,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 
@@ -2230,7 +2314,8 @@ Any [rollup function](#rollup-functions) for something other than [series select
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
 For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
-This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
 
 VictoriaMetrics performs subqueries in the following way:
 
@@ -2263,5 +2348,6 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`. 
-  This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
+  This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsChart.tsx

@@ -1,32 +1,66 @@
-import React, { FC, useRef, useState } from "preact/compat";
+import React, { FC, useMemo, useRef, useState } from "preact/compat";
 import "./style.scss";
 import "uplot/dist/uPlot.min.css";
 import useElementSize from "../../../hooks/useElementSize";
 import uPlot, { AlignedData } from "uplot";
 import { useEffect } from "react";
 import useBarHitsOptions from "./hooks/useBarHitsOptions";
-import TooltipBarHitsChart from "./TooltipBarHitsChart";
+import BarHitsTooltip from "./BarHitsTooltip/BarHitsTooltip";
 import { TimeParams } from "../../../types";
 import usePlotScale from "../../../hooks/uplot/usePlotScale";
 import useReadyChart from "../../../hooks/uplot/useReadyChart";
 import useZoomChart from "../../../hooks/uplot/useZoomChart";
 import classNames from "classnames";
+import { LogHits } from "../../../api/types";
+import { addSeries, delSeries, setBand } from "../../../utils/uplot";
+import { GraphOptions, GRAPH_STYLES } from "./types";
+import BarHitsOptions from "./BarHitsOptions/BarHitsOptions";
+import stack from "../../../utils/uplot/stack";
+import BarHitsLegend from "./BarHitsLegend/BarHitsLegend";
 
 interface Props {
+  logHits: LogHits[];
   data: AlignedData;
   period: TimeParams;
   setPeriod: ({ from, to }: { from: Date, to: Date }) => void;
+  onApplyFilter: (value: string) => void;
 }
-
-const BarHitsChart: FC<Props> = ({ data, period, setPeriod }) => {
+const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onApplyFilter }) => {
   const [containerRef, containerSize] = useElementSize();
   const uPlotRef = useRef<HTMLDivElement>(null);
   const [uPlotInst, setUPlotInst] = useState<uPlot>();
+  const [graphOptions, setGraphOptions] = useState<GraphOptions>({
+    graphStyle: GRAPH_STYLES.LINE_STEPPED,
+    stacked: false,
+    fill: false,
+  });
 
   const { xRange, setPlotScale } = usePlotScale({ period, setPeriod });
   const { onReadyChart, isPanning } = useReadyChart(setPlotScale);
   useZoomChart({ uPlotInst, xRange, setPlotScale });
-  const { options, focusDataIdx } = useBarHitsOptions({ xRange, containerSize, onReadyChart, setPlotScale });
+
+  const { data, bands } = useMemo(() => {
+    return graphOptions.stacked ? stack(_data, () => false) : { data: _data, bands: [] };
+  }, [graphOptions, _data]);
+
+  const { options, series, focusDataIdx } = useBarHitsOptions({
+    data,
+    logHits,
+    bands,
+    xRange,
+    containerSize,
+    onReadyChart,
+    setPlotScale,
+    graphOptions
+  });
+
+  useEffect(() => {
+    if (!uPlotInst) return;
+    delSeries(uPlotInst);
+    addSeries(uPlotInst, series, true);
+    setBand(uPlotInst, series);
+    uPlotInst.redraw();
+  }, [series]);
 
   useEffect(() => {
     if (!uPlotRef.current) return;
@@ -54,21 +88,31 @@ const BarHitsChart: FC<Props> = ({ data, period, setPeriod }) => {
   }, [data]);
 
   return (
-    <div
-      className={classNames({
-        "vm-bar-hits-chart": true,
-        "vm-bar-hits-chart_panning": isPanning
-      })}
-      ref={containerRef}
-    >
+    <div className="vm-bar-hits-chart__wrapper">
       <div
-        className="vm-line-chart__u-plot"
-        ref={uPlotRef}
-      />
-      <TooltipBarHitsChart
-        uPlotInst={uPlotInst}
-        focusDataIdx={focusDataIdx}
-      />
+        className={classNames({
+          "vm-bar-hits-chart": true,
+          "vm-bar-hits-chart_panning": isPanning
+        })}
+        ref={containerRef}
+      >
+        <div
+          className="vm-line-chart__u-plot"
+          ref={uPlotRef}
+        />
+        <BarHitsTooltip
+          uPlotInst={uPlotInst}
+          data={_data}
+          focusDataIdx={focusDataIdx}
+        />
+      </div>
+      <BarHitsOptions onChange={setGraphOptions}/>
+      {uPlotInst && (
+        <BarHitsLegend
+          uPlotInst={uPlotInst}
+          onApplyFilter={onApplyFilter}
+        />
+      )}
     </div>
   );
 };

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/BarHitsLegend.tsx

@@ -0,0 +1,68 @@
+import React, { FC, useCallback, useEffect, useState } from "preact/compat";
+import uPlot, { Series } from "uplot";
+import "./style.scss";
+import "../../Line/Legend/style.scss";
+import classNames from "classnames";
+import { MouseEvent } from "react";
+import { isMacOs } from "../../../../utils/detect-device";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+
+interface Props {
+  uPlotInst: uPlot;
+  onApplyFilter: (value: string) => void;
+}
+
+const BarHitsLegend: FC<Props> = ({ uPlotInst, onApplyFilter }) => {
+  const [series, setSeries] = useState<Series[]>([]);
+
+  const updateSeries = useCallback(() => {
+    const series = uPlotInst.series.filter(s => s.scale !== "x");
+    setSeries(series);
+  }, [uPlotInst]);
+
+  const handleClick = (target: Series) => (e: MouseEvent<HTMLDivElement>) => {
+    const metaKey = e.metaKey || e.ctrlKey;
+    if (!metaKey) {
+      target.show = !target.show;
+    } else {
+      onApplyFilter(target.label || "");
+    }
+
+    updateSeries();
+    uPlotInst.redraw();
+  };
+
+  useEffect(updateSeries, [uPlotInst]);
+
+  return (
+    <div className="vm-bar-hits-legend">
+      {series.map(s => (
+        <Tooltip
+          key={s.label}
+          title={(
+            <ul className="vm-bar-hits-legend-info">
+              <li>Click to {s.show ? "hide" : "show"} the _stream.</li>
+              <li>{isMacOs() ? "Cmd" : "Ctrl"} + Click to filter by the _stream.</li>
+            </ul>
+          )}
+        >
+          <div
+            className={classNames({
+              "vm-bar-hits-legend-item": true,
+              "vm-bar-hits-legend-item_hide": !s.show,
+            })}
+            onClick={handleClick(s)}
+          >
+            <div
+              className="vm-bar-hits-legend-item__marker"
+              style={{ backgroundColor: `${(s?.stroke as () => string)?.()}` }}
+            />
+            <div>{s.label}</div>
+          </div>
+        </Tooltip>
+      ))}
+    </div>
+  );
+};
+
+export default BarHitsLegend;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/style.scss

@@ -0,0 +1,35 @@
+@use "src/styles/variables" as *;
+
+.vm-bar-hits-legend {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0;
+  padding: 0 $padding-small $padding-small;
+
+  &-item {
+    display: grid;
+    grid-template-columns: auto 1fr;
+    align-items: center;
+    gap: 4px;
+    font-size: 12px;
+    padding: $padding-small;
+    border-radius: $border-radius-small;
+    cursor: pointer;
+    transition: 0.2s;
+
+    &:hover {
+      background-color: rgba(0, 0, 0, 0.05);
+    }
+
+    &_hide {
+      text-decoration: line-through;
+      opacity: 0.5;
+    }
+
+    &__marker {
+      width: 14px;
+      height: 14px;
+      border: $color-background-block;
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsOptions/BarHitsOptions.tsx

@@ -0,0 +1,116 @@
+import React, { FC, useEffect, useMemo, useRef } from "preact/compat";
+import { GraphOptions, GRAPH_STYLES } from "../types";
+import Switch from "../../../Main/Switch/Switch";
+import "./style.scss";
+import useStateSearchParams from "../../../../hooks/useStateSearchParams";
+import { useSearchParams } from "react-router-dom";
+import Button from "../../../Main/Button/Button";
+import classNames from "classnames";
+import { SettingsIcon } from "../../../Main/Icons";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+import Popper from "../../../Main/Popper/Popper";
+import useBoolean from "../../../../hooks/useBoolean";
+
+interface Props {
+  onChange: (options: GraphOptions) => void;
+}
+
+const BarHitsOptions: FC<Props> = ({ onChange }) => {
+  const [searchParams, setSearchParams] = useSearchParams();
+  const optionsButtonRef = useRef<HTMLDivElement>(null);
+  const {
+    value: openOptions,
+    toggle: toggleOpenOptions,
+    setFalse: handleCloseOptions,
+  } = useBoolean(false);
+
+  const [graphStyle, setGraphStyle] = useStateSearchParams(GRAPH_STYLES.LINE_STEPPED, "graph");
+  const [stacked, setStacked] = useStateSearchParams(false, "stacked");
+  const [fill, setFill] = useStateSearchParams(false, "fill");
+
+  const options: GraphOptions = useMemo(() => ({
+    graphStyle,
+    stacked,
+    fill,
+  }), [graphStyle, stacked, fill]);
+
+  const handleChangeGraphStyle = (val: string) => () => {
+    setGraphStyle(val as GRAPH_STYLES);
+    searchParams.set("graph", val);
+    setSearchParams(searchParams);
+  };
+
+  const handleChangeFill = (val: boolean) => {
+    setFill(val);
+    val ? searchParams.set("fill", "true") : searchParams.delete("fill");
+    setSearchParams(searchParams);
+  };
+
+  const handleChangeStacked = (val: boolean) => {
+    setStacked(val);
+    val ? searchParams.set("stacked", "true") : searchParams.delete("stacked");
+    setSearchParams(searchParams);
+  };
+
+  useEffect(() => {
+    onChange(options);
+  }, [options]);
+
+  return (
+    <div
+      className="vm-bar-hits-options"
+      ref={optionsButtonRef}
+    >
+      <Tooltip title="Graph settings">
+        <Button
+          variant="text"
+          color="primary"
+          startIcon={<SettingsIcon/>}
+          onClick={toggleOpenOptions}
+          ariaLabel="settings"
+        />
+      </Tooltip>
+      <Popper
+        open={openOptions}
+        placement="bottom-right"
+        onClose={handleCloseOptions}
+        buttonRef={optionsButtonRef}
+        title={"Graph settings"}
+      >
+        <div className="vm-bar-hits-options-settings">
+          <div className="vm-bar-hits-options-settings-item vm-bar-hits-options-settings-item_list">
+            <p className="vm-bar-hits-options-settings-item__title">Graph style:</p>
+            {Object.values(GRAPH_STYLES).map(style => (
+              <div
+                key={style}
+                className={classNames({
+                  "vm-list-item": true,
+                  "vm-list-item_active": graphStyle === style,
+                })}
+                onClick={handleChangeGraphStyle(style)}
+              >
+                {style}
+              </div>
+            ))}
+          </div>
+          <div className="vm-bar-hits-options-settings-item">
+            <Switch
+              label={"Stacked"}
+              value={stacked}
+              onChange={handleChangeStacked}
+            />
+          </div>
+          <div className="vm-bar-hits-options-settings-item">
+            <Switch
+              label={"Fill"}
+              value={fill}
+              onChange={handleChangeFill}
+            />
+          </div>
+        </div>
+      </Popper>
+    </div>
+  );
+};
+
+export default BarHitsOptions;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsOptions/style.scss

@@ -0,0 +1,35 @@
+@use "src/styles/variables" as *;
+
+.vm-bar-hits-options {
+  position: absolute;
+  top: $padding-small;
+  right: $padding-small;
+  z-index: 2;
+
+  &-settings {
+    display: grid;
+    align-items: flex-start;
+    gap: $padding-global;
+    min-width: 200px;
+
+    &-item {
+      border-bottom: $border-divider;
+      padding: 0 $padding-global $padding-global;
+
+      &_list {
+        padding: 0;
+      }
+
+      &__title {
+        font-size: $font-size-small;
+        color: $color-text-secondary;
+        padding: 0 $padding-small $padding-small;
+      }
+
+      &:last-child {
+        border-bottom: none;
+      }
+
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsTooltip/BarHitsTooltip.tsx

@@ -0,0 +1,125 @@
+import React, { FC, useMemo, useRef } from "preact/compat";
+import uPlot, { AlignedData } from "uplot";
+import dayjs from "dayjs";
+import { DATE_TIME_FORMAT } from "../../../../constants/date";
+import classNames from "classnames";
+import "./style.scss";
+import "../../ChartTooltip/style.scss";
+
+interface Props {
+  data: AlignedData;
+  uPlotInst?: uPlot;
+  focusDataIdx: number;
+}
+
+const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
+  const tooltipRef = useRef<HTMLDivElement>(null);
+
+  const tooltipData = useMemo(() => {
+    const series = uPlotInst?.series || [];
+    const [time, ...values] = data.map((d) => d[focusDataIdx] || 0);
+
+    const tooltipItems = values.map((value, i) => {
+      const targetSeries = series[i + 1];
+      const stroke = (targetSeries?.stroke as () => string)?.();
+      const label = targetSeries?.label || "other";
+      const show = targetSeries?.show;
+      return {
+        label,
+        stroke,
+        value,
+        show
+      };
+    }).filter(item => item.value > 0 && item.show).sort((a, b) => b.value - a.value);
+
+    const point = {
+      top: tooltipItems[0] ? uPlotInst?.valToPos?.(tooltipItems[0].value, "y") || 0 : 0,
+      left: uPlotInst?.valToPos?.(time, "x") || 0,
+    };
+
+    return {
+      point,
+      values: tooltipItems,
+      total: tooltipItems.reduce((acc, item) => acc + item.value, 0),
+      timestamp: dayjs(time * 1000).tz().format(DATE_TIME_FORMAT),
+    };
+  }, [focusDataIdx, uPlotInst, data]);
+
+  const tooltipPosition = useMemo(() => {
+    if (!uPlotInst || !tooltipData.total || !tooltipRef.current) return;
+
+    const { top, left } = tooltipData.point;
+    const uPlotPosition = {
+      left: parseFloat(uPlotInst.over.style.left),
+      top: parseFloat(uPlotInst.over.style.top)
+    };
+
+    const {
+      width: uPlotWidth,
+      height: uPlotHeight
+    } = uPlotInst.over.getBoundingClientRect();
+
+    const {
+      width: tooltipWidth,
+      height: tooltipHeight
+    } = tooltipRef.current.getBoundingClientRect();
+
+    const margin = 50;
+    const overflowX = left + tooltipWidth >= uPlotWidth ? tooltipWidth + (2 * margin) : 0;
+    const overflowY = top + tooltipHeight >= uPlotHeight ? tooltipHeight + (2 * margin) : 0;
+
+    const position = {
+      top: top + uPlotPosition.top + margin - overflowY,
+      left: left + uPlotPosition.left + margin - overflowX
+    };
+
+    if (position.left < 0) position.left = 20;
+    if (position.top < 0) position.top = 20;
+
+    return position;
+  }, [tooltipData, uPlotInst, tooltipRef.current]);
+
+  return (
+    <div
+      className={classNames({
+        "vm-chart-tooltip": true,
+        "vm-chart-tooltip_hits": true,
+        "vm-bar-hits-tooltip": true,
+        "vm-bar-hits-tooltip_visible": focusDataIdx !== -1 && tooltipData.values.length
+      })}
+      ref={tooltipRef}
+      style={tooltipPosition}
+    >
+      <div>
+        {tooltipData.values.map((item, i) => (
+          <div
+            className="vm-chart-tooltip-data"
+            key={i}
+          >
+            <span
+              className="vm-chart-tooltip-data__marker"
+              style={{ background: item.stroke }}
+            />
+            <p>
+              {item.label}: <b>{item.value}</b>
+            </p>
+          </div>
+        ))}
+      </div>
+      {tooltipData.values.length > 1 && (
+        <div className="vm-chart-tooltip-data">
+          <p>
+              Total records: <b>{tooltipData.total}</b>
+          </p>
+        </div>
+      )}
+      <div className="vm-chart-tooltip-header">
+        <div className="vm-chart-tooltip-header__title">
+          {tooltipData.timestamp}
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default BarHitsTooltip;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsTooltip/style.scss

@@ -0,0 +1,12 @@
+@use "src/styles/variables" as *;
+
+.vm-bar-hits-tooltip {
+  opacity: 0;
+  pointer-events: none;
+  gap: $padding-small;
+
+  &_visible {
+    opacity: 1;
+    pointer-events: auto;
+  }
+}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/TooltipBarHitsChart.tsx

@@ -1,89 +0,0 @@
-import React, { FC, useMemo, useRef } from "preact/compat";
-import uPlot from "uplot";
-import dayjs from "dayjs";
-import { DATE_TIME_FORMAT } from "../../../constants/date";
-import classNames from "classnames";
-import "./style.scss";
-import "../../../components/Chart/ChartTooltip/style.scss";
-
-interface Props {
-  uPlotInst?: uPlot;
-  focusDataIdx: number
-}
-
-const TooltipBarHitsChart: FC<Props> = ({ focusDataIdx, uPlotInst }) => {
-  const tooltipRef = useRef<HTMLDivElement>(null);
-
-  const tooltipData = useMemo(() => {
-    const value = uPlotInst?.data?.[1]?.[focusDataIdx];
-    const timestamp = uPlotInst?.data?.[0]?.[focusDataIdx] || 0;
-    const top = uPlotInst?.valToPos?.((value || 0), "y") || 0;
-    const left = uPlotInst?.valToPos?.(timestamp, "x") || 0;
-
-    return {
-      point: { top, left },
-      value,
-      timestamp: dayjs(timestamp * 1000).tz().format(DATE_TIME_FORMAT),
-    };
-  }, [focusDataIdx, uPlotInst]);
-
-  const tooltipPosition = useMemo(() => {
-    if (!uPlotInst || !tooltipData.value || !tooltipRef.current) return;
-
-    const { top, left } = tooltipData.point;
-    const uPlotPosition = {
-      left: parseFloat(uPlotInst.over.style.left),
-      top: parseFloat(uPlotInst.over.style.top)
-    };
-
-    const {
-      width: uPlotWidth,
-      height: uPlotHeight
-    } = uPlotInst.over.getBoundingClientRect();
-
-    const {
-      width: tooltipWidth,
-      height: tooltipHeight
-    } = tooltipRef.current.getBoundingClientRect();
-
-    const margin = 10;
-    const overflowX = left + tooltipWidth >= uPlotWidth ? tooltipWidth + (2 * margin) : 0;
-    const overflowY = top + tooltipHeight >= uPlotHeight ? tooltipHeight + (2 * margin) : 0;
-
-    const position = {
-      top: top + uPlotPosition.top + margin - overflowY,
-      left: left + uPlotPosition.left + margin - overflowX
-    };
-
-    if (position.left < 0) position.left = 20;
-    if (position.top < 0) position.top = 20;
-
-    return position;
-  }, [tooltipData, uPlotInst, tooltipRef.current]);
-
-  return (
-    <div
-      className={classNames({
-        "vm-chart-tooltip": true,
-        "vm-bar-hits-chart-tooltip": true,
-        "vm-bar-hits-chart-tooltip_visible": focusDataIdx !== -1
-      })}
-      ref={tooltipRef}
-      style={tooltipPosition}
-    >
-      <div className="vm-chart-tooltip-data">
-        Count of records:
-        <p className="vm-chart-tooltip-data__value">
-          <b>{tooltipData.value}</b>
-        </p>
-      </div>
-      <div className="vm-chart-tooltip-header">
-        <div className="vm-chart-tooltip-header__title">
-          {tooltipData.timestamp}
-        </div>
-      </div>
-    </div>
-  );
-};
-
-export default TooltipBarHitsChart;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/hooks/useBarHitsOptions.ts

@@ -2,42 +2,81 @@ import { useMemo, useState } from "preact/compat";
 import { getAxes, handleDestroy, setSelect } from "../../../../utils/uplot";
 import dayjs from "dayjs";
 import { dateFromSeconds, formatDateForNativeInput } from "../../../../utils/time";
-import uPlot, { Options } from "uplot";
+import uPlot, { AlignedData, Band, Options, Series } from "uplot";
 import { getCssVariable } from "../../../../utils/theme";
-import { barPaths } from "../../../../utils/uplot/bars";
 import { useAppState } from "../../../../state/common/StateContext";
 import { MinMax, SetMinMax } from "../../../../types";
+import { LogHits } from "../../../../api/types";
+import getSeriesPaths from "../../../../utils/uplot/paths";
+import { GraphOptions, GRAPH_STYLES } from "../types";
+
+const seriesColors = [
+  "color-log-hits-bar-1",
+  "color-log-hits-bar-2",
+  "color-log-hits-bar-3",
+  "color-log-hits-bar-4",
+  "color-log-hits-bar-5",
+];
+
+const strokeWidth = {
+  [GRAPH_STYLES.BAR]: 0.8,
+  [GRAPH_STYLES.LINE_STEPPED]: 1.2,
+  [GRAPH_STYLES.LINE]: 1.2,
+  [GRAPH_STYLES.POINTS]: 0,
+};
 
 interface UseGetBarHitsOptionsArgs {
+  data: AlignedData;
+  logHits: LogHits[];
   xRange: MinMax;
+  bands?: Band[];
   containerSize: { width: number, height: number };
   setPlotScale: SetMinMax;
   onReadyChart: (u: uPlot) => void;
+  graphOptions: GraphOptions;
 }
 
-const useBarHitsOptions = ({ xRange, containerSize, onReadyChart, setPlotScale }: UseGetBarHitsOptionsArgs) => {
+const useBarHitsOptions = ({
+  data,
+  logHits,
+  xRange,
+  bands,
+  containerSize,
+  onReadyChart,
+  setPlotScale,
+  graphOptions
+}: UseGetBarHitsOptionsArgs) => {
   const { isDarkTheme } = useAppState();
 
   const [focusDataIdx, setFocusDataIdx] = useState(-1);
 
-  const series = useMemo(() => [
-    {},
-    {
-      label: "y",
-      width: 1,
-      stroke: getCssVariable("color-log-hits-bar"),
-      fill: getCssVariable("color-log-hits-bar"),
-      paths: barPaths,
-    }
-  ], [isDarkTheme]);
-
   const setCursor = (u: uPlot) => {
     const dataIdx = u.cursor.idx ?? -1;
     setFocusDataIdx(dataIdx);
   };
 
+  const series: Series[] = useMemo(() => {
+    let colorN = 0;
+    return data.map((_d, i) => {
+      if (i === 0) return {}; // 0 index is xAxis(timestamps)
+      const fields = Object.values(logHits?.[i - 1]?.fields || {});
+      const label = fields.map((value) => value || "\"\"").join(", ");
+      const color = getCssVariable(label ? seriesColors[colorN] : "color-log-hits-bar-0");
+      if (label) colorN++;
+      return {
+        label: label || "other",
+        width: strokeWidth[graphOptions.graphStyle],
+        spanGaps: true,
+        stroke: color,
+        fill: graphOptions.fill ? color + "80" : "",
+        paths: getSeriesPaths(graphOptions.graphStyle),
+      };
+    });
+  }, [isDarkTheme, data, graphOptions]);
+
   const options: Options = useMemo(() => ({
     series,
+    bands,
     width: containerSize.width || (window.innerWidth / 2),
     height: containerSize.height || 200,
     cursor: {
@@ -55,6 +94,7 @@ const useBarHitsOptions = ({ xRange, containerSize, onReadyChart, setPlotScale }
       }
     },
     hooks: {
+      drawSeries: [],
       ready: [onReadyChart],
       setCursor: [setCursor],
       setSelect: [setSelect(setPlotScale)],
@@ -63,10 +103,11 @@ const useBarHitsOptions = ({ xRange, containerSize, onReadyChart, setPlotScale }
     legend: { show: false },
     axes: getAxes([{}, { scale: "y" }]),
     tzDate: ts => dayjs(formatDateForNativeInput(dateFromSeconds(ts))).local().toDate(),
-  }), [isDarkTheme]);
+  }), [isDarkTheme, series, bands]);
 
   return {
     options,
+    series,
     focusDataIdx,
   };
 };

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/style.scss

@@ -1,22 +1,18 @@
 @use "src/styles/variables" as *;
 
 .vm-bar-hits-chart {
-  height: 100%;
+  position: relative;
   width: 100%;
+  height: 200px;
+
+  &__wrapper {
+    display: flex;
+    flex-direction: column;
+    width: 100%;
+    height: 100%;
+  }
 
   &_panning {
     pointer-events: none;
   }
-
-  &-tooltip {
-    opacity: 0;
-    pointer-events: none;
-    width: 240px;
-    gap: $padding-small;
-
-    &_visible {
-      opacity: 1;
-      pointer-events: auto;
-    }
-  }
 }

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/types.ts

@@ -0,0 +1,12 @@
+export enum GRAPH_STYLES {
+  BAR = "Bars",
+  LINE = "Lines",
+  LINE_STEPPED = "Stepped lines",
+  POINTS = "Points",
+}
+
+export interface GraphOptions {
+  graphStyle: GRAPH_STYLES;
+  stacked: boolean;
+  fill: boolean;
+}

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -25,6 +25,12 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
   user-select: text;
   pointer-events: none;
 
+  &_hits {
+    white-space: pre-wrap;
+    word-break: break-all;
+    width: auto;
+  }
+
   &_sticky {
     pointer-events: auto;
     z-index: 99;
@@ -74,10 +80,22 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     justify-content: flex-start;
     gap: $padding-small;
 
+    &_margin-bottom {
+      margin-bottom: $padding-global;
+    }
+
+    &_margin-top {
+      margin-top: $padding-global;
+    }
+
     &__marker {
       width: $font-size;
       height: $font-size;
       border: 1px solid rgba($color-white, 0.5);
+
+      &_tranparent {
+        opacity: 0;
+      }
     }
 
     &__value {

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx

@@ -13,6 +13,7 @@ import ThemeControl from "../ThemeControl/ThemeControl";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import useBoolean from "../../../hooks/useBoolean";
 import { AppType } from "../../../types/appType";
+import SwitchMarkdownParsing from "../LogsSettings/MarkdownParsing/SwitchMarkdownParsing";
 
 const title = "Settings";
 
@@ -60,6 +61,10 @@ const GlobalSettings: FC = () => {
         onClose={handleClose}
       />
     },
+    {
+      show: isLogsApp,
+      component: <SwitchMarkdownParsing/>
+    },
     {
       show: true,
       component: <Timezones ref={timezoneSettingRef}/>

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/TenantsFields.tsx

@@ -0,0 +1,150 @@
+import React, { FC, useRef } from "preact/compat";
+import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
+import { ArrowDownIcon, QuestionIcon, StorageIcon } from "../../../Main/Icons";
+import Button from "../../../Main/Button/Button";
+import "./style.scss";
+import "../../TimeRangeSettings/ExecutionControls/style.scss";
+import classNames from "classnames";
+import Popper from "../../../Main/Popper/Popper";
+import { getAppModeEnable } from "../../../../utils/app-mode";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import TextField from "../../../Main/TextField/TextField";
+import useBoolean from "../../../../hooks/useBoolean";
+import useStateSearchParams from "../../../../hooks/useStateSearchParams";
+import { useSearchParams } from "react-router-dom";
+import { useEffect } from "react";
+
+const TenantsFields: FC = () => {
+  const appModeEnable = getAppModeEnable();
+  const { isMobile } = useDeviceDetect();
+  const timeDispatch = useTimeDispatch();
+
+  const [searchParams, setSearchParams] = useSearchParams();
+  const [accountID, setAccountID] = useStateSearchParams("0", "accountID");
+  const [projectID, setProjectID] = useStateSearchParams("0", "projectID");
+  const formattedTenant = `${accountID}:${projectID}`;
+
+  const buttonRef = useRef<HTMLDivElement>(null);
+
+  const {
+    value: openPopup,
+    toggle: toggleOpenPopup,
+    setFalse: handleClosePopup,
+  } = useBoolean(false);
+
+  const applyChanges = () => {
+    searchParams.set("accountID", accountID);
+    searchParams.set("projectID", projectID);
+    setSearchParams(searchParams);
+    handleClosePopup();
+    timeDispatch({ type: "RUN_QUERY" });
+  };
+
+  const handleReset = () => {
+    setAccountID(searchParams.get("accountID") || "0");
+    setProjectID(searchParams.get("projectID") || "0");
+  };
+
+  useEffect(() => {
+    if (openPopup) return;
+    handleReset();
+  }, [openPopup]);
+
+  return (
+    <div className="vm-tenant-input">
+      <Tooltip title="Define Tenant ID if you need request to another storage">
+        <div ref={buttonRef}>
+          {isMobile ? (
+            <div
+              className="vm-mobile-option"
+              onClick={toggleOpenPopup}
+            >
+              <span className="vm-mobile-option__icon"><StorageIcon/></span>
+              <div className="vm-mobile-option-text">
+                <span className="vm-mobile-option-text__label">Tenant ID</span>
+                <span className="vm-mobile-option-text__value">{formattedTenant}</span>
+              </div>
+              <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
+            </div>
+          ) : (
+            <Button
+              className={appModeEnable ? "" : "vm-header-button"}
+              variant="contained"
+              color="primary"
+              fullWidth
+              startIcon={<StorageIcon/>}
+              endIcon={(
+                <div
+                  className={classNames({
+                    "vm-execution-controls-buttons__arrow": true,
+                    "vm-execution-controls-buttons__arrow_open": openPopup,
+                  })}
+                >
+                  <ArrowDownIcon/>
+                </div>
+              )}
+              onClick={toggleOpenPopup}
+            >
+              {formattedTenant}
+            </Button>
+          )}
+        </div>
+      </Tooltip>
+      <Popper
+        open={openPopup}
+        placement="bottom-right"
+        onClose={handleClosePopup}
+        buttonRef={buttonRef}
+        title={isMobile ? "Define Tenant ID" : undefined}
+      >
+        <div
+          className={classNames({
+            "vm-list vm-tenant-input-list": true,
+            "vm-list vm-tenant-input-list_mobile": isMobile,
+            "vm-tenant-input-list_inline": true,
+          })}
+        >
+          <TextField
+            autofocus
+            label="accountID"
+            value={accountID}
+            onChange={setAccountID}
+            type="number"
+          />
+          <TextField
+            autofocus
+            label="projectID"
+            value={projectID}
+            onChange={setProjectID}
+            type="number"
+          />
+          <div className="vm-tenant-input-list__buttons">
+            <Tooltip title="Multitenancy in VictoriaLogs documentation">
+              <a
+                href="https://docs.victoriametrics.com/victorialogs/#multitenancy"
+                target="_blank"
+                rel="help noreferrer"
+              >
+                <Button
+                  variant="text"
+                  color="gray"
+                  startIcon={<QuestionIcon/>}
+                />
+              </a>
+            </Tooltip>
+            <Button
+              variant="contained"
+              color="primary"
+              onClick={applyChanges}
+            >
+              Apply
+            </Button>
+          </div>
+        </div>
+      </Popper>
+    </div>
+  );
+};
+
+export default TenantsFields;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/style.scss

@@ -17,11 +17,23 @@
       padding: 0 $padding-global $padding-small;
     }
 
+    &_inline {
+      display: grid;
+      gap: calc($padding-small/2);
+      padding: $padding-global;
+    }
+
     &__search {
       position: sticky;
       top: 0;
       padding: $padding-small $padding-global;
       background-color: $color-background-block;
     }
+
+    &__buttons {
+      display: flex;
+      justify-content: space-between;
+      gap: $padding-small;
+    }
   }
 }

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/style.scss

@@ -4,7 +4,7 @@
   display: flex;
   flex-direction: column;
   align-items: center;
-  gap: $padding-medium;
+  gap: $padding-large;
   width: 600px;
   padding-bottom: $padding-medium;
 
@@ -39,6 +39,13 @@
     margin-bottom: $padding-global;
   }
 
+  &__info {
+    padding-top: $padding-small;
+    font-size: $font-size-small;
+    color: $color-text-secondary;
+    line-height: 130%;
+  }
+
   &-url {
     display: flex;
     align-items: flex-start;

app/vmui/packages/vmui/src/components/Configurators/LogsSettings/MarkdownParsing/SwitchMarkdownParsing.tsx

@@ -0,0 +1,35 @@
+import React, { FC } from "preact/compat";
+import Switch from "../../../Main/Switch/Switch";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import { useLogsDispatch, useLogsState } from "../../../../state/logsPanel/LogsStateContext";
+
+const SwitchMarkdownParsing: FC = () => {
+  const { isMobile } = useDeviceDetect();
+  const { markdownParsing } = useLogsState();
+  const dispatch = useLogsDispatch();
+
+
+  const handleChangeMarkdownParsing = (val: boolean) => {
+    dispatch({ type: "SET_MARKDOWN_PARSING", payload: val });
+  };
+
+  return (
+    <div>
+      <div className="vm-server-configurator__title">
+        Markdown Parsing for Logs
+      </div>
+      <Switch
+        label={markdownParsing ? "Disable markdown parsing" : "Enable markdown parsing"}
+        value={markdownParsing}
+        onChange={handleChangeMarkdownParsing}
+        fullWidth={isMobile}
+      />
+      <div className="vm-server-configurator__info">
+        Toggle this switch to enable or disable the Markdown formatting for log entries.
+        Enabling this will parse log texts to Markdown.
+      </div>
+    </div>
+  );
+};
+
+export default SwitchMarkdownParsing;

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx

@@ -34,14 +34,25 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   }, [value, caretPosition]);
 
   const exprLastPart = useMemo(() => {
-    const parts = values.beforeCursor.split("}");
+    const regexpSplit = /\s(or|and|unless|default|ifnot|if|group_left|group_right)\s|}|\+|\|-|\*|\/|\^/i;
+    const parts = values.beforeCursor.split(regexpSplit);
     return parts[parts.length - 1];
   }, [values]);
 
   const metric = useMemo(() => {
-    const regexp = /\b[^{}(),\s]+(?={|$)/g;
-    const match = exprLastPart.match(regexp);
-    return match ? match[0] : "";
+    const regex1 = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
+    const matchAlt = [...exprLastPart.matchAll(regex1)];
+    if (matchAlt.length > 0 && matchAlt[0].groups && matchAlt[0].groups.metricName) {
+      return matchAlt[0].groups.metricName;
+    }
+
+    const regex2 = /^\s*\b(?<metricName>[^{}(),\s]+)(?={|$)/g;
+    const match = [...exprLastPart.matchAll(regex2)];
+    if (match.length > 0 && match[0].groups && match[0].groups.metricName) {
+      return match[0].groups.metricName;
+    }
+
+    return "";
   }, [exprLastPart]);
 
   const label = useMemo(() => {
@@ -51,7 +62,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   }, [exprLastPart]);
 
   const shouldSuppressAutoSuggestion = (value: string) => {
-    const pattern = /([{(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right)\b)/;
+    const pattern = /([{(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right|by|without|on|ignoring)\b)/i;
     const parts = value.split(/\s+/);
     const partsCount = parts.length;
     const lastPart = parts[partsCount - 1];
@@ -63,12 +74,16 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   };
 
   const context = useMemo(() => {
-    if (!values.beforeCursor || values.beforeCursor.endsWith("}") || shouldSuppressAutoSuggestion(values.beforeCursor)) {
+    const valueBeforeCursor = values.beforeCursor.trim();
+    const endOfClosedBrackets = ["}", ")"].some(char => valueBeforeCursor.endsWith(char));
+    const endOfClosedQuotes = !hasUnclosedQuotes(valueBeforeCursor) && ["`", "'", "\""].some(char => valueBeforeCursor.endsWith(char));
+    if (!values.beforeCursor || endOfClosedBrackets || endOfClosedQuotes || shouldSuppressAutoSuggestion(values.beforeCursor)) {
       return QueryContextType.empty;
     }
 
-    const labelRegexp = /\{[^}]*$/;
-    const labelValueRegexp = new RegExp(`(${escapeRegexp(metric)})?{?.+${escapeRegexp(label)}(=|!=|=~|!~)"?([^"]*)$`, "g");
+    const labelRegexp = /(?:by|without|on|ignoring)\s*\(\s*[^)]*$|\{[^}]*$/i;
+    const patternLabelValue = `(${escapeRegexp(metric)})?{?.+${escapeRegexp(label)}(=|!=|=~|!~)"?([^"]*)$`;
+    const labelValueRegexp = new RegExp(patternLabelValue, "g");
 
     switch (true) {
       case labelValueRegexp.test(values.beforeCursor):
@@ -81,7 +96,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   }, [values, metric, label]);
 
   const valueByContext = useMemo(() => {
-    const wordMatch = values.beforeCursor.match(/([\w_\-.:/]+(?![},]))$/);
+    const wordMatch = values.beforeCursor.match(/([\w_.:]+(?![},]))$/);
     return wordMatch ? wordMatch[0] : "";
   }, [values.beforeCursor]);
 
@@ -119,9 +134,10 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
     // Add quotes around the value if the context is labelValue
     if (context === QueryContextType.labelValue) {
       const quote = "\"";
-      const needsQuote = /(?:=|!=|=~|!~)$/.test(beforeValueByContext);
       valueAfterCursor = valueAfterCursor.replace(/^[^\s"|},]*/, "");
-      insert = `${needsQuote ? quote : ""}${insert}`;
+      const needsOpenQuote = /(?:=|!=|=~|!~)$/.test(beforeValueByContext);
+      const needsCloseQuote = valueAfterCursor.trim()[0] !== "\"";
+      insert = `${needsOpenQuote ? quote : ""}${insert}${needsCloseQuote ? quote : ""}`;
     }
 
     if (context === QueryContextType.label) {

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -520,3 +520,25 @@ export const DownloadIcon = () => (
     <path d="M19 9h-4V3H9v6H5l7 7 7-7zM5 18v2h14v-2H5z"></path>
   </svg>
 );
+
+export const ExpandIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M12 5.83 15.17 9l1.41-1.41L12 3 7.41 7.59 8.83 9zm0 12.34L8.83 15l-1.41 1.41L12 21l4.59-4.59L15.17 15z"
+    ></path>
+  </svg>
+);
+
+export const CollapseIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M7.41 18.59 8.83 20 12 16.83 15.17 20l1.41-1.41L12 14zm9.18-13.18L15.17 4 12 7.17 8.83 4 7.41 5.41 12 10z"
+    ></path>
+  </svg>
+);

app/vmui/packages/vmui/src/components/Main/Popper/style.scss

@@ -32,19 +32,19 @@
 
   &-header {
     display: grid;
-    grid-template-columns: 1fr auto;
-    gap: $padding-small;
+    grid-template-columns: 1fr 25px;
+    gap: $padding-global;
     align-items: center;
     justify-content: space-between;
     background-color: $color-background-block;
-    padding: $padding-small $padding-small $padding-small $padding-global;
+    padding: $padding-small $padding-global;
     border-radius: $border-radius-small $border-radius-small 0 0;
     color: $color-text;
     border-bottom: $border-divider;
     margin-bottom: $padding-global;
-    min-height: 51px;
 
     &__title {
+      font-size: $font-size-small;
       font-weight: bold;
       user-select: none;
     }

app/vmui/packages/vmui/src/components/Table/Table.tsx

@@ -6,11 +6,14 @@ import Tooltip from "../Main/Tooltip/Tooltip";
 import Button from "../Main/Button/Button";
 import { useEffect } from "preact/compat";
 
+type OrderDir = "asc" | "desc"
+
 interface TableProps<T> {
   rows: T[];
   columns: { title?: string, key: keyof Partial<T>, className?: string }[];
   defaultOrderBy: keyof T;
   copyToClipboard?: keyof T;
+  defaultOrderDir?: OrderDir;
   // TODO: Remove when pagination is implemented on the backend.
   paginationOffset: {
     startIndex: number;
@@ -18,9 +21,9 @@ interface TableProps<T> {
   }
 }
 
-const Table = <T extends object>({ rows, columns, defaultOrderBy, copyToClipboard, paginationOffset }: TableProps<T>) => {
+const Table = <T extends object>({ rows, columns, defaultOrderBy, defaultOrderDir, copyToClipboard, paginationOffset }: TableProps<T>) => {
   const [orderBy, setOrderBy] = useState<keyof T>(defaultOrderBy);
-  const [orderDir, setOrderDir] = useState<"asc" | "desc">("desc");
+  const [orderDir, setOrderDir] = useState<OrderDir>(defaultOrderDir || "desc");
   const [copied, setCopied] = useState<number | null>(null);
 
   // const sortedList = useMemo(() => stableSort(rows as [], getComparator(orderDir, orderBy)),

app/vmui/packages/vmui/src/components/Table/TableSettings/TableSettings.tsx

@@ -44,6 +44,14 @@ const TableSettings: FC<TableSettingsProps> = ({
     onChangeColumns(defaultColumns.includes(key) ? defaultColumns.filter(col => col !== key) : [...defaultColumns, key]);
   };
 
+  const toggleAllColumns = () => {
+    if (defaultColumns.length === columns.length) {
+      onChangeColumns([]);
+    } else {
+      onChangeColumns(columns);
+    }
+  };
+
   const handleResetColumns = () => {
     handleClose();
     onChangeColumns(columns);
@@ -105,6 +113,16 @@ const TableSettings: FC<TableSettingsProps> = ({
                 />
               </Tooltip>
             </div>
+            <div
+              className="vm-table-settings-popper-list__item vm-table-settings-popper-list__check_all"
+            >
+              <Checkbox
+                checked={defaultColumns.length === columns.length}
+                onChange={toggleAllColumns}
+                label="Check all"
+                disabled={tableCompact}
+              />
+            </div>
             {columns.map(col => (
               <div
                 className="vm-table-settings-popper-list__item"

app/vmui/packages/vmui/src/components/Table/TableSettings/style.scss

@@ -39,5 +39,9 @@
     &__item {
       font-size: $font-size;
     }
+    &__check_all {
+      padding: 0 0 $padding-global;
+      border-bottom: $border-divider;
+    }
   }
 }

app/vmui/packages/vmui/src/components/TraceQuery/TracingsView.tsx

@@ -1,7 +1,7 @@
 import React, { FC, useState } from "preact/compat";
 import Trace from "./Trace";
 import Button from "../Main/Button/Button";
-import { ArrowDownIcon, CodeIcon, DeleteIcon, DownloadIcon } from "../Main/Icons";
+import { CodeIcon, CollapseIcon, DeleteIcon, DownloadIcon, ExpandIcon } from "../Main/Icons";
 import "./style.scss";
 import NestedNav from "./NestedNav/NestedNav";
 import Alert from "../Main/Alert/Alert";
@@ -89,13 +89,7 @@ const TracingsView: FC<TraceViewProps> = ({ traces, jsonEditor = false, onDelete
               <Tooltip title={expandedTraces.includes(trace.idValue) ? "Collapse All" : "Expand All"}>
                 <Button
                   variant="text"
-                  startIcon={(
-                    <div
-                      className={classNames({
-                        "vm-tracings-view-trace-header__expand-icon": true,
-                        "vm-tracings-view-trace-header__expand-icon_open": expandedTraces.includes(trace.idValue) })}
-                    ><ArrowDownIcon/></div>
-                  )}
+                  startIcon={expandedTraces.includes(trace.idValue) ? <CollapseIcon/> : <ExpandIcon/> }
                   onClick={handleExpandAll(trace)}
                   ariaLabel={expandedTraces.includes(trace.idValue) ? "Collapse All" : "Expand All"}
                 />

app/vmui/packages/vmui/src/constants/palette.ts

@@ -15,7 +15,13 @@ export const darkPalette = {
   "box-shadow-popper": "rgba(0, 0, 0, 0.2) 0px 2px 8px 0px",
   "border-divider": "1px solid rgba(99, 110, 123, 0.5)",
   "color-hover-black": "rgba(0, 0, 0, 0.12)",
-  "color-log-hits-bar": "rgba(255, 255, 255, 0.18)"
+  // log hits chart colors
+  "color-log-hits-bar-0": "rgba(255, 255, 255, 0.18)",
+  "color-log-hits-bar-1": "#FFB74D",
+  "color-log-hits-bar-2": "#81C784",
+  "color-log-hits-bar-3": "#64B5F6",
+  "color-log-hits-bar-4": "#E57373",
+  "color-log-hits-bar-5": "#8a62f0",
 };
 
 export const lightPalette = {
@@ -35,5 +41,12 @@ export const lightPalette = {
   "box-shadow-popper": "rgba(0, 0, 0, 0.1) 0px 2px 8px 0px",
   "border-divider": "1px solid rgba(0, 0, 0, 0.15)",
   "color-hover-black": "rgba(0, 0, 0, 0.06)",
-  "color-log-hits-bar": "rgba(0, 0, 0, 0.18)"
+  // log hits chart colors
+  "color-log-hits-bar-0": "rgba(0, 0, 0, 0.18)",
+  "color-log-hits-bar-1": "#FFB74D",
+  "color-log-hits-bar-2": "#81C784",
+  "color-log-hits-bar-3": "#64B5F6",
+  "color-log-hits-bar-4": "#E57373",
+  "color-log-hits-bar-5": "#8a62f0",
+
 };

app/vmui/packages/vmui/src/contexts/AppContextProvider.tsx

@@ -3,10 +3,11 @@ import { TimeStateProvider } from "../state/time/TimeStateContext";
 import { QueryStateProvider } from "../state/query/QueryStateContext";
 import { CustomPanelStateProvider } from "../state/customPanel/CustomPanelStateContext";
 import { GraphStateProvider } from "../state/graph/GraphStateContext";
+import { DashboardsStateProvider } from "../state/dashboards/DashboardsStateContext";
+import { LogsStateProvider } from "../state/logsPanel/LogsStateContext";
 import { SnackbarProvider } from "./Snackbar";
 
 import { combineComponents } from "../utils/combine-components";
-import { DashboardsStateProvider } from "../state/dashboards/DashboardsStateContext";
 
 const providers = [
   AppStateProvider,
@@ -15,7 +16,8 @@ const providers = [
   CustomPanelStateProvider,
   GraphStateProvider,
   SnackbarProvider,
-  DashboardsStateProvider
+  DashboardsStateProvider,
+  LogsStateProvider
 ];
 
 export default combineComponents(...providers);

app/vmui/packages/vmui/src/hooks/useFetchQueryOptions.tsx

@@ -137,7 +137,7 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context }:
 
   // fetch labels
   useEffect(() => {
-    if (!serverUrl || !metric || context !== QueryContextType.label) {
+    if (!serverUrl || context !== QueryContextType.label) {
       return;
     }
     setLabels([]);
@@ -149,7 +149,7 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context }:
       urlSuffix: "labels",
       setter: setLabels,
       type: TypeData.label,
-      params: getQueryParams({ "match[]": `{__name__="${metricEscaped}"}` })
+      params: getQueryParams(metric ? { "match[]": `{__name__="${metricEscaped}"}` } : undefined)
     });
 
     return () => abortControllerRef.current?.abort();
@@ -157,20 +157,23 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context }:
 
   // fetch labelValues
   useEffect(() => {
-    if (!serverUrl || !metric || !label || context !== QueryContextType.labelValue) {
+    if (!serverUrl || !label || context !== QueryContextType.labelValue) {
       return;
     }
     setLabelValues([]);
 
     const metricEscaped = escapeDoubleQuotes(metric);
     const valueReEscaped = escapeDoubleQuotes(escapeRegexp(value));
+    const matchMetric = metric ? `__name__="${metricEscaped}"` : "";
+    const matchLabel = `${label}=~".*${valueReEscaped}.*"`;
+    const matchValue = [matchMetric, matchLabel].filter(Boolean).join(",");
 
     fetchData({
       value,
       urlSuffix: `label/${label}/values`,
       setter: setLabelValues,
       type: TypeData.labelValue,
-      params: getQueryParams({ "match[]": `{__name__="${metricEscaped}", ${label}=~".*${valueReEscaped}.*"}` })
+      params: getQueryParams({ "match[]": `{${matchValue}}` })
     });
 
     return () => abortControllerRef.current?.abort();

app/vmui/packages/vmui/src/hooks/useSortedCategories.ts

@@ -1,5 +1,5 @@
-import { useMemo } from "preact/compat";
-import { MetricBase } from "../api/types";
+import {useMemo} from "preact/compat";
+import {MetricBase} from "../api/types";
 
 export type MetricCategory = {
   key: string;
@@ -10,7 +10,7 @@ export const getColumns = (data: MetricBase[]): MetricCategory[] => {
   const columns: { [key: string]: { options: Set<string> } } = {};
   data.forEach(d =>
     Object.entries(d.metric).forEach(e =>
-      columns[e[0]] ? columns[e[0]].options.add(e[1]) : columns[e[0]] = { options: new Set([e[1]]) }
+      columns[e[0]] ? columns[e[0]].options.add(e[1]) : columns[e[0]] = {options: new Set([e[1]])}
     )
   );
 
@@ -22,7 +22,8 @@ export const getColumns = (data: MetricBase[]): MetricCategory[] => {
 
 export const useSortedCategories = (data: MetricBase[], displayColumns?: string[]): MetricCategory[] => (
   useMemo(() => {
+    if (!displayColumns) return [];
     const sortedColumns = getColumns(data);
-    return displayColumns ? sortedColumns.filter(col => displayColumns.includes(col.key)) : sortedColumns;
+    return sortedColumns.filter(col => displayColumns.includes(col.key));
   }, [data, displayColumns])
 );

app/vmui/packages/vmui/src/layouts/LogsLayout/ControlsLogsLayout.tsx

@@ -3,6 +3,7 @@ import classNames from "classnames";
 import GlobalSettings from "../../components/Configurators/GlobalSettings/GlobalSettings";
 import { ControlsProps } from "../Header/HeaderControls/HeaderControls";
 import { TimeSelector } from "../../components/Configurators/TimeRangeSettings/TimeSelector/TimeSelector";
+import TenantsFields from "../../components/Configurators/GlobalSettings/TenantsConfiguration/TenantsFields";
 
 const ControlsLogsLayout: FC<ControlsProps> = ({ isMobile }) => {
 
@@ -13,6 +14,7 @@ const ControlsLogsLayout: FC<ControlsProps> = ({ isMobile }) => {
         "vm-header-controls_mobile": isMobile,
       })}
     >
+      <TenantsFields/>
       <TimeSelector/>
       <GlobalSettings/>
     </div>

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx

@@ -1,4 +1,4 @@
-import React, { FC, useCallback, useEffect } from "preact/compat";
+import React, { FC, useCallback, useEffect, useState } from "preact/compat";
 import ExploreLogsBody from "./ExploreLogsBody/ExploreLogsBody";
 import useStateSearchParams from "../../hooks/useStateSearchParams";
 import useSearchParamsFromObject from "../../hooks/useSearchParamsFromObject";
@@ -9,7 +9,6 @@ import Alert from "../../components/Main/Alert/Alert";
 import ExploreLogsHeader from "./ExploreLogsHeader/ExploreLogsHeader";
 import "./style.scss";
 import { ErrorTypes, TimeParams } from "../../types";
-import { useState } from "react";
 import { useTimeState } from "../../state/time/TimeStateContext";
 import { getFromStorage, saveToStorage } from "../../utils/storage";
 import ExploreLogsBarChart from "./ExploreLogsBarChart/ExploreLogsBarChart";
@@ -27,11 +26,12 @@ const ExploreLogs: FC = () => {
 
   const [limit, setLimit] = useStateSearchParams(defaultLimit, "limit");
   const [query, setQuery] = useStateSearchParams("*", "query");
+  const [tmpQuery, setTmpQuery] = useState("");
   const [period, setPeriod] = useState<TimeParams>(periodState);
+  const [queryError, setQueryError] = useState<ErrorTypes | string>("");
+
   const { logs, isLoading, error, fetchLogs } = useFetchLogs(serverUrl, query, limit);
   const { fetchLogHits, ...dataLogHits } = useFetchLogHits(serverUrl, query);
-  const [queryError, setQueryError] = useState<ErrorTypes | string>("");
-  const [markdownParsing, setMarkdownParsing] = useState(getFromStorage("LOGS_MARKDOWN") === "true");
 
   const getPeriod = useCallback(() => {
     const relativeTimeOpts = relativeTimeOptions.find(d => d.id === relativeTime);
@@ -45,12 +45,13 @@ const ExploreLogs: FC = () => {
       setQueryError(ErrorTypes.validQuery);
       return;
     }
+    setQueryError("");
 
     const newPeriod = getPeriod();
     setPeriod(newPeriod);
-    fetchLogs(newPeriod);
-    fetchLogHits(newPeriod);
-
+    fetchLogs(newPeriod).then((isSuccess) => {
+      isSuccess && fetchLogHits(newPeriod);
+    }).catch(e => e);
     setSearchParamsFromKeys( {
       query,
       "g0.range_input": duration,
@@ -65,9 +66,13 @@ const ExploreLogs: FC = () => {
     saveToStorage("LOGS_LIMIT", `${limit}`);
   };
 
-  const handleChangeMarkdownParsing = (val: boolean) => {
-    saveToStorage("LOGS_MARKDOWN", `${val}`);
-    setMarkdownParsing(val);
+  const handleApplyFilter = (val: string) => {
+    setQuery(prev => `_stream: ${val === "other" ? "{}" : val} AND (${prev})`);
+  };
+
+  const handleUpdateQuery = () => {
+    setQuery(tmpQuery);
+    handleRunQuery();
   };
 
   useEffect(() => {
@@ -75,35 +80,32 @@ const ExploreLogs: FC = () => {
   }, [periodState]);
 
   useEffect(() => {
-    setQueryError("");
+    handleRunQuery();
+    setTmpQuery(query);
   }, [query]);
 
   return (
     <div className="vm-explore-logs">
       <ExploreLogsHeader
-        query={query}
+        query={tmpQuery}
         error={queryError}
         limit={limit}
-        markdownParsing={markdownParsing}
-        onChange={setQuery}
+        onChange={setTmpQuery}
         onChangeLimit={handleChangeLimit}
-        onRun={handleRunQuery}
-        onChangeMarkdownParsing={handleChangeMarkdownParsing}
+        onRun={handleUpdateQuery}
       />
-      {isLoading && <Spinner />}
+      {isLoading && <Spinner message={"Loading logs..."}/>}
       {error && <Alert variant="error">{error}</Alert>}
       {!error && (
         <ExploreLogsBarChart
           {...dataLogHits}
           query={query}
           period={period}
+          onApplyFilter={handleApplyFilter}
           isLoading={isLoading ? false : dataLogHits.isLoading}
         />
       )}
-      <ExploreLogsBody
-        data={logs}
-        markdownParsing={markdownParsing}
-      />
+      <ExploreLogsBody data={logs}/>
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBarChart/ExploreLogsBarChart.tsx

@@ -17,19 +17,34 @@ interface Props {
   period: TimeParams;
   error?: string;
   isLoading: boolean;
+  onApplyFilter: (value: string) => void;
 }
 
-const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading }) => {
+const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading, onApplyFilter }) => {
   const { isMobile } = useDeviceDetect();
   const timeDispatch = useTimeDispatch();
 
+  const getXAxis = (timestamps: string[]): number[] => {
+    return (timestamps.map(t => t ? dayjs(t).unix() : null)
+      .filter(Boolean) as number[])
+      .sort((a, b) => a - b);
+  };
+
+  const getYAxes = (logHits: LogHits[], timestamps: string[]) => {
+    return logHits.map(hits => {
+      return timestamps.map(t => {
+        const index = hits.timestamps.findIndex(ts => ts === t);
+        return index === -1 ? null : hits.values[index] || null;
+      });
+    });
+  };
+
   const data = useMemo(() => {
-    const hits = logHits[0];
-    if (!hits) return [[], []] as AlignedData;
-    const { values, timestamps } = hits;
-    const xAxis = timestamps.map(t => t ? dayjs(t).unix() : null).filter(Boolean);
-    const yAxis = values.map(v => v || null);
-    return [xAxis, yAxis] as AlignedData;
+    if (!logHits.length) return [[], []] as AlignedData;
+    const timestamps = Array.from(new Set(logHits.map(l => l.timestamps).flat()));
+    const xAxis = getXAxis(timestamps);
+    const yAxes = getYAxes(logHits, timestamps);
+    return [xAxis, ...yAxes] as AlignedData;
   }, [logHits]);
 
   const noDataMessage: string = useMemo(() => {
@@ -57,7 +72,10 @@ const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading }) =
         "vm-block_mobile": isMobile,
       })}
     >
-      {isLoading && <Spinner containerStyles={{ position: "absolute" }}/>}
+      {isLoading && <Spinner
+        message={"Loading hits stats..."}
+        containerStyles={{ position: "absolute" }}
+      />}
       {!error && noDataMessage && (
         <div className="vm-explore-logs-chart__empty">
           <Alert variant="info">{noDataMessage}</Alert>
@@ -72,9 +90,11 @@ const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading }) =
 
       {data && (
         <BarHitsChart
+          logHits={logHits}
           data={data}
           period={period}
           setPeriod={setPeriod}
+          onApplyFilter={onApplyFilter}
         />
       )}
     </section>

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBarChart/style.scss

@@ -5,7 +5,6 @@
   display: flex;
   align-items: center;
   justify-content: center;
-  height: 200px;
   padding: 0 0 0 $padding-small !important;
   width: calc(100vw - ($padding-medium * 2));
 

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx

@@ -1,4 +1,4 @@
-import React, { FC, useState, useMemo } from "preact/compat";
+import React, { FC, useState, useMemo, useRef } from "preact/compat";
 import JsonView from "../../../components/Views/JsonView/JsonView";
 import { CodeIcon, ListIcon, TableIcon } from "../../../components/Main/Icons";
 import Tabs from "../../../components/Main/Tabs/Tabs";
@@ -19,7 +19,6 @@ import { marked } from "marked";
 
 export interface ExploreLogBodyProps {
   data: Logs[];
-  markdownParsing: boolean;
 }
 
 enum DisplayType {
@@ -34,10 +33,11 @@ const tabs = [
   { label: "JSON", value: DisplayType.json, icon: <CodeIcon/> },
 ];
 
-const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, markdownParsing }) => {
+const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data }) => {
   const { isMobile } = useDeviceDetect();
   const { timezone } = useTimeState();
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
+  const groupSettingsRef = useRef<HTMLDivElement>(null);
 
   const [activeTab, setActiveTab] = useStateSearchParams(DisplayType.group, "view");
   const [displayColumns, setDisplayColumns] = useState<string[]>([]);
@@ -88,6 +88,9 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, markdownParsing }) =>
             items={tabs}
             onChange={handleChangeTab}
           />
+          <div className="vm-explore-logs-body-header__log-info">
+            Total logs returned: <b>{data.length}</b>
+          </div>
         </div>
         {activeTab === DisplayType.table && (
           <div className="vm-explore-logs-body-header__settings">
@@ -100,6 +103,12 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, markdownParsing }) =>
             />
           </div>
         )}
+        {activeTab === DisplayType.group && (
+          <div
+            className="vm-explore-logs-body-header__settings"
+            ref={groupSettingsRef}
+          />
+        )}
       </div>
 
       <div
@@ -123,7 +132,7 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, markdownParsing }) =>
               <GroupLogs
                 logs={logs}
                 columns={columns}
-                markdownParsing={markdownParsing}
+                settingsRef={groupSettingsRef}
               />
             )}
             {activeTab === DisplayType.json && (

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/TableLogs.tsx

@@ -39,7 +39,8 @@ const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, col
 
 
   const filteredColumns = useMemo(() => {
-    if (!displayColumns?.length || tableCompact) return tableColumns;
+    if (tableCompact) return tableColumns;
+    if (!displayColumns?.length) return [];
     return tableColumns.filter(c => displayColumns.includes(c.key as string));
   }, [tableColumns, displayColumns, tableCompact]);
 
@@ -48,7 +49,8 @@ const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, col
       <Table
         rows={logs}
         columns={filteredColumns}
-        defaultOrderBy={"_vmui_time"}
+        defaultOrderBy={"_time"}
+        defaultOrderDir={"desc"}
         copyToClipboard={"_vmui_data"}
         paginationOffset={{ startIndex: 0, endIndex: Infinity }}
       />

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/style.scss

@@ -13,6 +13,14 @@
       align-items: center;
       gap: $padding-small;
     }
+
+    &__log-info {
+      flex-grow: 1;
+      text-align: right;
+      padding-right: $padding-global;
+      color: $color-text-secondary;
+
+    }
   }
 
   &__empty {

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/ExploreLogsHeader.tsx

@@ -6,28 +6,23 @@ import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import Button from "../../../components/Main/Button/Button";
 import QueryEditor from "../../../components/Configurators/QueryEditor/QueryEditor";
 import TextField from "../../../components/Main/TextField/TextField";
-import Switch from "../../../components/Main/Switch/Switch";
 
 export interface ExploreLogHeaderProps {
   query: string;
   limit: number;
   error?: string;
-  markdownParsing: boolean;
   onChange: (val: string) => void;
   onChangeLimit: (val: number) => void;
   onRun: () => void;
-  onChangeMarkdownParsing: (val: boolean) => void;
 }
 
 const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
   query,
   limit,
   error,
-  markdownParsing,
   onChange,
   onChangeLimit,
   onRun,
-  onChangeMarkdownParsing,
 }) => {
   const { isMobile } = useDeviceDetect();
 
@@ -78,14 +73,7 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
         />
       </div>
       <div className="vm-explore-logs-header-bottom">
-        <div className="vm-explore-logs-header-bottom-contols">
-          <Switch
-            label={"Markdown parsing"}
-            value={markdownParsing}
-            onChange={onChangeMarkdownParsing}
-            fullWidth={isMobile}
-          />
-        </div>
+        <div className="vm-explore-logs-header-bottom-contols"></div>
         <div className="vm-explore-logs-header-bottom-helpful">
           <a
             className="vm-link vm-link_with-icon"

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx

@@ -1,4 +1,4 @@
-import React, { FC, useEffect, useMemo } from "preact/compat";
+import React, { FC, useCallback, useEffect, useMemo, useRef } from "preact/compat";
 import { MouseEvent, useState } from "react";
 import "./style.scss";
 import { Logs } from "../../../api/types";
@@ -9,89 +9,213 @@ import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
 import GroupLogsItem from "./GroupLogsItem";
 import { useAppState } from "../../../state/common/StateContext";
 import classNames from "classnames";
+import Button from "../../../components/Main/Button/Button";
+import { CollapseIcon, ExpandIcon, StorageIcon } from "../../../components/Main/Icons";
+import Popper from "../../../components/Main/Popper/Popper";
+import TextField from "../../../components/Main/TextField/TextField";
+import useBoolean from "../../../hooks/useBoolean";
+import useStateSearchParams from "../../../hooks/useStateSearchParams";
+import { useSearchParams } from "react-router-dom";
+
+const WITHOUT_GROUPING = "No Grouping";
 
 interface TableLogsProps {
   logs: Logs[];
   columns: string[];
-  markdownParsing: boolean;
+  settingsRef: React.Ref<HTMLDivElement>;
 }
 
-const GroupLogs: FC<TableLogsProps> = ({ logs, markdownParsing }) => {
+const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
   const { isDarkTheme } = useAppState();
   const copyToClipboard = useCopyToClipboard();
+  const [searchParams, setSearchParams] = useSearchParams();
 
+  const [expandGroups, setExpandGroups] = useState<boolean[]>([]);
+  const [groupBy, setGroupBy] = useStateSearchParams("_stream", "groupBy");
   const [copied, setCopied] = useState<string | null>(null);
+  const [searchKey, setSearchKey] = useState("");
+  const optionsButtonRef = useRef<HTMLDivElement>(null);
+
+  const {
+    value: openOptions,
+    toggle: toggleOpenOptions,
+    setFalse: handleCloseOptions,
+  } = useBoolean(false);
+
+  const expandAll = useMemo(() => expandGroups.every(Boolean), [expandGroups]);
+
+  const logsKeys = useMemo(() => {
+    const excludeKeys = ["_msg", "_time", "_vmui_time", "_vmui_data", "_vmui_markdown"];
+    const uniqKeys = Array.from(new Set(logs.map(l => Object.keys(l)).flat()));
+    const keys = [WITHOUT_GROUPING, ...uniqKeys.filter(k => !excludeKeys.includes(k))];
+
+    if (!searchKey) return keys;
+    try {
+      const regexp = new RegExp(searchKey, "i");
+      const found = keys.filter((item) => regexp.test(item));
+      return found.sort((a,b) => (a.match(regexp)?.index || 0) - (b.match(regexp)?.index || 0));
+    } catch (e) {
+      return [];
+    }
+  }, [logs, searchKey]);
 
   const groupData = useMemo(() => {
-    return groupByMultipleKeys(logs, ["_stream"]).map((item) => {
-      const streamValue = item.values[0]?._stream || "";
-      const pairs = streamValue.slice(1, -1).match(/(?:[^\\,]+|\\,)+?(?=,|$)/g) || [streamValue];
+    return groupByMultipleKeys(logs, [groupBy]).map((item) => {
+      const streamValue = item.values[0]?.[groupBy] || "";
+      const pairs = /^{.+}$/.test(streamValue)
+        ? streamValue.slice(1, -1).match(/(\\.|[^,])+/g) || [streamValue]
+        : [streamValue];
       return {
         ...item,
         pairs: pairs.filter(Boolean),
       };
     });
-  }, [logs]);
+  }, [logs, groupBy]);
 
-  const handleClickByPair = (pair: string) => async (e: MouseEvent<HTMLDivElement>) => {
+  const handleClickByPair = (value: string) => async (e: MouseEvent<HTMLDivElement>) => {
     e.stopPropagation();
-    const isCopied = await copyToClipboard(`${pair.replace(/=/, ": ")}`);
+    const isKeyValue = /(.+)?=(".+")/.test(value);
+    const copyValue = isKeyValue ? `${value.replace(/=/, ": ")}` : `${groupBy}: "${value}"`;
+    const isCopied = await copyToClipboard(copyValue);
     if (isCopied) {
-      setCopied(pair);
+      setCopied(value);
     }
   };
 
+  const handleSelectGroupBy = (key: string) => () => {
+    setGroupBy(key);
+    searchParams.set("groupBy", key);
+    setSearchParams(searchParams);
+    handleCloseOptions();
+  };
+
+  const handleToggleExpandAll = useCallback(() => {
+    setExpandGroups(new Array(groupData.length).fill(!expandAll));
+  }, [expandAll]);
+
+  const handleChangeExpand = (i: number) => (value: boolean) => {
+    setExpandGroups((prev) => {
+      const newExpandGroups = [...prev];
+      newExpandGroups[i] = value;
+      return newExpandGroups;
+    });
+
+  };
+
   useEffect(() => {
     if (copied === null) return;
     const timeout = setTimeout(() => setCopied(null), 2000);
     return () => clearTimeout(timeout);
   }, [copied]);
 
+  useEffect(() => {
+    setExpandGroups(new Array(groupData.length).fill(true));
+  }, [groupData]);
+
   return (
-    <div className="vm-group-logs">
-      {groupData.map((item) => (
-        <div
-          className="vm-group-logs-section"
-          key={item.keys.join("")}
-        >
-          <Accordion
-            defaultExpanded={true}
-            title={(
-              <div className="vm-group-logs-section-keys">
-                <span className="vm-group-logs-section-keys__title">Group by _stream:</span>
-                {item.pairs.map((pair) => (
-                  <Tooltip
-                    title={copied === pair ? "Copied" : "Copy to clipboard"}
-                    key={`${item.keys.join("")}_${pair}`}
-                    placement={"top-center"}
-                  >
-                    <div
-                      className={classNames({
-                        "vm-group-logs-section-keys__pair": true,
-                        "vm-group-logs-section-keys__pair_dark": isDarkTheme
-                      })}
-                      onClick={handleClickByPair(pair)}
+    <>
+      <div className="vm-group-logs">
+        {groupData.map((item, i) => (
+          <div
+            className="vm-group-logs-section"
+            key={item.keys.join("")}
+          >
+            <Accordion
+              key={String(expandGroups[i])}
+              defaultExpanded={expandGroups[i]}
+              onChange={handleChangeExpand(i)}
+              title={groupBy !== WITHOUT_GROUPING && (
+                <div className="vm-group-logs-section-keys">
+                  <span className="vm-group-logs-section-keys__title">Group by <code>{groupBy}</code>:</span>
+                  {item.pairs.map((pair) => (
+                    <Tooltip
+                      title={copied === pair ? "Copied" : "Copy to clipboard"}
+                      key={`${item.keys.join("")}_${pair}`}
+                      placement={"top-center"}
                     >
-                      {pair}
-                    </div>
-                  </Tooltip>
+                      <div
+                        className={classNames({
+                          "vm-group-logs-section-keys__pair": true,
+                          "vm-group-logs-section-keys__pair_dark": isDarkTheme
+                        })}
+                        onClick={handleClickByPair(pair)}
+                      >
+                        {pair}
+                      </div>
+                    </Tooltip>
+                  ))}
+                  <span className="vm-group-logs-section-keys__count">{item.values.length} entries</span>
+                </div>
+              )}
+            >
+              <div className="vm-group-logs-section-rows">
+                {item.values.map((value) => (
+                  <GroupLogsItem
+                    key={`${value._msg}${value._time}`}
+                    log={value}
+                  />
                 ))}
               </div>
-            )}
-          >
-            <div className="vm-group-logs-section-rows">
-              {item.values.map((value) => (
-                <GroupLogsItem
-                  key={`${value._msg}${value._time}`}
-                  log={value}
-                  markdownParsing={markdownParsing}
-                />
-              ))}
+            </Accordion>
+          </div>
+        ))}
+      </div>
+
+
+      {settingsRef.current && React.createPortal((
+        <div className="vm-group-logs-header">
+          <Tooltip title={expandAll ? "Collapse All" : "Expand All"}>
+            <Button
+              variant="text"
+              startIcon={expandAll ? <CollapseIcon/> : <ExpandIcon/> }
+              onClick={handleToggleExpandAll}
+              ariaLabel={expandAll ? "Collapse All" : "Expand All"}
+            />
+          </Tooltip>
+          <Tooltip title={"Group by"}>
+            <div ref={optionsButtonRef}>
+              <Button
+                variant="text"
+                startIcon={<StorageIcon/> }
+                onClick={toggleOpenOptions}
+                ariaLabel={"Group by"}
+              />
             </div>
-          </Accordion>
+          </Tooltip>
+          {
+            <Popper
+              open={openOptions}
+              placement="bottom-right"
+              onClose={handleCloseOptions}
+              buttonRef={optionsButtonRef}
+            >
+              <div className="vm-list vm-group-logs-header-keys">
+                <div className="vm-group-logs-header-keys__search">
+                  <TextField
+                    label="Search key"
+                    value={searchKey}
+                    onChange={setSearchKey}
+                    type="search"
+                  />
+                </div>
+                {logsKeys.map(id => (
+                  <div
+                    className={classNames({
+                      "vm-list-item": true,
+                      "vm-list-item_active": id === groupBy
+                    })}
+                    key={id}
+                    onClick={handleSelectGroupBy(id)}
+                  >
+                    {id}
+                  </div>
+                ))}
+              </div>
+            </Popper>
+          }
         </div>
-      ))}
-    </div>
+      ), settingsRef.current)}
+    </>
   );
 };
 

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx

@@ -7,19 +7,21 @@ import Tooltip from "../../../components/Main/Tooltip/Tooltip";
 import { ArrowDownIcon, CopyIcon } from "../../../components/Main/Icons";
 import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
 import classNames from "classnames";
+import { useLogsState } from "../../../state/logsPanel/LogsStateContext";
 
 interface Props {
   log: Logs;
-  markdownParsing: boolean;
 }
 
-const GroupLogsItem: FC<Props> = ({ log, markdownParsing }) => {
+const GroupLogsItem: FC<Props> = ({ log }) => {
   const {
     value: isOpenFields,
     toggle: toggleOpenFields,
   } = useBoolean(false);
 
-  const excludeKeys = ["_stream", "_msg", "_time", "_vmui_time", "_vmui_data", "_vmui_markdown"];
+  const { markdownParsing } = useLogsState();
+
+  const excludeKeys = ["_msg", "_vmui_time", "_vmui_data", "_vmui_markdown"];
   const fields = Object.entries(log).filter(([key]) => !excludeKeys.includes(key));
   const hasFields = fields.length > 0;
 

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss

@@ -3,6 +3,22 @@
 .vm-group-logs {
   margin-top: calc(-1 * $padding-medium);
 
+  &-header {
+    display: flex;
+    align-items: center;
+    justify-content: flex-end;
+    gap: $padding-global;
+
+    &-keys {
+      max-height: 300px;
+      overflow: auto;
+
+      &__search {
+        padding: $padding-small;
+      }
+    }
+  }
+
   &-section {
     &-keys {
       display: flex;
@@ -14,6 +30,24 @@
 
       &__title {
         font-weight: bold;
+
+        code {
+          font-family: monospace;
+          &:before {
+            content: "\"";
+          }
+          &:after {
+            content: "\"";
+          }
+        }
+      }
+
+      &__count {
+        flex-grow: 1;
+        text-align: right;
+        font-size: $font-size-small;
+        color: $color-text-secondary;
+        padding-right: calc($padding-large * 3);
       }
 
       &__pair {

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogHits.ts

@@ -4,8 +4,11 @@ import { ErrorTypes, TimeParams } from "../../../types";
 import { LogHits } from "../../../api/types";
 import dayjs from "dayjs";
 import { LOGS_BARS_VIEW } from "../../../constants/logs";
+import { useSearchParams } from "react-router-dom";
 
 export const useFetchLogHits = (server: string, query: string) => {
+  const [searchParams] = useSearchParams();
+
   const [logHits, setLogHits] = useState<LogHits[]>([]);
   const [isLoading, setIsLoading] = useState<{[key: number]: boolean;}>([]);
   const [error, setError] = useState<ErrorTypes | string>();
@@ -22,15 +25,55 @@ export const useFetchLogHits = (server: string, query: string) => {
     return {
       signal,
       method: "POST",
+      headers: {
+        AccountID: searchParams.get("accountID") || "0",
+        ProjectID: searchParams.get("projectID") || "0",
+      },
       body: new URLSearchParams({
         query: query.trim(),
         step: `${step}ms`,
         start: start.toISOString(),
         end: end.toISOString(),
+        field: "_stream" // In the future, this field can be made configurable
+
       })
     };
   };
 
+  const accumulateHits = (resultHit: LogHits, hit: LogHits) => {
+    resultHit.total = (resultHit.total || 0) + (hit.total || 0);
+    hit.timestamps.forEach((timestamp, i) => {
+      const index = resultHit.timestamps.findIndex(t => t === timestamp);
+      if (index === -1) {
+        resultHit.timestamps.push(timestamp);
+        resultHit.values.push(hit.values[i]);
+      } else {
+        resultHit.values[index] += hit.values[i];
+      }
+    });
+    return resultHit;
+  };
+
+  const getHitsWithTop = (hits: LogHits[]) => {
+    const topN = 5;
+    const defaultHit = { fields: {}, timestamps: [], values: [], total: 0 };
+
+    const hitsByTotal = hits.sort((a, b) => (b.total || 0) - (a.total || 0));
+    const result = [];
+
+    const otherHits: LogHits = hitsByTotal.slice(topN).reduce(accumulateHits, defaultHit);
+    if (otherHits.total) {
+      result.push(otherHits);
+    }
+
+    const topHits: LogHits[] = hitsByTotal.slice(0, topN);
+    if (topHits.length) {
+      result.push(...topHits);
+    }
+
+    return result;
+  };
+
   const fetchLogHits = useCallback(async (period: TimeParams) => {
     abortControllerRef.current.abort();
     abortControllerRef.current = new AbortController();
@@ -59,7 +102,7 @@ export const useFetchLogHits = (server: string, query: string) => {
         setError(error);
       }
 
-      setLogHits(!hits ? [] : hits);
+      setLogHits(!hits ? [] : getHitsWithTop(hits));
     } catch (e) {
       if (e instanceof Error && e.name !== "AbortError") {
         setError(String(e));
@@ -68,7 +111,7 @@ export const useFetchLogHits = (server: string, query: string) => {
       }
     }
     setIsLoading(prev => ({ ...prev, [id]: false }));
-  }, [url, query]);
+  }, [url, query, searchParams]);
 
   return {
     logHits,

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogs.ts

@@ -3,8 +3,11 @@ import { getLogsUrl } from "../../../api/logs";
 import { ErrorTypes, TimeParams } from "../../../types";
 import { Logs } from "../../../api/types";
 import dayjs from "dayjs";
+import { useSearchParams } from "react-router-dom";
 
 export const useFetchLogs = (server: string, query: string, limit: number) => {
+  const [searchParams] = useSearchParams();
+
   const [logs, setLogs] = useState<Logs[]>([]);
   const [isLoading, setIsLoading] = useState<{[key: number]: boolean;}>([]);
   const [error, setError] = useState<ErrorTypes | string>();
@@ -16,7 +19,9 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
     signal,
     method: "POST",
     headers: {
-      "Accept": "application/stream+json",
+      Accept: "application/stream+json",
+      AccountID: searchParams.get("accountID") || "0",
+      ProjectID: searchParams.get("projectID") || "0",
     },
     body: new URLSearchParams({
       query: query.trim(),
@@ -52,21 +57,24 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
         setError(text);
         setLogs([]);
         setIsLoading(prev => ({ ...prev, [id]: false }));
-        return;
+        return false;
       }
 
       const lines = text.split("\n").filter(line => line).slice(0, limit);
       const data = lines.map(parseLineToJSON).filter(line => line) as Logs[];
       setLogs(data);
+      setIsLoading(prev => ({ ...prev, [id]: false }));
+      return true;
     } catch (e) {
+      setIsLoading(prev => ({ ...prev, [id]: false }));
       if (e instanceof Error && e.name !== "AbortError") {
         setError(String(e));
         console.error(e);
         setLogs([]);
       }
+      return false;
     }
-    setIsLoading(prev => ({ ...prev, [id]: false }));
-  }, [url, query, limit]);
+  }, [url, query, limit, searchParams]);
 
   return {
     logs,

app/vmui/packages/vmui/src/state/logsPanel/LogsStateContext.tsx

@@ -0,0 +1,24 @@
+import React, { createContext, FC, useContext, useMemo, useReducer } from "preact/compat";
+import { LogsAction, LogsState, initialLogsState, reducer } from "./reducer";
+import { Dispatch } from "react";
+
+type LogsStateContextType = { state: LogsState, dispatch: Dispatch<LogsAction> };
+
+export const LogsStateContext = createContext<LogsStateContextType>({} as LogsStateContextType);
+
+export const useLogsState = (): LogsState => useContext(LogsStateContext).state;
+export const useLogsDispatch = (): Dispatch<LogsAction> => useContext(LogsStateContext).dispatch;
+
+export const LogsStateProvider: FC = ({ children }) => {
+  const [state, dispatch] = useReducer(reducer, initialLogsState);
+
+  const contextValue = useMemo(() => {
+    return { state, dispatch };
+  }, [state, dispatch]);
+
+  return <LogsStateContext.Provider value={contextValue}>
+    {children}
+  </LogsStateContext.Provider>;
+};
+
+

app/vmui/packages/vmui/src/state/logsPanel/reducer.ts

@@ -0,0 +1,26 @@
+import { getFromStorage, saveToStorage } from "../../utils/storage";
+
+export interface LogsState {
+  markdownParsing: boolean;
+}
+
+export type LogsAction =
+  | { type: "SET_MARKDOWN_PARSING", payload: boolean }
+
+
+export const initialLogsState: LogsState = {
+  markdownParsing: getFromStorage("LOGS_MARKDOWN") === "true",
+};
+
+export function reducer(state: LogsState, action: LogsAction): LogsState {
+  switch (action.type) {
+    case "SET_MARKDOWN_PARSING":
+      saveToStorage("LOGS_MARKDOWN", `${ action.payload}`);
+      return {
+        ...state,
+        markdownParsing: action.payload
+      };
+    default:
+      throw new Error();
+  }
+}

app/vmui/packages/vmui/src/utils/regexp.ts

@@ -8,6 +8,6 @@ export const escapeDoubleQuotes = (s: string) => {
 };
 
 export const hasUnclosedQuotes = (str: string) => {
-  const matches = str.match(/"/g);
+  const matches = str.match(/["`']/g);
   return matches ? matches.length % 2 !== 0 : false;
 };

app/vmui/packages/vmui/src/utils/uplot/bars.ts

@@ -1,14 +0,0 @@
-import uPlot from "uplot";
-import { LOGS_BARS_VIEW } from "../../constants/logs";
-
-export const barPaths = (
-  u: uPlot,
-  seriesIdx: number,
-  idx0: number,
-  idx1: number,
-): uPlot.Series.Paths | null => {
-  const barSize = (u.under.clientWidth/LOGS_BARS_VIEW ) - 1;
-  const barsPathBuilderFactory = uPlot?.paths?.bars?.({ size: [0.96, barSize] });
-  return barsPathBuilderFactory ? barsPathBuilderFactory(u, seriesIdx, idx0, idx1) : null;
-};
-

app/vmui/packages/vmui/src/utils/uplot/paths.ts

@@ -0,0 +1,38 @@
+import uPlot, { Series } from "uplot";
+import { LOGS_BARS_VIEW } from "../../constants/logs";
+import { GRAPH_STYLES } from "../../components/Chart/BarHitsChart/types";
+
+const barPaths = (
+  u: uPlot,
+  seriesIdx: number,
+  idx0: number,
+  idx1: number,
+): Series.Paths | null => {
+  const barSize = (u.under.clientWidth/LOGS_BARS_VIEW ) - 1;
+  const pathBuilderFactory = uPlot?.paths?.bars?.({ size: [0.96, barSize] });
+  return pathBuilderFactory ? pathBuilderFactory(u, seriesIdx, idx0, idx1) : null;
+};
+
+const lineSteppedPaths = (
+  u: uPlot,
+  seriesIdx: number,
+  idx0: number,
+  idx1: number,
+): Series.Paths | null => {
+  const pathBuilderFactory = uPlot?.paths?.stepped?.({ align: 1 });
+  return pathBuilderFactory ? pathBuilderFactory(u, seriesIdx, idx0, idx1) : null;
+};
+
+const getSeriesPaths = (type?: GRAPH_STYLES) => {
+  switch (type) {
+    case GRAPH_STYLES.BAR:
+      return barPaths;
+    case GRAPH_STYLES.LINE_STEPPED:
+      return lineSteppedPaths;
+    default:
+      return;
+  }
+};
+
+export default getSeriesPaths;
+

app/vmui/packages/vmui/src/utils/uplot/stack.ts

@@ -0,0 +1,33 @@
+// taken from https://github.com/leeoniya/uPlot/blob/master/demos/stack.js
+
+import { AlignedData, Band } from "uplot";
+
+function stack(data: AlignedData, omit: (i: number) => boolean) {
+  const data2 = [];
+  let bands = [];
+  const d0Len = data[0].length;
+  const accum = Array(d0Len);
+
+  for (let i = 0; i < d0Len; i++)
+    accum[i] = 0;
+
+  for (let i = 1; i < data.length; i++)
+    data2.push(omit(i) ? data[i] : data[i].map((v, i) => (accum[i] += +(v ?? 0))));
+
+  for (let i = 1; i < data.length; i++)
+    !omit(i) && bands.push({
+      series: [
+        data.findIndex((_s, j) => j > i && !omit(j)),
+        i,
+      ],
+    });
+
+  bands = bands.filter(b => b.series[1] > -1);
+
+  return {
+    data: [data[0]].concat(data2) as AlignedData,
+    bands: bands as Band[],
+  };
+}
+
+export default stack;

dashboards/vm/vmagent.json

@@ -3548,7 +3548,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3564,7 +3565,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 13
+            "y": 37
           },
           "id": 48,
           "options": {
@@ -3654,7 +3655,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3670,7 +3672,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 13
+            "y": 37
           },
           "id": 76,
           "options": {
@@ -3758,7 +3760,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3774,7 +3777,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 20
+            "y": 44
           },
           "id": 132,
           "options": {
@@ -3864,7 +3867,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3880,7 +3884,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 20
+            "y": 44
           },
           "id": 133,
           "options": {
@@ -3969,7 +3973,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3985,7 +3990,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 51
           },
           "id": 20,
           "options": {
@@ -4073,7 +4078,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4089,7 +4095,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 51
           },
           "id": 126,
           "options": {
@@ -4176,7 +4182,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4192,7 +4199,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 59
           },
           "id": 46,
           "options": {
@@ -4230,6 +4237,110 @@
           "title": "Scrape response size 0.99 quantile ($instance)",
           "type": "timeseries"
         },
+        {
+          "datasource": {
+            "type": "victoriametrics-datasource",
+            "uid": "$ds"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 59
+          },
+          "id": 148,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.2.6",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)",
+              "format": "time_series",
+              "interval": "",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Scrape duration 0.99 quantile ($instance)",
+          "type": "timeseries"
+        },
         {
           "datasource": {
             "type": "victoriametrics-datasource",
@@ -4279,7 +4390,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4295,7 +4407,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 67
           },
           "id": 31,
           "options": {
@@ -4575,8 +4687,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4680,8 +4791,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4798,8 +4908,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4934,8 +5043,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5037,8 +5145,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5134,8 +5241,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5238,8 +5344,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5349,8 +5454,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5447,8 +5551,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5545,8 +5648,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5693,8 +5795,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5798,8 +5899,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5903,8 +6003,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -6008,8 +6107,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -6112,8 +6210,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
-                    "value": null
+                    "color": "transparent"
                   },
                   {
                     "color": "red",
@@ -6315,8 +6412,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
-                    "value": null
+                    "color": "transparent"
                   },
                   {
                     "color": "red",

dashboards/vmagent.json

@@ -3547,7 +3547,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3563,7 +3564,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 13
+            "y": 37
           },
           "id": 48,
           "options": {
@@ -3653,7 +3654,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3669,7 +3671,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 13
+            "y": 37
           },
           "id": 76,
           "options": {
@@ -3757,7 +3759,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3773,7 +3776,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 20
+            "y": 44
           },
           "id": 132,
           "options": {
@@ -3863,7 +3866,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3879,7 +3883,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 20
+            "y": 44
           },
           "id": 133,
           "options": {
@@ -3968,7 +3972,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3984,7 +3989,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 51
           },
           "id": 20,
           "options": {
@@ -4072,7 +4077,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4088,7 +4094,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 51
           },
           "id": 126,
           "options": {
@@ -4175,7 +4181,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4191,7 +4198,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 59
           },
           "id": 46,
           "options": {
@@ -4229,6 +4236,110 @@
           "title": "Scrape response size 0.99 quantile ($instance)",
           "type": "timeseries"
         },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 59
+          },
+          "id": 148,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.2.6",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)",
+              "format": "time_series",
+              "interval": "",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Scrape duration 0.99 quantile ($instance)",
+          "type": "timeseries"
+        },
         {
           "datasource": {
             "type": "prometheus",
@@ -4278,7 +4389,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4294,7 +4406,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 67
           },
           "id": 31,
           "options": {
@@ -4574,8 +4686,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4679,8 +4790,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4797,8 +4907,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4933,8 +5042,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5036,8 +5144,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5133,8 +5240,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5237,8 +5343,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5348,8 +5453,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5446,8 +5550,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5544,8 +5647,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5692,8 +5794,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5797,8 +5898,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5902,8 +6002,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -6007,8 +6106,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -6111,8 +6209,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
-                    "value": null
+                    "color": "transparent"
                   },
                   {
                     "color": "red",
@@ -6314,8 +6411,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
-                    "value": null
+                    "color": "transparent"
                   },
                   {
                     "color": "red",