Compare commits

..

22 Commits

Author SHA1 Message Date
Jayice
7d558511d0 make linter happy 2026-06-15 15:46:41 +08:00
JAYICE
a2bb3f70a5 Merge branch 'master' into issue-10600 2026-06-15 15:40:08 +08:00
Jayice
2c496f4a38 address review comments 2026-06-15 15:25:35 +08:00
Roman Khavronenko
d52de359d5 app/vmselect: log calls to /api/v1/admin/tsdb/delete_series
The log message will display when deletion API was called, how many
series it deleted and what params were used. This should help
identifying events of metrics deletion.

Example:
```
2026-06-12T13:02:28.006Z        info    VictoriaMetrics/app/vmselect/prometheus/prometheus.go:529       /api/v1/admin/tsdb/delete_series has been called for "[{__name__=\"vm_http_request_errors_total\"}]". Deleted 0 series.
```
2026-06-15 09:07:39 +02:00
Zasda Yusuf Mikail
892f4aced2 lib/httpserver: allow disabling server hostname header
When responding to an HTTP request, VictoriaMetrics components include the X-Server-Hostname. 
While this may be useful for debugging, it also leaks the hostname.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11067
2026-06-15 09:05:36 +02:00
Jayice
ef0ae0fb9a address review comments 2026-06-05 14:40:28 +08:00
Jayice
4cbedc6b1b Merge remote-tracking branch 'origin/issue-10600' into issue-10600 2026-06-05 14:32:01 +08:00
Jayice
4f5cd15163 address review comments 2026-06-05 14:30:21 +08:00
JAYICE
5b161ce283 Merge branch 'master' into issue-10600 2026-06-05 13:44:59 +08:00
Jayice
43773e1d5c address review comments 2026-06-04 21:29:30 +08:00
Jayice
517c17b744 address review comments 2026-06-01 20:10:44 +08:00
Jayice
1c2622d8ae address review comments 2026-06-01 20:04:58 +08:00
Jayice
9a836dac59 address review comments 2026-06-01 20:00:52 +08:00
Jayice
799ecb0a08 support specify label and value to filter metrics to mdx url 2026-06-01 19:50:00 +08:00
Jayice
c88dc19052 address review comments 2026-06-01 14:52:16 +08:00
Jayice
919049f9e2 push slice back to pool 2026-05-25 17:34:41 +08:00
Jayice
24efe47c6a add unit test & address review comments 2026-05-25 17:08:23 +08:00
Jayice
f8d99d9289 polish documentation 2026-04-27 18:02:41 +08:00
Jayice
333a015be5 update CHANGELOG.md 2026-04-27 15:15:59 +08:00
JAYICE
b6196524ba Merge branch 'master' into issue-10600
Signed-off-by: JAYICE <1185430411@qq.com>
2026-04-27 15:13:02 +08:00
Jayice
e9f1bb911c add documentation 2026-04-27 15:10:17 +08:00
Jayice
12b79143dc implement mdx for remote write 2026-04-21 15:48:46 +08:00
21 changed files with 703 additions and 90 deletions

View File

@@ -12,6 +12,7 @@ import (
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/mdx"
"github.com/cespare/xxhash/v2"
"github.com/VictoriaMetrics/metrics"
@@ -103,6 +104,9 @@ var (
"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
disableMetadataPerURL = flagutil.NewArrayBool("remoteWrite.disableMetadata", "Whether to disable sending metadata to the corresponding -remoteWrite.url. "+
"By default, metadata sending is controlled by the global -enableMetadata flag")
enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
)
var (
@@ -304,6 +308,10 @@ func initRemoteWriteCtxs(urls []string) {
}
fs.RegisterPathFsMetrics(*tmpDataPath)
if slices.Contains(*enableMdx, true) && *shardByURL {
logger.Fatalf("-remoteWrite.mdx.enable and -remoteWrite.shardByURL cannot be set to true simultaneously.")
}
if *shardByURL {
consistentHashNodes := make([]string, 0, len(urls))
for i, url := range urls {
@@ -859,6 +867,7 @@ type remoteWriteCtx struct {
sas atomic.Pointer[streamaggr.Aggregators]
deduplicator *streamaggr.Deduplicator
mdxFilter *mdx.Filter
streamAggrKeepInput bool
streamAggrDropInput bool
@@ -873,6 +882,7 @@ type remoteWriteCtx struct {
rowsPushedAfterRelabel *metrics.Counter
rowsDroppedByRelabel *metrics.Counter
rowsPreservedByMdx *metrics.Counter
pushFailures *metrics.Counter
metadataDroppedOnPushFailure *metrics.Counter
@@ -959,7 +969,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
for i := range pss {
pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
}
rwctx := &remoteWriteCtx{
idx: argIdx,
fq: fq,
@@ -976,6 +985,15 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
}
rwctx.initStreamAggrConfig()
if enableMdx.GetOptionalArg(argIdx) {
rwctx.mdxFilter = mdx.NewFilter()
rwctx.rowsPreservedByMdx = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_mdx_rows_preserved_total{path=%q,url=%q}`, queuePath, sanitizedURL))
_ = metrics.NewGauge(fmt.Sprintf(`vmagent_mdx_tracked_vm_instances{path=%q,url=%q}`, queuePath, sanitizedURL), func() float64 {
return float64(rwctx.mdxFilter.VmInstancesCount())
})
}
return rwctx
}
@@ -989,6 +1007,10 @@ func (rwctx *remoteWriteCtx) MustStop() {
rwctx.deduplicator.MustStop()
rwctx.deduplicator = nil
}
if rwctx.mdxFilter != nil {
rwctx.mdxFilter.MustStop()
rwctx.mdxFilter = nil
}
for _, ps := range rwctx.pss {
ps.MustStop()
@@ -1004,6 +1026,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
rwctx.rowsPushedAfterRelabel = nil
rwctx.rowsDroppedByRelabel = nil
rwctx.rowsPreservedByMdx = nil
}
// TryPushTimeSeries sends tss series to the configured remote write endpoint
@@ -1016,20 +1039,33 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
if rctx == nil {
return
}
putRelabelCtx(rctx)
*v = prompb.ResetTimeSeries(tss)
tssPool.Put(v)
putRelabelCtx(rctx)
}()
if rwctx.mdxFilter != nil {
tssResP := tssPool.Get().(*[]prompb.TimeSeries)
tssRes := rwctx.mdxFilter.Filter(tss, *tssResP)
defer func() {
*tssResP = prompb.ResetTimeSeries(tssRes)
tssPool.Put(tssResP)
}()
if len(tssRes) == 0 {
return true
}
}
// Apply relabeling
rcs := allRelabelConfigs.Load()
pcs := rcs.perURL[rwctx.idx]
if pcs.Len() > 0 {
rctx = getRelabelCtx()
// Make a copy of tss before applying relabeling in order to prevent
// from affecting time series for other remoteWrite.url configs.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
rctx = getRelabelCtx()
v = tssPool.Get().(*[]prompb.TimeSeries)
tss = append(*v, tss...)
rowsCountBeforeRelabel := getRowsCount(tss)

View File

@@ -28,6 +28,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -525,6 +526,7 @@ func DeleteHandler(startTime time.Time, r *http.Request) error {
if deletedCount > 0 {
promql.ResetRollupResultCache()
}
logger.Infof("/api/v1/admin/tsdb/delete_series has been called for %q. Deleted %d series.", sq.FiltersString(), deletedCount)
return nil
}

View File

@@ -26,6 +26,10 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
* FEATURE: all VictoriaMetrics components: add `-http.header.disableServerHostname` command-line flag for disabling the `X-Server-Hostname` HTTP response header. See [#11067](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11067). Thanks to @zasdaym for contribution.
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): log calls to [/api/v1/admin/tsdb/delete_series](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series) API handler. This should help to identify events of metrics deletion from the database.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange): the ability to route only metrics from VictoriaMetrics services to a specific `-remoteWrite.url`. MDX is useful for building monitoring-of-monitoring where one remote storage should receive the full metric stream and another should receive only VictoriaMetrics metrics. Enable per destination with `-remoteWrite.mdx.enable=true`. See [#10600](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10600).
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix issue with producing aggregated samples with identical timestamps between flushes. See PR [#10808](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10808) for details.
## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)

View File

@@ -95,6 +95,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -268,6 +268,37 @@ for the collected samples. Examples:
```sh
./vmagent -remoteWrite.url=http://remote-storage/api/v1/write -streamAggr.dropInputLabels=replica -streamAggr.dedupInterval=60s
```
### Monitoring Data eXchange
The Monitoring Data eXchange (MDX) feature allows `vmagent` to forward only VictoriaMetrics metrics to selected `-remoteWrite.url` destinations while dropping metrics from non-VictoriaMetrics services.
To enable MDX, set `-remoteWrite.mdx.enable=true` for the target URL and `-remoteWrite.mdx.enable=false` for other URLs:
```sh
./vmagent \
-remoteWrite.url=http://service-to-keep-all-metrics:8428/api/v1/write \
-remoteWrite.mdx.enable=false \
-remoteWrite.url=http://service-to-keep-only-vm-metrics:8428/api/v1/write \
-remoteWrite.mdx.enable=true
```
When MDX is enabled for a `-remoteWrite.url`, `vmagent` forwards only metrics that:
- come from the target that exposes the `vm_app_version` metric (emitted by all VictoriaMetrics components)
- contain the `victoriametrics_app=true` label, which will be added automatically to the metrics if the instance was deployed via [VictoriaMetrics Operator](https://docs.victoriametrics.com/operator/).
`victoriametrics_app=true` label will be added to all metrics that are preserved by MDX if it's absent.
- contain the label specified via `-mdx.label`.
```sh
./vmagent \
-remoteWrite.url=http://service-to-keep-only-vm-metrics:8428/api/v1/write \
-remoteWrite.mdx.enable=true \
-mdx.label="service=victoriametrics"
```
In this configuration, metrics with the label `service=victoriametrics` are preserved even if their scrape targets do not expose `vm_app_version` metric.
The number of VictoriaMetrics metrics preserved by MDX is exposed as `vmagent_remotewrite_mdx_rows_preserved_total`.
### Life of a sample
@@ -285,18 +316,20 @@ flowchart TB
F --> G[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#replication-and-high-availability">replicate</a> to each <b>-remoteWrite.url</b><br/>or <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages">shard</a> if <b>-remoteWrite.shardByURL</b> is set]
%% Left branch
G --> H1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
H1 --> H2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
H2 --> H3["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
H3 --> H4[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
H4 --> H5[[push to <b>-remoteWrite.url</b>]]
G --> H1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange/">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
H1 --> H2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
H2 --> H3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
H4 --> H5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
H5 --> H6[[push to <b>-remoteWrite.url</b>]]
%% Right branch
G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
R1 --> R2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
R2 --> R3["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
R3 --> R4[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
R4 --> R5[[push to <b>-remoteWrite.url</b>]]
G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
R1 --> R2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
R2 --> R3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
R4 --> R5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
R5 --> R6[[push to <b>-remoteWrite.url</b>]]
```
Scraping has additional settings that can be applied before samples are pushed to the processing pipeline above:

View File

@@ -74,6 +74,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -546,7 +546,7 @@ tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags) and
## Reading rules from object storage
The [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
[Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
from object storage:
* `./bin/vmalert -rule=s3://bucket/dir/alert.rules` would read rules from the given path at S3 bucket
@@ -563,8 +563,6 @@ The following [command-line flags](#flags) can be used for fine-tuning access to
* `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set.
* `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default.
See [providing credentials as a file](https://docs.victoriametrics.com/victoriametrics/vmbackup/#providing-credentials-as-a-file) for details on how to create and use credentials to access S3-compatible buckets and Google Cloud Storage.
## Topology examples
The following sections are showing how `vmalert` may be used and configured

View File

@@ -115,6 +115,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -59,6 +59,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/ .
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -204,80 +204,38 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
### Providing credentials as a file
`vmbackup`, `vmbackupmanager`, and [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can load credentials from a file via the `-credsFilePath` flag to access remote S3-compatible buckets and Google Cloud Storage.
Obtaining credentials from a file.
To use a credential file, add the flag:
Add flag `-credsFilePath=/etc/credentials` with the following content:
```sh
-credsFilePath=/etc/credentials
```
* for S3 (AWS, MinIO or other S3 compatible storages):
The argument should point to a file with one of the formats below, depending on the storage provider.
```sh
[default]
aws_access_key_id=theaccesskey
aws_secret_access_key=thesecretaccesskeyvalue
```
#### S3 (AWS and S3-compatible)
* for GCP cloud storage:
1. In AWS, [create an IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html) or role with permissions to read and write the target bucket.
2. [Create an access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) for that IAM identity and copy the **Access key** and **Secret access key** values
3. On the machine running `vmbackup`, create a credentials file with the following content and point `-credsFilePath` to it:
```ini
[default]
aws_access_key_id=YOUR_AWS_ACCESS_KEY
aws_secret_access_key=YOUR_AWS_SECRET_ACCESS_KEY
```
This format matches the standard shared AWS credentials file used by the [AWS CLI](https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-files.html) and [AWS SDKs](https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html).
For S3-compatible backends such as [MinIO](https://www.min.io/) or [Ceph](https://ceph.io/), create access keys in the respective
system and use the same file format and set a custom endpoint with `-customS3Endpoint`.
For example:
```sh
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=s3://victoriametrics-backup/backup01 \
-customS3Endpoint=http://minio.example.local:9000 \
-credsFilePath=/etc/credentials
```
#### Google Cloud Storage (GCS)
To create an IAM user and download the credential file, follow these steps:
1. Open the Google Cloud Console and go to **IAM & Admin → Service Accounts**.
2. Click **Create service account**.
3. Enter a service account name.
4. Assign the role the account needs to access Google Cloud Storage. See [IAM permissions for JSON methods](https://docs.cloud.google.com/storage/docs/access-control/iam-json) for more details.
5. Open the service account, go to **Keys**, then click **Add key → Create new key**.
6. Choose **JSON** as the key type
7. Save the downloaded JSON file on the machine running `vmbackup` and point `-credsFilePath` to it. The file contents look similar to:
```json
{
"type": "service_account",
"project_id": "project-id",
"private_key_id": "key-id",
"private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
"client_email": "service-account-email",
"client_id": "client-id",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
}
```
This JSON is the standard service account key format defined by [Google Cloud IAM](https://developers.google.com/workspace/guides/create-credentials) and is used by Google client libraries and tools.
#### Azure Blob Storage
Azure Blob Storage uses environment variables rather than `-credsFilePath` in `vmbackup`. See [providing credentials via env variables](https://docs.victoriametrics.com/victoriametrics/vmbackup/#providing-credentials-via-env-variables) for details.
```json
{
"type": "service_account",
"project_id": "project-id",
"private_key_id": "key-id",
"private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
"client_email": "service-account-email",
"client_id": "client-id",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
}
```
### Providing credentials via env variables
Obtaining credentials from environment variables.
Obtaining credentials from env variables.
* For AWS S3 compatible storages set env variable `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
Also you can set env variable `AWS_SHARED_CREDENTIALS_FILE` with path to credentials file.
@@ -442,6 +400,8 @@ Run `vmbackup -help` in order to see all the available options:
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -575,6 +575,8 @@ command-line flags:
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -496,6 +496,8 @@ Below is the list of configuration flags (it can be viewed by running `./vmgatew
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -76,6 +76,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -102,6 +102,8 @@ Run `vmrestore -help` in order to see all the available options:
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -75,6 +75,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -68,6 +68,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
-http.header.csp string
Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"
-http.header.disableServerHostname
Whether to disable 'X-Server-Hostname' header in HTTP responses
-http.header.frameOptions string
Value for 'X-Frame-Options' header
-http.header.hsts string

View File

@@ -64,9 +64,10 @@ var (
connTimeout = flag.Duration("http.connTimeout", 2*time.Minute, "Incoming connections to -httpListenAddr are closed after the configured timeout. "+
"This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections")
headerHSTS = flag.String("http.header.hsts", "", "Value for 'Strict-Transport-Security' header, recommended: 'max-age=31536000; includeSubDomains'")
headerFrameOptions = flag.String("http.header.frameOptions", "", "Value for 'X-Frame-Options' header")
headerCSP = flag.String("http.header.csp", "", `Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"`)
headerHSTS = flag.String("http.header.hsts", "", "Value for 'Strict-Transport-Security' header, recommended: 'max-age=31536000; includeSubDomains'")
headerFrameOptions = flag.String("http.header.frameOptions", "", "Value for 'X-Frame-Options' header")
headerCSP = flag.String("http.header.csp", "", `Value for 'Content-Security-Policy' header, recommended: "default-src 'self'"`)
headerDisableServerHostname = flag.Bool("http.header.disableServerHostname", false, "Whether to disable 'X-Server-Hostname' header in HTTP responses")
disableCORS = flag.Bool("http.disableCORS", false, `Disable CORS for all origins (*)`)
)
@@ -329,7 +330,9 @@ func handlerWrapper(w http.ResponseWriter, r *http.Request, rh RequestHandler) {
if *headerCSP != "" {
h.Add("Content-Security-Policy", *headerCSP)
}
h.Add("X-Server-Hostname", hostname)
if !*headerDisableServerHostname {
h.Add("X-Server-Hostname", hostname)
}
requestsTotal.Inc()
if whetherToCloseConn(r) {
connTimeoutClosedConns.Inc()

View File

@@ -228,4 +228,30 @@ func TestHandlerWrapper(t *testing.T) {
if got := h.Get("Content-Security-Policy"); got != cspHeader {
t.Fatalf("unexpected CSP header; got %q; want %q", got, cspHeader)
}
if got := h.Get("X-Server-Hostname"); got != hostname {
t.Fatalf("unexpected X-Server-Hostname header; got %q; want %q", got, hostname)
}
}
func TestHandlerWrapperDisableServerHostnameHeader(t *testing.T) {
origDisableServerHostname := *headerDisableServerHostname
*headerDisableServerHostname = true
defer func() {
*headerDisableServerHostname = origDisableServerHostname
}()
req, _ := http.NewRequest("GET", "/health", nil)
srv := &server{s: &http.Server{}}
w := &httptest.ResponseRecorder{}
handlerWrapper(w, req, func(w http.ResponseWriter, r *http.Request) bool {
return builtinRoutesHandler(srv, r, w, func(_ http.ResponseWriter, _ *http.Request) bool {
return true
})
})
if got := w.Header().Get("X-Server-Hostname"); got != "" {
t.Fatalf("unexpected X-Server-Hostname header; got %q; want empty value", got)
}
}

196
lib/mdx/filter.go Normal file
View File

@@ -0,0 +1,196 @@
package mdx
import (
"flag"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
)
var (
vmLabel = flag.String("mdx.label", "", "Optional label in the form 'name=value' used to identify VictoriaMetrics metrics for MDX. Metrics containing the specified label are forwarded to `-remoteWrite.url` endpoints configured with `-remoteWrite.mdx.enable=true`.")
vmAppLabelName = "victoriametrics_app"
)
// Filter manages the list of VictoriaMetrics instances discovered from previous data flow, and uses it to filter out metrics that are not from VictoriaMetrics instances.
type Filter struct {
mu sync.RWMutex
wg sync.WaitGroup
stopCh chan struct{}
vmInstance map[string]*atomic.Int64
filterByCustomLabelName string
filterByCustomLabelValue string
}
func NewFilter() *Filter {
filter := &Filter{
vmInstance: make(map[string]*atomic.Int64),
stopCh: make(chan struct{}),
}
if len(*vmLabel) != 0 {
n := strings.IndexByte(*vmLabel, '=')
if n < 0 {
logger.Fatalf("missing '=' in `-mdx.label`. It must contain label in the form `name=value`; got %q", *vmLabel)
}
filter.filterByCustomLabelName = (*vmLabel)[:n]
filter.filterByCustomLabelValue = (*vmLabel)[n+1:]
}
filter.wg.Go(filter.cleanStale)
return filter
}
func (filter *Filter) VmInstancesCount() int {
filter.mu.RLock()
defer filter.mu.RUnlock()
return len(filter.vmInstance)
}
func (filter *Filter) cleanStale() {
entryTTL := time.Hour * 1
ttlSec := int64(entryTTL.Seconds())
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for {
select {
case <-ticker.C:
filter.mu.Lock()
currTs := time.Now().Unix()
dst := make(map[string]*atomic.Int64, len(filter.vmInstance))
for k, v := range filter.vmInstance {
if currTs-v.Load() < ttlSec {
dst[k] = v
}
}
if len(dst) != len(filter.vmInstance) {
filter.vmInstance = dst
}
filter.mu.Unlock()
case <-filter.stopCh:
return
}
}
}
func (filter *Filter) MustStop() {
if filter == nil {
return
}
close(filter.stopCh)
filter.wg.Wait()
}
func (filter *Filter) Filter(tss []prompb.TimeSeries, resTss []prompb.TimeSeries) []prompb.TimeSeries {
currTs := time.Now().Unix()
var identicalKey []byte
nextTss:
for _, ts := range tss {
var hasVersionLabel, triedJobInstance bool
var job, instance string
for i, label := range ts.Labels {
if label.Name == vmAppLabelName && label.Value == "true" {
resTss = append(resTss, ts)
continue nextTss
}
if filter.filterByCustomLabelName != "" && label.Name == filter.filterByCustomLabelName && label.Value == filter.filterByCustomLabelValue {
// add victoriametrics_app=true label if absent.
hasVmAppLabel := false
for j := i + 1; j < len(ts.Labels); j++ {
if ts.Labels[j].Name == vmAppLabelName {
hasVmAppLabel = true
break
}
}
if !hasVmAppLabel {
ts.Labels = append(ts.Labels, prompb.Label{Name: vmAppLabelName, Value: "true"})
}
resTss = append(resTss, ts)
continue nextTss
}
if label.Name == "__name__" && label.Value == "vm_app_version" {
hasVersionLabel = true
}
if instance == "" && label.Name == "instance" {
if label.Value == "" {
continue
}
instance = label.Value
}
if job == "" && label.Name == "job" {
if label.Value == "" {
continue
}
job = label.Value
}
if !triedJobInstance && job != "" && instance != "" {
identicalKey = identicalKey[:0]
identicalKey = strconv.AppendQuote(identicalKey, job)
identicalKey = append(identicalKey, ':')
identicalKey = strconv.AppendQuote(identicalKey, instance)
filter.mu.RLock()
ptr, found := filter.vmInstance[bytesutil.ToUnsafeString(identicalKey)]
filter.mu.RUnlock()
if found {
ptr.Store(currTs)
// add victoriametrics_app=true label if absent.
hasVmAppLabel := false
for j := i + 1; j < len(ts.Labels); j++ {
if ts.Labels[j].Name == vmAppLabelName {
hasVmAppLabel = true
break
}
}
if !hasVmAppLabel {
ts.Labels = append(ts.Labels, prompb.Label{Name: vmAppLabelName, Value: "true"})
}
resTss = append(resTss, ts)
continue nextTss
}
triedJobInstance = true
}
if hasVersionLabel && job != "" && instance != "" {
identicalKey = identicalKey[:0]
identicalKey = strconv.AppendQuote(identicalKey, job)
identicalKey = append(identicalKey, ':')
identicalKey = strconv.AppendQuote(identicalKey, instance)
v := &atomic.Int64{}
v.Store(currTs)
filter.mu.Lock()
filter.vmInstance[string(identicalKey)] = v
filter.mu.Unlock()
// add victoriametrics_app=true label if absent.
hasVmAppLabel := false
for j := i + 1; j < len(ts.Labels); j++ {
if ts.Labels[j].Name == vmAppLabelName {
hasVmAppLabel = true
break
}
}
if !hasVmAppLabel {
ts.Labels = append(ts.Labels, prompb.Label{Name: vmAppLabelName, Value: "true"})
}
resTss = append(resTss, ts)
continue nextTss
}
}
}
return resTss
}

329
lib/mdx/filter_test.go Normal file
View File

@@ -0,0 +1,329 @@
package mdx
import (
"fmt"
"sort"
"strings"
"testing"
"testing/synctest"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
)
func timeSeriessToString(tss []prompb.TimeSeries) string {
a := make([]string, len(tss))
for i, ts := range tss {
a[i] = timeSeriesToString(ts)
}
sort.Strings(a)
return strings.Join(a, "")
}
func timeSeriesToString(ts prompb.TimeSeries) string {
labelsString := promrelabel.LabelsToString(ts.Labels)
return fmt.Sprintf("%s\n", labelsString)
}
func TestMdxInstanceFilter(t *testing.T) {
originalVmLabel := *vmLabel
*vmLabel = "service=victoriametrics"
filter := NewFilter()
f := func(input []prompb.TimeSeries, expectedOutput []prompb.TimeSeries, expectedInstanceMap map[string]int64) {
t.Helper()
output := filter.Filter(input, []prompb.TimeSeries{})
outputString := timeSeriessToString(output)
expectedOutputString := timeSeriessToString(expectedOutput)
if outputString != expectedOutputString {
t.Fatalf("unexpected output; got %s; want %s", outputString, expectedOutputString)
}
if len(filter.vmInstance) != len(expectedInstanceMap) {
t.Fatalf("unexpected instance map length; got %d; want %d", len(filter.vmInstance), len(expectedInstanceMap))
}
for k := range expectedInstanceMap {
if filter.vmInstance[k] == nil {
t.Fatalf("missing instance in filter.vmInstance: %q", k)
}
}
}
// the first call
f([]prompb.TimeSeries{
// 1. metrics with vm_app_version and different order of labels.
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics2:8428"},
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics3:8428"},
{Name: "__name__", Value: "vm_app_version"},
},
},
// 2.
// metrics without vm_app_version but with service=victoriametrics that is specified in `-vm.label`.
// it will be preserved, but won't be registered in instance map in MDX
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "service", Value: "victoriametrics"},
},
},
// 3. metrics with vm_app_version and service=victoriametrics should be preserved.
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics5:8428"},
{Name: "job", Value: "test"},
{Name: "service", Value: "victoriametrics"},
{Name: "__name__", Value: "vm_app_version"},
},
},
// 4. metrics without vm_app_version and `service=victoriametrics` but with `victoriametrics_app=true`, which should be preserved.
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics6:8428"},
{Name: "job", Value: "test"},
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "victoriametrics_app", Value: "true"},
},
},
// 5. metrics without vm_app_version and service=victoriametrics and `victoriametrics_app=true`, which should be filtered out.
{
Labels: []prompb.Label{
{Name: "__name__", Value: "go_gc_duration_seconds"},
{Name: "instance", Value: "node-exporter1"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "http_request_duration_seconds"},
{Name: "instance", Value: "service1"},
{Name: "job", Value: "test"},
},
},
},
// `victoriametrics_app=true` should be added to all preserved metrics if absent.
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics2:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics3:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "service", Value: "victoriametrics"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics5:8428"},
{Name: "job", Value: "test"},
{Name: "__name__", Value: "vm_app_version"},
{Name: "service", Value: "victoriametrics"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics6:8428"},
{Name: "job", Value: "test"},
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "victoriametrics_app", Value: "true"},
},
},
},
// only instances that are discovered via `vm_app_version` will be registered in instance map in MDX.
map[string]int64{
fmt.Sprintf("%q:%q", "test", "victoria-metrics1:8428"): 0,
fmt.Sprintf("%q:%q", "test", "victoria-metrics2:8428"): 0,
fmt.Sprintf("%q:%q", "test", "victoria-metrics3:8428"): 0,
})
// the second call
f([]prompb.TimeSeries{
// 1. metrics without vm_app_version, but the instances were already registered in the previous call, so it will be preserved.
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_rows_inserted_total"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vminsert_request_duration_seconds_bucket"},
{Name: "instance", Value: "victoria-metrics2:8428"},
{Name: "job", Value: "test"},
},
},
// 2. metrics without vm_app_version, `service=victoriametrics` and `victoriametrics_app=true`, and the instance wasn't already registered in the previous call, so it will be dropped.
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vminsert_request_duration_seconds_bucket"},
{Name: "instance", Value: "victoria-metrics7:8428"},
{Name: "job", Value: "test"},
},
},
// 3. metrics with service=victoriametrics.
{
Labels: []prompb.Label{
{Name: "service", Value: "victoriametrics"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "job", Value: "test"},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_rows_inserted_total"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vminsert_request_duration_seconds_bucket"},
{Name: "instance", Value: "victoria-metrics2:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "service", Value: "victoriametrics"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
},
// only instances that are discovered via `vm_app_version` will be registered in instance map in MDX.
map[string]int64{
fmt.Sprintf("%q:%q", "test", "victoria-metrics1:8428"): 0,
fmt.Sprintf("%q:%q", "test", "victoria-metrics2:8428"): 0,
fmt.Sprintf("%q:%q", "test", "victoria-metrics3:8428"): 0,
})
*vmLabel = originalVmLabel
}
func TestMdxInstanceCleanup(t *testing.T) {
t.Helper()
synctest.Test(t, func(t *testing.T) {
filter := NewFilter()
filter.Filter([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "go_gc_duration_seconds"},
{Name: "instance", Value: "node-exporter1"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "http_request_duration_seconds"},
{Name: "instance", Value: "service1"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "vmagent1:8429"},
{Name: "job", Value: "test"},
},
}}, []prompb.TimeSeries{},
)
f := func(expectedInstanceMap map[string]int64) {
t.Helper()
if len(filter.vmInstance) != len(expectedInstanceMap) {
t.Fatalf("unexpected instance map length; got %d; want %d", len(filter.vmInstance), len(expectedInstanceMap))
}
for k := range expectedInstanceMap {
if filter.vmInstance[k] == nil {
t.Fatalf("missing instance in filter.vmInstance: %q", k)
}
}
}
time.Sleep(59 * time.Minute)
// the entries should not be cleaned.
f(map[string]int64{
fmt.Sprintf("%q:%q", "test", "victoria-metrics1:8428"): 0,
fmt.Sprintf("%q:%q", "test", "vmagent1:8429"): 0,
})
// receive samples from victoria-metrics1:8428 after 59 minutes.
// so the entry will be refreshed.
filter.Filter([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
}}, []prompb.TimeSeries{},
)
time.Sleep(2 * time.Minute)
// no samples from vmagent1:8429 in the last hour, so it should be removed from the mdx instance list.
f(map[string]int64{
fmt.Sprintf("%q:%q", "test", "victoria-metrics1:8428"): 0,
})
filter.MustStop()
})
}

View File

@@ -468,15 +468,21 @@ func (tf *TagFilter) Unmarshal(src []byte) ([]byte, error) {
return src, nil
}
// String returns string representation of the search query.
// String returns string representation of the search query: tag filters and time range.
func (sq *SearchQuery) String() string {
start := TimestampToHumanReadableFormat(sq.MinTimestamp)
end := TimestampToHumanReadableFormat(sq.MaxTimestamp)
a := sq.FiltersString()
return fmt.Sprintf("filters=%s, timeRange=[%s..%s]", a, start, end)
}
// FiltersString returns string representation of the tag filters.
func (sq *SearchQuery) FiltersString() []string {
a := make([]string, len(sq.TagFilterss))
for i, tfs := range sq.TagFilterss {
a[i] = tagFiltersToString(tfs)
}
start := TimestampToHumanReadableFormat(sq.MinTimestamp)
end := TimestampToHumanReadableFormat(sq.MaxTimestamp)
return fmt.Sprintf("filters=%s, timeRange=[%s..%s]", a, start, end)
return a
}
func tagFiltersToString(tfs []TagFilter) string {