Compare commits

..

10 Commits

Author SHA1 Message Date
Jayice
534c57b79a make scrape_series_added negative when the number of series decreased 2026-03-18 16:02:09 +08:00
Aliaksandr Valialkin
e47abd6385 docs/victoriametrics/Articles.md: add https://clovisc.medium.com/monitoring-pipeline-with-blackbox-exporter-prometheus-victoriametrics-and-vmalert-0ab020c7202a 2026-03-18 02:42:49 +01:00
Aliaksandr Valialkin
c04a5a597d docs/victoriametrics/Articles.md: add https://apprecode.com/blog/a-complete-guide-to-victoriametrics-a-prometheus-comparison-and-kubernetes-monitoring-implementation 2026-03-18 02:41:34 +01:00
JAYICE
e695d5f425 app/vmselect: retry with new connection when previous rpc fail on a broken connection
This commit adds a rpc retry by dialing a new connection instead of
getting an old one from the connection pool when the previous rpc error
is `io.EOF`.

It helps prevent broken connections from remaining for too long and
causing failed requests and partial responses during `vmstorage` rolling
restart period

fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10314
2026-03-17 11:01:16 +01:00
andriibeee
2bb03f6e34 lib/storage, lib/mergeset: properly account inmemoryPart refCount
Previously inmemoryPart refCount was not properly decremented.

Previous behavior:
* createInmemoryPart called newPartWrapperFromInmemoryPart and returns a partWrapper with refCount=1
* multiple parts are merged in mustMergeInmemoryPartsFinal, which creates a new merged part
* the source partWrappers are never decRef'd
* Since refCount never reaches 0, putInmemoryPart and (*part).MustClose are never called 

 This commit properly decrements refCount at mustMergeInmemoryPartsFinal. 

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10086
2026-03-17 10:54:08 +01:00
Br1an
92f03344eb lib/promscrape/discovery/yandexcloud: add folder_ids option
This commit adds a new `folder_ids` field in
`yandexcloud_sd_configs` that allows users to specify Yandex Cloud
folder IDs directly, bypassing the organization->cloud->folder hierarchy
traversal.

Previously, the Yandex Cloud service discovery required traversing the
entire resource hierarchy (organizations -> clouds -> folders ->
instances) to discover instances. This works when the Service Account
has permissions at all levels. However, some Service Accounts may only
have permissions at the folder level, causing discovery to fail when it
cannot access organization or cloud resources.

With this change, users can now configure folder IDs directly:

```yaml
yandexcloud_sd_configs:
  - service: compute
    folder_ids:
      - folder-id-1
      - folder-id-2
```

When `folder_ids` is specified, the discovery skips the hierarchy
traversal and directly queries instances from the specified folders.
This is a backward-compatible change - when `folder_ids` is not
specified, the existing behavior is preserved.

fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10587
2026-03-17 10:51:05 +01:00
Artem Fetishev
e3360b87ff docs: run make docs-update-flags
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-03-16 16:59:51 +01:00
Artem Fetishev
4c98b912fa docs: bump version to v1.138.0
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-03-16 16:54:04 +01:00
Artem Fetishev
225e2e870b deplyoment/docker: bump version to v1.138.0
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-03-16 16:48:43 +01:00
Artem Fetishev
2b078301c1 docs/CHANGELOG.md: update changelog with LTS release notes
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-03-16 15:23:05 +01:00
27 changed files with 327 additions and 110 deletions

View File

@@ -3,7 +3,7 @@ services:
# It scrapes targets defined in --promscrape.config
# And forward them to --remoteWrite.url
vmagent:
image: victoriametrics/vmagent:v1.137.0
image: victoriametrics/vmagent:v1.138.0
depends_on:
- "vmauth"
ports:
@@ -42,14 +42,14 @@ services:
# vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
# where N is number of vmstorages (2 in this case).
vmstorage-1:
image: victoriametrics/vmstorage:v1.137.0-cluster
image: victoriametrics/vmstorage:v1.138.0-cluster
volumes:
- strgdata-1:/storage
command:
- "--storageDataPath=/storage"
restart: always
vmstorage-2:
image: victoriametrics/vmstorage:v1.137.0-cluster
image: victoriametrics/vmstorage:v1.138.0-cluster
volumes:
- strgdata-2:/storage
command:
@@ -59,7 +59,7 @@ services:
# vminsert is ingestion frontend. It receives metrics pushed by vmagent,
# pre-process them and distributes across configured vmstorage shards.
vminsert-1:
image: victoriametrics/vminsert:v1.137.0-cluster
image: victoriametrics/vminsert:v1.138.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -68,7 +68,7 @@ services:
- "--storageNode=vmstorage-2:8400"
restart: always
vminsert-2:
image: victoriametrics/vminsert:v1.137.0-cluster
image: victoriametrics/vminsert:v1.138.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -80,7 +80,7 @@ services:
# vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
# vmselect collects results from configured `--storageNode` shards.
vmselect-1:
image: victoriametrics/vmselect:v1.137.0-cluster
image: victoriametrics/vmselect:v1.138.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -90,7 +90,7 @@ services:
- "--vmalert.proxyURL=http://vmalert:8880"
restart: always
vmselect-2:
image: victoriametrics/vmselect:v1.137.0-cluster
image: victoriametrics/vmselect:v1.138.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -105,7 +105,7 @@ services:
# read requests from Grafana, vmui, vmalert among vmselects.
# It can be used as an authentication proxy.
vmauth:
image: victoriametrics/vmauth:v1.137.0
image: victoriametrics/vmauth:v1.138.0
depends_on:
- "vmselect-1"
- "vmselect-2"
@@ -119,7 +119,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
image: victoriametrics/vmalert:v1.137.0
image: victoriametrics/vmalert:v1.138.0
depends_on:
- "vmauth"
ports:

View File

@@ -3,7 +3,7 @@ services:
# It scrapes targets defined in --promscrape.config
# And forward them to --remoteWrite.url
vmagent:
image: victoriametrics/vmagent:v1.137.0
image: victoriametrics/vmagent:v1.138.0
depends_on:
- "victoriametrics"
ports:
@@ -18,7 +18,7 @@ services:
# VictoriaMetrics instance, a single process responsible for
# storing metrics and serve read requests.
victoriametrics:
image: victoriametrics/victoria-metrics:v1.137.0
image: victoriametrics/victoria-metrics:v1.138.0
ports:
- 8428:8428
- 8089:8089
@@ -59,7 +59,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
image: victoriametrics/vmalert:v1.137.0
image: victoriametrics/vmalert:v1.138.0
depends_on:
- "victoriametrics"
- "alertmanager"

View File

@@ -1,6 +1,6 @@
services:
vmagent:
image: victoriametrics/vmagent:v1.137.0
image: victoriametrics/vmagent:v1.138.0
depends_on:
- "victoriametrics"
ports:
@@ -14,7 +14,7 @@ services:
restart: always
victoriametrics:
image: victoriametrics/victoria-metrics:v1.137.0
image: victoriametrics/victoria-metrics:v1.138.0
ports:
- 8428:8428
volumes:
@@ -40,7 +40,7 @@ services:
restart: always
vmalert:
image: victoriametrics/vmalert:v1.137.0
image: victoriametrics/vmalert:v1.138.0
depends_on:
- "victoriametrics"
ports:

View File

@@ -107,6 +107,8 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
* [Backing up VictoriaMetrics Data: A Complete Guide](https://medium.com/@kanakaraju896/backing-up-victoriametrics-data-a-complete-guide-24473c74450f)
* [Unlocking the Power of VictoriaMetrics: A Prometheus Alternative](https://developer-friendly.blog/blog/2024/06/17/unlocking-the-power-of-victoriametrics-a-prometheus-alternative/)
* [How to Master Kubernetes Observability: Multi-Cluster Monitoring with VictoriaMetrics, Loki, and Grafana](https://www.keyvalue.systems/blog/kubernetes-observability-with-victoriametrics-loki-grafana/)
* [A Complete Guide to VictoriaMetrics, a Prometheus Comparison, and Kubernetes Monitoring Implementation](https://apprecode.com/blog/a-complete-guide-to-victoriametrics-a-prometheus-comparison-and-kubernetes-monitoring-implementation)
* [Monitoring Pipeline with Prometheus and VictoriaMetrics](https://clovisc.medium.com/monitoring-pipeline-with-blackbox-exporter-prometheus-victoriametrics-and-vmalert-0ab020c7202a)
## Third-party articles and slides about VictoriaLogs

View File

@@ -58,9 +58,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
```sh
docker pull victoriametrics/victoria-metrics:v1.137.0
docker pull victoriametrics/victoria-metrics:v1.138.0
docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
victoriametrics/victoria-metrics:v1.137.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
victoriametrics/victoria-metrics:v1.138.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
```
_For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._

View File

@@ -29,6 +29,9 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): show `seriesCountByMetricName` table when a label is in focus in the [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer). See [#10630](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10630). Thanks to @Roshan1299 for the contribution.
* FEATURE: [dashboards/unused-metrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/unused-metrics.json): add a new dashboard for exploring stored metrics based on [Caridnality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) and [ingested metrics usage API](https://docs.victoriametrics.com/victoriametrics/#track-ingested-metrics-usage). The dashboard requires [Infinity Grafana plugin](https://grafana.com/grafana/plugins/yesoreyeram-infinity-datasource/) to be installed. See [#10617](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10617) for details.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): retry RPC by dialing a new connection instead of reusing a pooled one when the previous attempt fails with `io.EOF`, `broken pipe` or `reset by peer`. This reduces query failures caused by stale connections to restarted vmstorage nodes. See [#10314](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10314)
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): make `scrape_series_added` negative when the number of series exposed by target decrease or fail to scrape the target. See [#10653](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10653).
## [v1.138.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.138.0)
Released at 2026-03-13
@@ -43,6 +46,7 @@ Released at 2026-03-13
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support negative values for the group `eval_offset` option, which allows starting group evaluation at `groupInterval-abs(eval_offset)` within `[0...groupInterval]`. See [#10424](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10424).
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Disable `/graphite/tags/tagSeries` and `/graphite/tags/tagMultiSeries` for Graphite tag registration since it is unlikely it is used in context of VictoriaMetrics. See [10544](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544).
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): rename debug tools buttons for clarity. See [#10453](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10453).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): [`yandexcloud_sd_configs`](https://docs.victoriametrics.com/victoriametrics/sd_configs/#yandexcloud_sd_configs) now supports `folder_ids` for limiting discovery to specific folders. See [#10623](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10623).
* BUGFIX: all VictoriaMetrics components: replace `histogram` with `untyped` metric metadata type for [VictoriaMetrics histograms](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) when `-metrics.exposeMetadata` is set. See [#82](https://github.com/VictoriaMetrics/metrics/issues/82).
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly route requests to `default_url`. Previously, `request_path` query arg could be set incorrectly during concurrent requests. See [#10626](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10626).
@@ -84,6 +88,27 @@ It enables back `Discovered targets` debug UI by default.
* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
## [v1.136.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.2)
Released at 2026-03-13
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
* SECURITY: upgrade Go builder from Go1.26.0 to Go1.26.1. See [the list of issues addressed in Go1.26.1](https://github.com/golang/go/issues?q=milestone%3AGo1.26.1%20label%3ACherryPickApproved).
FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Disable `/graphite/tags/tagSeries` and `/graphite/tags/tagMultiSeries` for Graphite tag registration since it is unlikely it is used in context of VictoriaMetrics. See [10544](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add [histogram_fraction](https://docs.victoriametrics.com/victoriametrics/metricsql/#histogram_fraction) function to calculate the fraction of buckets falling between lowerLe and upperLe. See [#5346](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5346).
* BUGFIX: all VictoriaMetrics components: replace `histogram` with `untyped` metric metadata type for [VictoriaMetrics histograms](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) when `-metrics.exposeMetadata` is set. See [#82](https://github.com/VictoriaMetrics/metrics/issues/82).
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly route requests to `default_url`. Previously, `request_path` query arg could be set incorrectly during concurrent requests. See [#10626](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10626).
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not enable ACL when uploading backups to S3-compatible endpoints by default. ACL is not always supported by S3-compatible endpoints and it is not recommended to use ACLs to limit access to objects. See [#10539](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10539) for more details.
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): overwrite s3 object metadata while syncing latest backups with other backup types. See [#10639](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10639).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly attach `host` label to the time series ingested via [/datadog/api/beta/sketches](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/#) API. See [#10557](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): use `increase_pure` instead of `rate` for histogram heatmaps in Explore Metrics to correctly display the first observation in each new bucket. See [#10365](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10365). Thanks to @ab0utbla-k for the contribution.
* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix inaccurate `vm_filestream_write_duration_seconds_total` due to duplicate counting . After the fix, `vm_filestream_write_duration_seconds_total` will track the duration spent on calling the `write(2)` system call properly. See [#10564](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10564).
## [v1.136.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.1)
Released at 2026-03-02
@@ -252,6 +277,24 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
## [v1.122.17](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.17)
Released at 2026-03-13
**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
* SECURITY: upgrade Go builder from Go1.24.13 to Go1.25.8. See [the list of issues addressed in Go1.25.8](https://github.com/golang/go/issues?q=milestone%3AGo1.25.8+label%3ACherryPickApproved).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add [histogram_fraction](https://docs.victoriametrics.com/victoriametrics/metricsql/#histogram_fraction) function to calculate the fraction of buckets falling between lowerLe and upperLe. See [#5346](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5346).
* BUGFIX: all VictoriaMetrics components: replace `histogram` with `untyped` metric metadata type for [VictoriaMetrics histograms](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) when `-metrics.exposeMetadata` is set. See [#82](https://github.com/VictoriaMetrics/metrics/issues/82).
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly route requests to `default_url`. Previously, `request_path` query arg could be set incorrectly during concurrent requests. See [#10626](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10626).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly attach `host` label to the time series ingested via [/datadog/api/beta/sketches](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/#) API. See [#10557](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): use `increase_pure` instead of `rate` for histogram heatmaps in Explore Metrics to correctly display the first observation in each new bucket. See [#10365](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10365). Thanks to @ab0utbla-k for the contribution.
* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix inaccurate `vm_filestream_write_duration_seconds_total` due to duplicate counting . After the fix, `vm_filestream_write_duration_seconds_total` will track the duration spent on calling the `write(2)` system call properly. See [#10564](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10564).
## [v1.122.16](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.16)
Released at 2026-02-27

View File

@@ -117,7 +117,7 @@ It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [
Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
and [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.137.0-enterprise.tar.gz`.
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.138.0-enterprise.tar.gz`.
In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -135,8 +135,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
```sh
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.137.0/victoria-metrics-linux-amd64-v1.137.0-enterprise.tar.gz
tar -xzf victoria-metrics-linux-amd64-v1.137.0-enterprise.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.138.0/victoria-metrics-linux-amd64-v1.138.0-enterprise.tar.gz
tar -xzf victoria-metrics-linux-amd64-v1.138.0-enterprise.tar.gz
./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
```
@@ -151,7 +151,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.137.0-enterprise`.
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.138.0-enterprise`.
In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
flag as described in the [binary-releases](#binary-releases) section.
@@ -161,13 +161,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
```sh
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.137.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.138.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
```
Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
```sh
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.137.0-enterprise -licenseFile=/path/to/vm-license
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.138.0-enterprise -licenseFile=/path/to/vm-license
```
Example docker-compose configuration:
@@ -177,7 +177,7 @@ version: "3.5"
services:
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.137.0
image: victoriametrics/victoria-metrics:v1.138.0
ports:
- 8428:8428
volumes:
@@ -209,7 +209,7 @@ is used to provide the license key in plain-text:
```yaml
server:
image:
tag: v1.137.0-enterprise
tag: v1.138.0-enterprise
license:
key: {BASE64_ENCODED_LICENSE_KEY}
@@ -220,7 +220,7 @@ In order to provide the license key via existing secret, the following values fi
```yaml
server:
image:
tag: v1.137.0-enterprise
tag: v1.138.0-enterprise
license:
secret:
@@ -270,7 +270,7 @@ spec:
license:
key: {BASE64_ENCODED_LICENSE_KEY}
image:
tag: v1.137.0-enterprise
tag: v1.138.0-enterprise
```
In order to provide the license key via an existing secret, the following custom resource is used:
@@ -287,7 +287,7 @@ spec:
name: vm-license
key: license
image:
tag: v1.137.0-enterprise
tag: v1.138.0-enterprise
```
Example secret with license key:
@@ -338,7 +338,7 @@ Builds are available for amd64 and arm64 architectures.
Example archive:
`victoria-metrics-linux-amd64-v1.137.0-enterprise.tar.gz`
`victoria-metrics-linux-amd64-v1.138.0-enterprise.tar.gz`
Includes:
@@ -347,7 +347,7 @@ Includes:
Example Docker image:
`victoriametrics/victoria-metrics:v1.137.0-enterprise-fips` uses the FIPS-compatible binary and based on `scratch` image.
`victoriametrics/victoria-metrics:v1.138.0-enterprise-fips` uses the FIPS-compatible binary and based on `scratch` image.
## Monitoring license expiration

View File

@@ -35,8 +35,8 @@ scrape_configs:
After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
```sh
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.137.0/victoria-metrics-linux-amd64-v1.137.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.137.0.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.138.0/victoria-metrics-linux-amd64-v1.138.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.138.0.tar.gz
```
Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
```yaml
# Download and unpack single-node VictoriaMetrics
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.137.0/victoria-metrics-linux-amd64-v1.137.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.137.0.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.138.0/victoria-metrics-linux-amd64-v1.138.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.138.0.tar.gz
# Run single-node VictoriaMetrics with the given scrape.yaml
./victoria-metrics-prod -promscrape.config=scrape.yaml

View File

@@ -1803,6 +1803,13 @@ scrape_configs:
#
# api_endpoint: "https://api.cloud.yandex.net"
# folder_ids is an optional list of folder IDs for direct discovery.
# If set, only the given folders are queried and the folder enumeration
# step is skipped.
#
# folder_ids:
# - "b1gxxxxxxxxxxxxxxxxx"
# yandex_passport_oauth_token is an optional OAuth token
# for querying yandexcloud API. See https://cloud.yandex.com/en-ru/docs/iam/concepts/authorization/oauth-token
#

View File

@@ -45,7 +45,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
-dryRun
Whether to check config files without running VictoriaMetrics. The following config files are checked: -promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag
-enableMetadata
Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock
Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
-enableTCP6
Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
-envflag.enable
@@ -287,7 +287,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
-promscrape.dockerswarmSDCheckInterval duration
Interval for checking for changes in dockerswarm. This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#dockerswarm_sd_configs for details (default 30s)
-promscrape.dropOriginalLabels
Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. Note that this reduces debuggability for improper per-target relabeling configs (default false)
Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. Note that this reduces debuggability for improper per-target relabeling configs
-promscrape.ec2SDCheckInterval duration
Interval for checking for changes in ec2. This works only if ec2_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#ec2_sd_configs for details (default 1m0s)
-promscrape.eurekaSDCheckInterval duration
@@ -556,7 +556,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
-storage.minFreeDiskSpaceBytes size
The minimum free disk space at -storageDataPath after which the storage stops accepting new data
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 10000000)
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 100000000)
-storage.trackMetricNamesStats
Whether to track ingest and query requests for timeseries metric names. This feature allows to track metric names unused at query requests. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage (default true)
-storageDataPath string

View File

@@ -598,7 +598,7 @@ and attaches `instance`, `job` and other target-specific labels to these metrics
The limit can be set via `label_limit` option at [scrape_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs).
This metric is exposed only if the `label_limit` is set.
* `scrape_series_added` - **an approximate** number of new [series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) the given target generates during the current scrape.
* `scrape_series_added` - **an approximate** number of exposed [series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) the given target changes during the current scrape.
This metric allows detecting targets (identified by `instance` label),
which lead to [high churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate).
For example, the following [MetricsQL query](https://docs.victoriametrics.com/victoriametrics/metricsql/) returns targets,
@@ -611,6 +611,7 @@ and attaches `instance`, `job` and other target-specific labels to these metrics
`vmagent` sets `scrape_series_added` to zero when it runs with `-promscrape.noStaleMarkers` command-line flag
or when it scrapes target with `no_stale_markers: true` option, e.g. when [staleness markers](#prometheus-staleness-markers) are disabled.
When `vmagent` starts, the `scrape_series_added` for all targets will be increased because all series are newly added for a newly started `vmagent`.
* `scrape_series_limit` - the limit on the number of unique [series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) the given target can expose according to [these docs](#cardinality-limiter).
This metric is exposed only if the series limit is set.

View File

@@ -33,7 +33,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
-dryRun
Whether to check config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag
-enableMetadata
Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock
Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
-enableMultitenantHandlers
Whether to process incoming data via multitenant insert handlers according to https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers according to https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data .See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy for details
-enableTCP6
@@ -253,7 +253,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
-promscrape.dockerswarmSDCheckInterval duration
Interval for checking for changes in dockerswarm. This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#dockerswarm_sd_configs for details (default 30s)
-promscrape.dropOriginalLabels
Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. Note that this reduces debuggability for improper per-target relabeling configs (default false)
Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. Note that this reduces debuggability for improper per-target relabeling configs
-promscrape.ec2SDCheckInterval duration
Interval for checking for changes in ec2. This works only if ec2_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#ec2_sd_configs for details (default 1m0s)
-promscrape.eurekaSDCheckInterval duration

View File

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
Download and unpack vmctl:
```sh
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.137.0/vmutils-darwin-arm64-v1.137.0.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.138.0/vmutils-darwin-arm64-v1.138.0.tar.gz
tar xzf vmutils-darwin-arm64-v1.137.0.tar.gz
tar xzf vmutils-darwin-arm64-v1.138.0.tar.gz
```
Once binary is unpacked, see the full list of supported modes by running the following command:

View File

@@ -29,7 +29,7 @@ OPTIONS:
--prom-filter-time-end value The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
--prom-filter-label value Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.
--prom-filter-label-value value Prometheus regular expression to filter label from "prom-filter-label" flag. (default: ".*")
--prom-tmp-dir-path value Path to directory to be used for temporary files. (default: "/var/folders/ds/3kj5p3v17ll0hsyvq380ryvm0000gn/T/")
--prom-tmp-dir-path value Path to directory to be used for temporary files. (default: "/tmp")
--vm-addr value VictoriaMetrics address to perform import requests.
Should be the same as --httpListenAddr value for single-node version or vminsert component.
When importing into the clustered version do not forget to set additionally --vm-account-id flag.

View File

@@ -37,7 +37,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
-dropSamplesOnOverload
Whether to drop incoming samples if the destination vmstorage node is overloaded and/or unavailable. This prioritizes cluster availability over consistency, e.g. the cluster continues accepting all the ingested samples, but some of them may be dropped if vmstorage nodes are temporarily unavailable and/or overloaded. The drop of samples happens before the replication, so it's not recommended to use this flag with -replicationFactor enabled.
-enableMetadata
Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock
Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
-enableTCP6
Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
-envflag.enable

View File

@@ -232,7 +232,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
-storage.minFreeDiskSpaceBytes size
The minimum free disk space at -storageDataPath after which the storage stops accepting new data
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 10000000)
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 100000000)
-storage.trackMetricNamesStats
Whether to track ingest and query requests for timeseries metric names. This feature allows to track metric names unused at query requests. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage (default true)
-storage.vminsertConnsShutdownDuration duration

View File

@@ -949,6 +949,13 @@ func (tb *Table) mustMergeInmemoryParts(pws []*partWrapper) []*partWrapper {
return pwsResult
}
// mustMergeInmemoryPartsFinal merges the given in-memory part wrappers (pws)
// into a single new in-memory part wrapper.
//
// It panics if the input slice pws is empty (though the caller should prevent
// this). If the pws contains only one element, it is returned as is. Finally,
// when len(pws) > 1, the source pws are merged, and their ref count is
// decremented.
func (tb *Table) mustMergeInmemoryPartsFinal(pws []*partWrapper) *partWrapper {
if len(pws) == 0 {
logger.Panicf("BUG: pws must contain at least a single item")
@@ -969,7 +976,11 @@ func (tb *Table) mustMergeInmemoryPartsFinal(pws []*partWrapper) *partWrapper {
}
flushToDiskDeadline := getFlushToDiskDeadline(pws, tb.flushInterval)
return tb.mustMergeIntoInmemoryPart(bsrs, flushToDiskDeadline)
pw := tb.mustMergeIntoInmemoryPart(bsrs, flushToDiskDeadline)
for _, srcPW := range pws {
srcPW.decRef()
}
return pw
}
func (tb *Table) createInmemoryPart(ibs []*inmemoryBlock) *partWrapper {

View File

@@ -301,3 +301,56 @@ func testReopenTable(t *testing.T, path string, itemsCount int) {
tb.MustClose()
}
}
func TestTableMustMergeInmemoryPartsFinal_pwsRefCount(t *testing.T) {
path := t.Name()
fs.MustRemoveDir(path)
defer fs.MustRemoveDir(path)
var isReadOnly atomic.Bool
tb := MustOpenTable(path, 0, nil, nil, &isReadOnly)
defer tb.MustClose()
generatePartWrappers := func(n int) []*partWrapper {
pws := make([]*partWrapper, n)
for i := range n {
var ib inmemoryBlock
items := bytes.Repeat([]byte{byte(i)}, 1024)
ib.Add(items)
pw := tb.createInmemoryPart([]*inmemoryBlock{&ib})
pws[i] = pw
}
return pws
}
assertRefCount := func(pws []*partWrapper, want int32) {
t.Helper()
for _, pw := range pws {
if got := pw.refCount.Load(); got != want {
t.Fatalf("unexpected inmemory part wrapper ref count: got %d, want %d", got, want)
}
}
}
var (
pwsSrc []*partWrapper
pwFinal *partWrapper
)
// single source part wrapper
pwsSrc = generatePartWrappers(1)
assertRefCount(pwsSrc, 1)
pwFinal = tb.mustMergeInmemoryPartsFinal(pwsSrc)
if pwFinal != pwsSrc[0] {
t.Fatalf("mustMergeInmemoryPartsFinal must return the original wrapper for a single source part")
}
assertRefCount(pwsSrc, 1)
assertRefCount([]*partWrapper{pwFinal}, 1)
// many source part wrappers
pwsSrc = generatePartWrappers(100)
assertRefCount(pwsSrc, 1)
pwFinal = tb.mustMergeInmemoryPartsFinal(pwsSrc)
assertRefCount(pwsSrc, 0)
assertRefCount([]*partWrapper{pwFinal}, 1)
}

View File

@@ -33,6 +33,7 @@ type apiConfig struct {
client *http.Client
yandexPassportOAuth *yandexPassportOAuth
serviceEndpoints map[string]string
folderIDs []string
// credsLock protects the refresh of creds
credsLock sync.Mutex
@@ -67,6 +68,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
client: &http.Client{
Transport: rt,
},
folderIDs: sdc.FolderIDs,
}
apiEndpoint := sdc.APIEndpoint
if apiEndpoint == "" {

View File

@@ -9,22 +9,30 @@ import (
)
func getInstancesLabels(cfg *apiConfig) ([]*promutil.Labels, error) {
organizations, err := cfg.getOrganizations()
if err != nil {
return nil, err
}
clouds, err := cfg.getClouds(organizations)
if err != nil {
return nil, err
}
folders, err := cfg.getFolders(clouds)
if err != nil {
return nil, err
var folderIDs []string
if len(cfg.folderIDs) > 0 {
folderIDs = cfg.folderIDs
} else {
organizations, err := cfg.getOrganizations()
if err != nil {
return nil, err
}
clouds, err := cfg.getClouds(organizations)
if err != nil {
return nil, err
}
folders, err := cfg.getFolders(clouds)
if err != nil {
return nil, err
}
for _, fld := range folders {
folderIDs = append(folderIDs, fld.ID)
}
}
var instances []instance
for _, fld := range folders {
inst, err := cfg.getInstances(fld.ID)
for _, folderID := range folderIDs {
inst, err := cfg.getInstances(folderID)
if err != nil {
return nil, err
}

View File

@@ -21,6 +21,7 @@ type SDConfig struct {
Service string `yaml:"service"`
YandexPassportOAuthToken *promauth.Secret `yaml:"yandex_passport_oauth_token,omitempty"`
APIEndpoint string `yaml:"api_endpoint,omitempty"`
FolderIDs []string `yaml:"folder_ids,omitempty"`
TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"`
}

View File

@@ -577,7 +577,8 @@ func (sw *scrapeWork) processDataOneShot(scrapeTimestamp, realTimestamp int64, b
// The returned value for seriesAdded may be bigger than the real number of added series
// if some series were removed during relabeling.
// This is a trade-off between performance and accuracy.
seriesAdded = getSeriesAdded(lastScrapeStr, bodyString)
// The value can be negative if the number of exposed series decreased or the scrape failed.
seriesAdded = getSeriesDiff(lastScrapeStr, bodyString)
}
samplesDropped := 0
if sw.seriesLimitExceeded.Load() || !areIdenticalSeries {
@@ -688,7 +689,8 @@ func (sw *scrapeWork) processDataInStreamMode(scrapeTimestamp, realTimestamp int
// The returned value for seriesAdded may be bigger than the real number of added series
// if some series were removed during relabeling.
// This is a trade-off between performance and accuracy.
seriesAdded = getSeriesAdded(lastScrapeStr, bodyString)
// The value can be negative if the number of exposed series decreased or the scrape failed.
seriesAdded = getSeriesDiff(lastScrapeStr, bodyString)
}
responseSize := len(bodyString)
@@ -813,12 +815,9 @@ func (wc *writeRequestCtx) reset() {
var writeRequestCtxPool leveledWriteRequestCtxPool
func getSeriesAdded(lastScrape, currScrape string) int {
if currScrape == "" {
return 0
}
bodyString := parser.GetRowsDiff(currScrape, lastScrape)
return strings.Count(bodyString, "\n")
func getSeriesDiff(lastScrape, currScrape string) int {
added, deleted := parser.GetRowsDiff(currScrape, lastScrape)
return strings.Count(added, "\n") - strings.Count(deleted, "\n")
}
func (sw *scrapeWork) initSeriesLimiter() {
@@ -876,7 +875,7 @@ func (sw *scrapeWork) sendStaleSeries(lastScrape, currScrape string, timestamp i
}
bodyString := lastScrape
if currScrape != "" {
bodyString = parser.GetRowsDiff(lastScrape, currScrape)
bodyString, _ = parser.GetRowsDiff(lastScrape, currScrape)
}
if bodyString != "" {
// Send stale markers in streaming mode in order to reduce memory usage

View File

@@ -485,10 +485,11 @@ func prevBackslashesCount(s string) int {
return n
}
// GetRowsDiff returns rows from s1, which are missing in s2.
//
// GetRowsDiff
// The first returned string contains rows from s1, which are missing in s2.
// The second returned string contains rows from s2, which are missing in s1.
// The returned rows have default value 0 and have no timestamps.
func GetRowsDiff(s1, s2 string) string {
func GetRowsDiff(s1, s2 string) (string, string) {
li1 := getLinesIterator()
li2 := getLinesIterator()
defer func() {
@@ -497,33 +498,48 @@ func GetRowsDiff(s1, s2 string) string {
}()
li1.Init(s1)
li2.Init(s2)
if !li1.NextKey() {
return ""
var diff1, diff2 []byte
has1 := li1.NextKey()
has2 := li2.NextKey()
if !has1 && !has2 {
return "", ""
}
var diff []byte
if !li2.NextKey() {
diff = appendKeys(diff, li1)
return string(diff)
if !has1 {
diff2 = appendKeys(diff2, li2)
return "", string(diff2)
}
if !has2 {
diff1 = appendKeys(diff1, li1)
return string(diff1), ""
}
for {
switch bytes.Compare(li1.Key, li2.Key) {
case -1:
diff = appendKey(diff, li1.Key)
diff1 = appendKey(diff1, li1.Key)
if !li1.NextKey() {
return string(diff)
diff2 = appendKeys(diff2, li2)
return string(diff1), string(diff2)
}
case 1:
diff2 = appendKey(diff2, li2.Key)
if !li2.NextKey() {
diff1 = appendKeys(diff1, li1)
return string(diff1), string(diff2)
}
case 0:
if !li1.NextKey() {
return string(diff)
if !li2.NextKey() {
return string(diff1), string(diff2)
}
diff2 = appendKeys(diff2, li2)
return string(diff1), string(diff2)
}
if !li2.NextKey() {
diff = appendKeys(diff, li1)
return string(diff)
}
case 1:
if !li2.NextKey() {
diff = appendKeys(diff, li1)
return string(diff)
diff1 = appendKeys(diff1, li1)
return string(diff1), string(diff2)
}
}
}

View File

@@ -7,21 +7,24 @@ import (
)
func TestGetRowsDiff(t *testing.T) {
f := func(s1, s2, resultExpected string) {
f := func(s1, s2, addedExpected, deletedExpected string) {
t.Helper()
result := GetRowsDiff(s1, s2)
if result != resultExpected {
t.Fatalf("unexpected result for GetRowsDiff(%q, %q); got %q; want %q", s1, s2, result, resultExpected)
added, deleted := GetRowsDiff(s1, s2)
if added != addedExpected {
t.Fatalf("unexpected added result for GetRowsDiff(%q, %q); got %q; want %q", s1, s2, added, addedExpected)
}
if deleted != deletedExpected {
t.Fatalf("unexpected deleted result for GetRowsDiff(%q, %q); got %q; want %q", s1, s2, deleted, deletedExpected)
}
}
f("", "", "")
f("", "foo 1", "")
f(" ", "foo 1", "")
f("foo 123", "", "foo 0\n")
f("foo 123", "bar 3", "foo 0\n")
f("foo 123", "bar 3\nfoo 344", "")
f("foo{x=\"y\", z=\"a a a\"} 123", "bar 3\nfoo{x=\"y\", z=\"b b b\"} 344", "foo{x=\"y\",z=\"a a a\"} 0\n")
f("foo{bar=\"baz\"} 123\nx 3.4 5\ny 5 6", "x 34 342", "foo{bar=\"baz\"} 0\ny 0\n")
f("", "", "", "")
f("", "foo 1", "", "foo 0\n")
f(" ", "foo 1", "", "foo 0\n")
f("foo 123", "", "foo 0\n", "")
f("foo 123", "bar 3", "foo 0\n", "bar 0\n")
f("foo 123", "bar 3\nfoo 344", "", "bar 0\n")
f("foo{x=\"y\", z=\"a a a\"} 123", "bar 3\nfoo{x=\"y\", z=\"b b b\"} 344", "foo{x=\"y\",z=\"a a a\"} 0\n", "bar 0\nfoo{x=\"y\",z=\"b b b\"} 0\n")
f("foo{bar=\"baz\"} 123\nx 3.4 5\ny 5 6", "x 34 342", "foo{bar=\"baz\"} 0\ny 0\n", "")
}
func TestAreIdenticalSeriesFast(t *testing.T) {

View File

@@ -137,9 +137,9 @@ container_ulimits_soft{container="kube-scheduler",id="/kubelet/kubepods/burstabl
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
diff := GetRowsDiff(s2, s1)
if diff != "foo 0\n" {
panic(fmt.Errorf("unexpected diff; got %q; want %q", diff, "foo 0\n"))
added, _ := GetRowsDiff(s2, s1)
if added != "foo 0\n" {
panic(fmt.Errorf("unexpected diff; got %q; want %q", added, "foo 0\n"))
}
}
})

View File

@@ -803,10 +803,17 @@ func (pt *partition) mustMergeInmemoryParts(pws []*partWrapper) []*partWrapper {
return pwsResult
}
// mustMergeInmemoryPartsFinal merges the given in-memory part wrappers (pws) into a single new in-memory part wrapper.
// It panics if the input slice pws is empty (though the caller should prevent this).
// Returns nil if the merge results in an empty part (e.g., due to retention filters removing all data).
// Otherwise, returns the wrapper for the merged part.
// mustMergeInmemoryPartsFinal merges the given in-memory part wrappers (pws)
// into a single new in-memory part wrapper.
//
// It panics if the input slice pws is empty (though the caller should prevent
// this). If the pws contains only one element, it is returned as is. Finally,
// when len(pws) > 1, the source pws are merged, and their ref count is
// decremented.
//
// Returns nil if the merge results in an empty part (e.g., due to retention
// filters removing all data). Otherwise, returns the wrapper for the merged
// part.
func (pt *partition) mustMergeInmemoryPartsFinal(pws []*partWrapper) *partWrapper {
if len(pws) == 0 {
logger.Panicf("BUG: pws must contain at least a single item")
@@ -853,6 +860,9 @@ func (pt *partition) mustMergeInmemoryPartsFinal(pws []*partWrapper) *partWrappe
if err != nil {
logger.Panicf("FATAL: cannot merge inmemoryBlocks: %s", err)
}
for _, pw := range pws {
pw.decRef()
}
// The resulting part is empty, no need to create a part wrapper
if ph.BlocksCount == 0 {

View File

@@ -183,10 +183,9 @@ func TestMergeInMemoryPartsEmptyResult(t *testing.T) {
rows[i].PrecisionBits = 64
}
pws = append(pws, &partWrapper{
mp: newTestInmemoryPart(rows),
p: &part{},
})
mp := newTestInmemoryPart(rows)
pw := newPartWrapperFromInmemoryPart(mp, time.Time{})
pws = append(pws, pw)
}
pwsNew := pt.mustMergeInmemoryParts(pws)
@@ -195,6 +194,68 @@ func TestMergeInMemoryPartsEmptyResult(t *testing.T) {
}
}
func TestMergeInMemoryPartsFinal_pwsRefCount(t *testing.T) {
defer testRemoveAll(t)
generatePartWrappers := func(n int) []*partWrapper {
var pws []*partWrapper
for range n {
var rows []rawRow
for i := range 10 {
row := rawRow{
TSID: TSID{MetricID: uint64(i)},
Value: float64(i),
Timestamp: time.Now().UnixMilli() + int64(i),
PrecisionBits: 64,
}
rows = append(rows, row)
}
var mp inmemoryPart
mp.InitFromRows(rows)
pw := newPartWrapperFromInmemoryPart(&mp, time.Time{})
pws = append(pws, pw)
}
return pws
}
assertRefCount := func(pws []*partWrapper, want int32) {
t.Helper()
for _, pw := range pws {
if got := pw.refCount.Load(); got != want {
t.Fatalf("unexpected inmemory part wrapper ref count: got %d, want %d", got, want)
}
}
}
s := MustOpenStorage(t.Name(), OpenOptions{})
defer s.MustClose()
ptw := s.tb.MustGetPartition(time.Now().UnixMilli())
defer s.tb.PutPartition(ptw)
pt := ptw.pt
var (
pwsSrc []*partWrapper
pwFinal *partWrapper
)
// single source part wrapper
pwsSrc = generatePartWrappers(1)
assertRefCount(pwsSrc, 1)
pwFinal = pt.mustMergeInmemoryPartsFinal(pwsSrc)
if pwFinal != pwsSrc[0] {
t.Fatalf("mustMergeInmemoryPartsFinal must return the original wrapper for a single source part")
}
assertRefCount(pwsSrc, 1)
assertRefCount([]*partWrapper{pwFinal}, 1)
// many source part wrappers
pwsSrc = generatePartWrappers(100)
assertRefCount(pwsSrc, 1)
pwFinal = pt.mustMergeInmemoryPartsFinal(pwsSrc)
assertRefCount(pwsSrc, 0)
assertRefCount([]*partWrapper{pwFinal}, 1)
}
func testCreatePartition(t *testing.T, timestamp int64, s *Storage) *partition {
t.Helper()
small := filepath.Join(t.Name(), smallDirname)