mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-04 09:31:57 +03:00
Compare commits
14 Commits
vmauth-rea
...
v1.134.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
eaf7a68c92 | ||
|
|
c5e43e1c91 | ||
|
|
b343f541f0 | ||
|
|
a23a902953 | ||
|
|
54c60706ca | ||
|
|
cd2e11b7cf | ||
|
|
5423d5e93a | ||
|
|
48819b6781 | ||
|
|
c4bff27f46 | ||
|
|
432b313a48 | ||
|
|
7bd5d19f62 | ||
|
|
8d18bc288f | ||
|
|
ff6e5c2983 | ||
|
|
23af0086d8 |
@@ -869,17 +869,17 @@ func getScrapeInterval(timestamps []int64, defaultInterval int64) int64 {
|
||||
return defaultInterval
|
||||
}
|
||||
|
||||
// Estimate scrape interval as 0.6 quantile for the first 20 intervals.
|
||||
tsPrev := timestamps[0]
|
||||
timestamps = timestamps[1:]
|
||||
// Estimate scrape interval as 0.6 quantile of the last 20 intervals.
|
||||
tsPrev := timestamps[len(timestamps)-1]
|
||||
timestamps = timestamps[:len(timestamps)-1]
|
||||
if len(timestamps) > 20 {
|
||||
timestamps = timestamps[:20]
|
||||
timestamps = timestamps[len(timestamps)-20:]
|
||||
}
|
||||
a := getFloat64s()
|
||||
intervals := a.A[:0]
|
||||
for _, ts := range timestamps {
|
||||
intervals = append(intervals, float64(ts-tsPrev))
|
||||
tsPrev = ts
|
||||
for i := len(timestamps) - 1; i >= 0; i-- {
|
||||
intervals = append(intervals, float64(tsPrev-timestamps[i]))
|
||||
tsPrev = timestamps[i]
|
||||
}
|
||||
scrapeInterval := int64(quantile(0.6, intervals))
|
||||
a.A = intervals
|
||||
|
||||
209
app/vmselect/vmui/assets/index-B6lol36n.js
Normal file
209
app/vmselect/vmui/assets/index-B6lol36n.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/vmselect/vmui/assets/index-VQRcNK83.css
Normal file
1
app/vmselect/vmui/assets/index-VQRcNK83.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
66
app/vmselect/vmui/assets/vendor-EZef-S_8.js
Normal file
66
app/vmselect/vmui/assets/vendor-EZef-S_8.js
Normal file
File diff suppressed because one or more lines are too long
@@ -37,10 +37,10 @@
|
||||
<meta property="og:title" content="UI for VictoriaMetrics">
|
||||
<meta property="og:url" content="https://victoriametrics.com/">
|
||||
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
|
||||
<script type="module" crossorigin src="./assets/index-Clpj_g75.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-D5YL0cqB.js">
|
||||
<script type="module" crossorigin src="./assets/index-B6lol36n.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-EZef-S_8.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-jEWkrqzO.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-VQRcNK83.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25.5 AS build-web-stage
|
||||
FROM golang:1.25.6 AS build-web-stage
|
||||
COPY build /build
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
@@ -23,7 +23,18 @@ export const countsToFills = (u: uPlot, seriesIdx: number) => {
|
||||
}
|
||||
}
|
||||
|
||||
// no valid counts
|
||||
if (!isFinite(minCount) || !isFinite(maxCount)) {
|
||||
return counts.map(() => -1);
|
||||
}
|
||||
|
||||
const range = maxCount - minCount;
|
||||
|
||||
// all counts are the same
|
||||
if (range === 0) {
|
||||
return counts.map(c => (c > hideThreshold ? 0 : -1));
|
||||
}
|
||||
|
||||
const paletteSize = palette.length;
|
||||
const indexedFills = Array(counts.length);
|
||||
|
||||
@@ -40,9 +51,9 @@ export const heatmapPaths = () => (u: uPlot, seriesIdx: number) => {
|
||||
const cellGap = Math.round(devicePixelRatio);
|
||||
|
||||
uPlot.orient(u, seriesIdx, (
|
||||
series,
|
||||
dataX,
|
||||
dataY,
|
||||
_series,
|
||||
_dataX,
|
||||
_dataY,
|
||||
scaleX,
|
||||
scaleY,
|
||||
valToPosX,
|
||||
@@ -51,8 +62,8 @@ export const heatmapPaths = () => (u: uPlot, seriesIdx: number) => {
|
||||
yOff,
|
||||
xDim,
|
||||
yDim,
|
||||
moveTo,
|
||||
lineTo,
|
||||
_moveTo,
|
||||
_lineTo,
|
||||
rect
|
||||
) => {
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
@@ -80,7 +91,7 @@ export const heatmapPaths = () => (u: uPlot, seriesIdx: number) => {
|
||||
const cys = ys.slice(0, yBinQty).map((y: number) => {
|
||||
return Math.round(valToPosY(y, scaleY, yDim, yOff) - ySize / 2);
|
||||
});
|
||||
const cxs = Array.from({ length: xBinQty }, (v, i) => {
|
||||
const cxs = Array.from({ length: xBinQty }, (_v, i) => {
|
||||
return Math.round(valToPosX(xs[i * yBinQty], scaleX, xDim, xOff) - xSize);
|
||||
});
|
||||
|
||||
@@ -114,7 +125,7 @@ export const heatmapPaths = () => (u: uPlot, seriesIdx: number) => {
|
||||
export const convertPrometheusToVictoriaMetrics = (buckets: MetricResult[]): MetricResult[] => {
|
||||
if (!buckets.every(a => a.metric.le)) return buckets;
|
||||
|
||||
const sortedBuckets = buckets.sort((a,b) => parseFloat(a.metric.le) - parseFloat(b.metric.le));
|
||||
const sortedBuckets = buckets.sort((a, b) => parseFloat(a.metric.le) - parseFloat(b.metric.le));
|
||||
const group = buckets[0]?.group || 1;
|
||||
let prevBucket: MetricResult = { metric: { le: "" }, values: [], group };
|
||||
const result: MetricResult[] = [];
|
||||
@@ -169,5 +180,29 @@ export const normalizeData = (buckets: MetricResult[], isHistogram?: boolean): M
|
||||
return { ...bucket, values };
|
||||
}) as MetricResult[];
|
||||
|
||||
return result.filter(r => !r.values.every(v => v[1] === "0"));
|
||||
// Indices of buckets that have any non-zero values
|
||||
const idxsWithData = result
|
||||
.map((r, i) => (r.values.every(v => v[1] === "0") ? -1 : i))
|
||||
.filter(i => i !== -1);
|
||||
|
||||
const countWithData = idxsWithData.length;
|
||||
|
||||
// No data at all, or too few buckets to bother slicing
|
||||
if (countWithData === 0 || result.length <= 3) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// More than one non-empty bucket: keep only buckets with data
|
||||
if (countWithData > 1) {
|
||||
return result.filter((_, i) => idxsWithData.includes(i));
|
||||
}
|
||||
|
||||
// Keep the only non-empty bucket plus its adjacent buckets (if available)
|
||||
const idx = idxsWithData[0];
|
||||
const keep = new Set<number>([idx]);
|
||||
|
||||
if (idx - 1 >= 0) keep.add(idx - 1);
|
||||
if (idx + 1 < result.length) keep.add(idx + 1);
|
||||
|
||||
return result.filter((_, i) => keep.has(i));
|
||||
};
|
||||
|
||||
@@ -193,9 +193,10 @@ func testDeduplication(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier,
|
||||
}},
|
||||
{Metric: map[string]string{"__name__": "metric4"}, Samples: []*apptest.Sample{
|
||||
// If multiple raw samples have the same timestamp on the
|
||||
// given -dedup.minScrapeInterval discrete interval, then
|
||||
// stale markers are preferred over any other value.
|
||||
{Timestamp: ts10, Value: decimal.StaleNaN},
|
||||
// given -dedup.minScrapeInterval discrete interval,
|
||||
// always prefer a non-decimal.StaleNaN value,
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196
|
||||
{Timestamp: ts10, Value: 50},
|
||||
}},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.23.2
|
||||
ROOT_IMAGE_SCRATCH ?= scratch
|
||||
CERTS_IMAGE := alpine:3.23.2
|
||||
|
||||
GO_BUILDER_IMAGE := golang:1.25.5
|
||||
GO_BUILDER_IMAGE := golang:1.25.6
|
||||
|
||||
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
|
||||
BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
|
||||
|
||||
@@ -8,6 +8,7 @@ menu:
|
||||
tags: []
|
||||
aliases:
|
||||
- /CaseStudies.html
|
||||
- /casestudies.html
|
||||
- /casestudies/index.html
|
||||
- /casestudies/
|
||||
---
|
||||
|
||||
@@ -1345,7 +1345,7 @@ This aligns with the [staleness rules in Prometheus](https://prometheus.io/docs/
|
||||
|
||||
If multiple raw samples have **the same timestamp** on the given `-dedup.minScrapeInterval` discrete interval,
|
||||
then the sample with **the biggest value** is kept.
|
||||
[Stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers) are preferred over any other value.
|
||||
Numerical values are preferred over [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers).
|
||||
|
||||
[Prometheus staleness markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers) are processed as any other value during de-duplication.
|
||||
If raw sample with the biggest timestamp on `-dedup.minScrapeInterval` contains a stale marker, then it is kept after the deduplication.
|
||||
|
||||
@@ -26,20 +26,30 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
## tip
|
||||
|
||||
## [v1.134.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.134.0)
|
||||
|
||||
Released at 2026-01-16
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.25.5 to Go1.25.5. See [the list of issues addressed in Go1.25.6](https://github.com/golang/go/issues?q=milestone%3AGo1.25.6%20label%3ACherryPickApproved).
|
||||
|
||||
* FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): refine `VictoriaMetrics - single` dashboard and aligned it with the [VictoriaMetrics - cluster](https://grafana.com/grafana/dashboards/11176) dashboard. For the full list of changes see [#10132-comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10187#issuecomment-3696769466) and [#10260](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10260).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): add `vm_persistentqueue_free_disk_space_bytes` metric for vmagent's persistentqueue capacity. See [#10193](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10193).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): add `vm_persistentqueue_free_disk_space_bytes` metric for vmagent's persistent queue capacity. See [#10193](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10193).
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `vm_rollup_result_cache_requests_total` which tracks the number of requests to the query rollup cache. See [#10117](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10117).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add `localStorage` availability checks with error reporting. See [#10085](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10085).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add `VMUI:`-prefixed `localStorage` keys and legacy key migration.
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add a metric `vmauth_http_request_errors_total{reason="client_canceled"}` to measure client cancelled requests. This should help with debugging vmauth issues. See [#10233](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10233).
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): do not retry client canceled requests. See [#10233](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10233).
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add explicit month duration unit (`M`) for `-retentionPeriod` flag. This allows users to specify retention periods in months more explicitly. See [#10181](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10181).
|
||||
* FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add `Persistent queue Full ETA` panel to the `Drilldown` section. This panel helps estimate how much time remains until `vmagent` starts dropping incoming metrics. See [#10193](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10193).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add support for `$isPartial` variable in alerting rule annotation [templating](https://docs.victoriametrics.com/victoriametrics/vmalert/#templating). This allows users to include an additional warning message in alerts triggered by partial query responses. See [#4531](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4531).
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): calculate the lookbehind window as the median of the intervals between the last 20 raw samples within the requested time range for range queries. Previously, this calculation was based on the first 20 samples, using the last 20 samples should improve accuracy for recent data. See [#10281](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10281).
|
||||
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix configuration reloading for `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` when vmagent is launched with empty files. Previously, if vmagent started with an empty config, subsequent config reloads were ignored. See [#10211](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10211).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Fixed a missing path error for `http://<victoriametrics-addr>:8428/zabbixconnector/api/v1/history`. See PR [10214](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10214).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent slow ingestion requests and CPU usage spikes during midnight daily-index creation. See [#10064](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10064).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): fix a missing path error for `http://<victoriametrics-addr>:8428/zabbixconnector/api/v1/history`. See [10214](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10214).
|
||||
* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): reduce default value for `storage.vminsertConnsShutdownDuration` flag from `25s` to `10s` seconds. It reduces probability of ungraceful storage shutdown at Kubernetes based environments, which has 30 seconds default graceful termination period value. See [#10273](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10273)
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): remove legacy `tenantID` query param and use the URL path as the single source of truth for multitenancy. See [#10232](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10232).
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix heatmap rendering issues where charts could break or appear empty when bucket values were uniform or sparse. See [#10240](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10240).
|
||||
* BUGFIX: all VictoriaMetrics components: prefer numerical values over [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers) when samples share the same timestamp during deduplication. See [#10196](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196#issuecomment-3738433938).
|
||||
* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly return results for `/api/v1/labels` and `/api/v1/label/{}/values` when `match[]`, `extra_filters` or `extra_labels` are specified. See [#10294](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10294)
|
||||
|
||||
## [v1.133.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.133.0)
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ Since VictoriaMetrics is Prometheus-compatible TSDB it's possible to use set of
|
||||
* Select `Time Series Chart` in `Type` dropdown.
|
||||
* Select `Prometheus Time Series Query` in `Query Type` dropdown.
|
||||
* Select the Prometheus datasource you configured above from the `Prometheus Datasource` dropdown.
|
||||
* Type a valid [MetricsQL expression](https://docs.victoriametrics.com/victoriametrics/metricsql) you want to build a graph for in `PromQL Expression` field. It may reference a variable defined in variables plugin section using `${var-name}` expression or with specific formats like `${var:pipe}` for regex filters or `${var:csv}` for comma-separated values.
|
||||
* Type a valid [MetricsQL expression](https://docs.victoriametrics.com/victoriametrics/metricsql/) you want to build a graph for in `PromQL Expression` field. It may reference a variable defined in variables plugin section using `${var-name}` expression or with specific formats like `${var:pipe}` for regex filters or `${var:csv}` for comma-separated values.
|
||||
* Click `Add` to save a panel.
|
||||
|
||||

|
||||
|
||||
@@ -727,7 +727,7 @@ response, where most of them are `ephemeral`.
|
||||
|
||||
Sometimes, the lookbehind window for locating the datapoint isn't big enough and the graph will contain a gap. For range
|
||||
queries, lookbehind window isn't equal to the `step` parameter. It is calculated as the median of the intervals between
|
||||
the first 20 raw samples in the requested time range. In this way, VictoriaMetrics automatically adjusts the lookbehind
|
||||
the last 20 raw samples in the requested time range. In this way, VictoriaMetrics automatically adjusts the lookbehind
|
||||
window to fill gaps and detect stale series at the same time.
|
||||
|
||||
Range queries are mostly used for plotting time series data over specified time ranges. These queries are extremely
|
||||
|
||||
@@ -11,6 +11,7 @@ aliases:
|
||||
- /relabeling.html
|
||||
- /relabeling/index.html
|
||||
- /relabeling/
|
||||
- /victoria-metrics/relabeling/
|
||||
---
|
||||
|
||||
The relabeling cookbook provides practical examples and patterns for
|
||||
|
||||
@@ -234,7 +234,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
|
||||
-storage.trackMetricNamesStats
|
||||
Whether to track ingest and query requests for timeseries metric names. This feature allows to track metric names unused at query requests. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage (default true)
|
||||
-storage.vminsertConnsShutdownDuration duration
|
||||
The time needed for gradual closing of vminsert connections during graceful shutdown. Bigger duration reduces spikes in CPU, RAM and disk IO load on the remaining vmstorage nodes during rolling restart. Smaller duration reduces the time needed to close all the vminsert connections, thus reducing the time for graceful shutdown. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#improving-re-routing-performance-during-restart (default 25s)
|
||||
The time needed for gradual closing of vminsert connections during graceful shutdown. Bigger duration reduces spikes in CPU, RAM and disk IO load on the remaining vmstorage nodes during rolling restart. Smaller duration reduces the time needed to close all the vminsert connections, thus reducing the time for graceful shutdown. Configured value must be always lower than graceful shutdown period configured by the orchestration platform (terminationGracePeriodSeconds for Kubernetes). See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#improving-re-routing-performance-during-restart (default 10s)
|
||||
-storageDataPath string
|
||||
Path to storage data (default "vmstorage-data")
|
||||
-tls array
|
||||
|
||||
2
go.mod
2
go.mod
@@ -1,6 +1,6 @@
|
||||
module github.com/VictoriaMetrics/VictoriaMetrics
|
||||
|
||||
go 1.25.5
|
||||
go 1.25.6
|
||||
|
||||
require (
|
||||
cloud.google.com/go/storage v1.57.0
|
||||
|
||||
@@ -220,7 +220,8 @@ func (dmc *dateMetricIDCache) syncLocked() {
|
||||
}
|
||||
|
||||
func (dmc *dateMetricIDCache) startRotation() {
|
||||
d := timeutil.AddJitterToDuration(10 * time.Minute)
|
||||
// 1 hour was chosen based on https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10064#issuecomment-3749046726
|
||||
d := timeutil.AddJitterToDuration(time.Hour)
|
||||
ticker := time.NewTicker(d)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
|
||||
@@ -47,13 +47,16 @@ func DeduplicateSamples(srcTimestamps []int64, srcValues []float64, dedupInterva
|
||||
j := i
|
||||
tsPrev := srcTimestamps[j]
|
||||
vPrev := srcValues[j]
|
||||
// if multiple samples have the same timestamp, choose the maximum value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3333;
|
||||
// always prefer a non-decimal.StaleNaN value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196
|
||||
for j > 0 && srcTimestamps[j-1] == tsPrev {
|
||||
j--
|
||||
if decimal.IsStaleNaN(srcValues[j]) {
|
||||
// always prefer decimal.IsStaleNaN to avoid inconsistency when comparing values
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7674
|
||||
continue
|
||||
}
|
||||
if decimal.IsStaleNaN(vPrev) {
|
||||
vPrev = srcValues[j]
|
||||
break
|
||||
continue
|
||||
}
|
||||
if srcValues[j] > vPrev {
|
||||
vPrev = srcValues[j]
|
||||
@@ -70,14 +73,16 @@ func DeduplicateSamples(srcTimestamps []int64, srcValues []float64, dedupInterva
|
||||
j := len(srcTimestamps) - 1
|
||||
tsPrev := srcTimestamps[j]
|
||||
vPrev := srcValues[j]
|
||||
// Invariant: vPrev > srcValues[j]
|
||||
// if multiple samples have the same timestamp, choose the maximum value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3333;
|
||||
// always prefer a non-decimal.StaleNaN value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196
|
||||
for j > 0 && srcTimestamps[j-1] == tsPrev {
|
||||
j--
|
||||
if decimal.IsStaleNaN(srcValues[j]) {
|
||||
// always prefer decimal.IsStaleNaN to avoid inconsistency when comparing values
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7674
|
||||
continue
|
||||
}
|
||||
if decimal.IsStaleNaN(vPrev) {
|
||||
vPrev = srcValues[j]
|
||||
break
|
||||
continue
|
||||
}
|
||||
if srcValues[j] > vPrev {
|
||||
vPrev = srcValues[j]
|
||||
@@ -106,13 +111,16 @@ func deduplicateSamplesDuringMerge(srcTimestamps, srcValues []int64, dedupInterv
|
||||
j := i
|
||||
tsPrev := srcTimestamps[j]
|
||||
vPrev := srcValues[j]
|
||||
// if multiple samples have the same timestamp, choose the maximum value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3333;
|
||||
// always prefer a non-decimal.StaleNaN value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196
|
||||
for j > 0 && srcTimestamps[j-1] == tsPrev {
|
||||
j--
|
||||
if decimal.IsStaleNaNInt64(srcValues[j]) {
|
||||
// always prefer decimal.IsStaleNaN to avoid inconsistency when comparing values
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7674
|
||||
continue
|
||||
}
|
||||
if decimal.IsStaleNaNInt64(vPrev) {
|
||||
vPrev = srcValues[j]
|
||||
break
|
||||
continue
|
||||
}
|
||||
if srcValues[j] > vPrev {
|
||||
vPrev = srcValues[j]
|
||||
@@ -129,19 +137,16 @@ func deduplicateSamplesDuringMerge(srcTimestamps, srcValues []int64, dedupInterv
|
||||
j := len(srcTimestamps) - 1
|
||||
tsPrev := srcTimestamps[j]
|
||||
vPrev := srcValues[j]
|
||||
if decimal.IsStaleNaNInt64(vPrev) {
|
||||
// fast path - decimal.StaleNaN is always preferred to other values on interval
|
||||
dstTimestamps = append(dstTimestamps, tsPrev)
|
||||
dstValues = append(dstValues, vPrev)
|
||||
return dstTimestamps, dstValues
|
||||
}
|
||||
// if multiple samples have the same timestamp, choose the maximum value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3333;
|
||||
// always prefer a non-decimal.StaleNaN value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196
|
||||
for j > 0 && srcTimestamps[j-1] == tsPrev {
|
||||
j--
|
||||
if decimal.IsStaleNaNInt64(srcValues[j]) {
|
||||
// always prefer decimal.IsStaleNaN to avoid inconsistency when comparing values
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7674
|
||||
continue
|
||||
}
|
||||
if decimal.IsStaleNaNInt64(vPrev) {
|
||||
vPrev = srcValues[j]
|
||||
break
|
||||
continue
|
||||
}
|
||||
if srcValues[j] > vPrev {
|
||||
vPrev = srcValues[j]
|
||||
|
||||
@@ -81,19 +81,17 @@ func TestDeduplicateSamplesWithIdenticalTimestamps(t *testing.T) {
|
||||
f(time.Second, []int64{1001, 1001}, []float64{2, 1}, []int64{1001}, []float64{2})
|
||||
f(time.Second, []int64{1000, 1001, 1001, 1001, 2001}, []float64{1, 2, 5, 3, 0}, []int64{1000, 1001, 2001}, []float64{1, 5, 0})
|
||||
|
||||
// position of decimal.StaleNaN shouldn't matter during deduplication
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7674
|
||||
f(time.Second, []int64{1000, 1000}, []float64{2, decimal.StaleNaN}, []int64{1000}, []float64{decimal.StaleNaN})
|
||||
f(time.Second, []int64{1000, 1000}, []float64{decimal.StaleNaN, 2}, []int64{1000}, []float64{decimal.StaleNaN})
|
||||
f(time.Second, []int64{1000, 1000, 1000}, []float64{1, decimal.StaleNaN, 2}, []int64{1000}, []float64{decimal.StaleNaN})
|
||||
// verify decimal.StaleNaN is NOT preferred on timestamp conflicts
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196
|
||||
f(time.Second, []int64{1000, 1000}, []float64{2, decimal.StaleNaN}, []int64{1000}, []float64{2})
|
||||
f(time.Second, []int64{1000, 1000}, []float64{decimal.StaleNaN, 2}, []int64{1000}, []float64{2})
|
||||
f(time.Second, []int64{1000, 1000, 1000}, []float64{1, decimal.StaleNaN, 2}, []int64{1000}, []float64{2})
|
||||
// compare with Inf values
|
||||
f(time.Second, []int64{1000, 1000}, []float64{math.Inf(1), decimal.StaleNaN}, []int64{1000}, []float64{decimal.StaleNaN})
|
||||
f(time.Second, []int64{1000, 1000}, []float64{decimal.StaleNaN, math.Inf(1)}, []int64{1000}, []float64{decimal.StaleNaN})
|
||||
f(time.Second, []int64{1000, 1000, 1000}, []float64{math.Inf(1), decimal.StaleNaN, math.Inf(-1)}, []int64{1000}, []float64{decimal.StaleNaN})
|
||||
// verify decimal.StaleNaN is preferred only on timestamp conflicts
|
||||
f(time.Second, []int64{1000, 1000, 2000}, []float64{1, decimal.StaleNaN, 2}, []int64{1000, 2000}, []float64{decimal.StaleNaN, 2})
|
||||
f(time.Second, []int64{1000, 1000, 2000, 2000}, []float64{1, decimal.StaleNaN, 2, 3}, []int64{1000, 2000}, []float64{decimal.StaleNaN, 3})
|
||||
f(time.Second, []int64{1000, 1000, 1000, 2000, 2000}, []float64{1, decimal.StaleNaN, 6, 2, 3}, []int64{1000, 2000}, []float64{decimal.StaleNaN, 3})
|
||||
f(time.Second, []int64{1000, 1000}, []float64{math.Inf(1), decimal.StaleNaN}, []int64{1000}, []float64{math.Inf(1)})
|
||||
f(time.Second, []int64{1000, 1000, 1000}, []float64{math.Inf(1), decimal.StaleNaN, math.Inf(-1)}, []int64{1000}, []float64{math.Inf(1)})
|
||||
f(time.Second, []int64{1000, 1000, 2000, 2000}, []float64{1, decimal.StaleNaN, 2, 3}, []int64{1000, 2000}, []float64{1, 3})
|
||||
f(time.Second, []int64{1000, 1000, 2000, 2000}, []float64{decimal.StaleNaN, decimal.StaleNaN, 2, 3}, []int64{1000, 2000}, []float64{decimal.StaleNaN, 3})
|
||||
f(time.Second, []int64{1000, 1000, 1000, 2000, 2000}, []float64{1, decimal.StaleNaN, 6, 2, 3}, []int64{1000, 2000}, []float64{6, 3})
|
||||
}
|
||||
|
||||
func TestDeduplicateSamplesDuringMergeWithIdenticalTimestamps(t *testing.T) {
|
||||
@@ -124,20 +122,18 @@ func TestDeduplicateSamplesDuringMergeWithIdenticalTimestamps(t *testing.T) {
|
||||
f(time.Second, []int64{1001, 1001}, []int64{2, 1}, []int64{1001}, []int64{2})
|
||||
f(time.Second, []int64{1000, 1001, 1001, 1001, 2001}, []int64{1, 2, 5, 3, 0}, []int64{1000, 1001, 2001}, []int64{1, 5, 0})
|
||||
|
||||
// verify decimal.StaleNaN is NOT preferred on timestamp conflicts
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196
|
||||
staleNaN, _ := decimal.FromFloat(decimal.StaleNaN)
|
||||
// position of decimal.StaleNaN shouldn't matter during deduplication
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7674
|
||||
f(time.Second, []int64{1000, 1000}, []int64{2, staleNaN}, []int64{1000}, []int64{staleNaN})
|
||||
f(time.Second, []int64{1000, 1000}, []int64{staleNaN, 2}, []int64{1000}, []int64{staleNaN})
|
||||
f(time.Second, []int64{1000, 1000, 1000}, []int64{1, staleNaN, 2}, []int64{1000}, []int64{staleNaN})
|
||||
f(time.Second, []int64{1000, 1000}, []int64{2, staleNaN}, []int64{1000}, []int64{2})
|
||||
f(time.Second, []int64{1000, 1000}, []int64{staleNaN, 2}, []int64{1000}, []int64{2})
|
||||
f(time.Second, []int64{1000, 1000, 1000}, []int64{1, staleNaN, 2}, []int64{1000}, []int64{2})
|
||||
// compare with max values
|
||||
f(time.Second, []int64{1000, 1000}, []int64{math.MaxInt64, staleNaN}, []int64{1000}, []int64{staleNaN})
|
||||
f(time.Second, []int64{1000, 1000}, []int64{staleNaN, math.MaxInt64}, []int64{1000}, []int64{staleNaN})
|
||||
f(time.Second, []int64{1000, 1000, 1000}, []int64{math.MaxInt64, staleNaN, math.MaxInt64}, []int64{1000}, []int64{staleNaN})
|
||||
// verify decimal.StaleNaN is preferred only on timestamp conflicts
|
||||
f(time.Second, []int64{1000, 1000, 2000}, []int64{1, staleNaN, 2}, []int64{1000, 2000}, []int64{staleNaN, 2})
|
||||
f(time.Second, []int64{1000, 1000, 2000, 2000}, []int64{1, staleNaN, 2, 3}, []int64{1000, 2000}, []int64{staleNaN, 3})
|
||||
f(time.Second, []int64{1000, 1000, 1000, 2000, 2000}, []int64{1, staleNaN, math.MaxInt64, 2, 3}, []int64{1000, 2000}, []int64{staleNaN, 3})
|
||||
f(time.Second, []int64{1000, 1000}, []int64{math.MaxInt64, staleNaN}, []int64{1000}, []int64{math.MaxInt64})
|
||||
f(time.Second, []int64{1000, 1000, 1000}, []int64{math.MaxInt64, staleNaN, math.MaxInt64}, []int64{1000}, []int64{math.MaxInt64})
|
||||
f(time.Second, []int64{1000, 1000, 2000}, []int64{1, staleNaN, 2}, []int64{1000, 2000}, []int64{1, 2})
|
||||
f(time.Second, []int64{1000, 1000, 2000, 2000}, []int64{1, staleNaN, 2, 3}, []int64{1000, 2000}, []int64{1, 3})
|
||||
f(time.Second, []int64{1000, 1000, 1000, 2000, 2000}, []int64{1, staleNaN, math.MaxInt64, 2, 3}, []int64{1000, 2000}, []int64{math.MaxInt64, 3})
|
||||
}
|
||||
|
||||
func TestDeduplicateSamples(t *testing.T) {
|
||||
|
||||
@@ -166,6 +166,8 @@ func getTagFiltersLoopsCacheSize() uint64 {
|
||||
return uint64(float64(memory.Allowed()) / 128)
|
||||
}
|
||||
|
||||
var maxMetricIDsForDirectLabelsLookup int = 100e3
|
||||
|
||||
func mustOpenIndexDB(id uint64, tr TimeRange, name, path string, s *Storage, isReadOnly *atomic.Bool, noRegisterNewSeries bool) *indexDB {
|
||||
if s == nil {
|
||||
logger.Panicf("BUG: Storage must not be nil")
|
||||
@@ -574,11 +576,12 @@ func (is *indexSearch) searchLabelNamesWithFiltersOnTimeRange(qt *querytracer.Tr
|
||||
func (is *indexSearch) searchLabelNamesWithFiltersOnDate(qt *querytracer.Tracer, tfss []*TagFilters, date uint64, maxLabelNames, maxMetrics int) (map[string]struct{}, error) {
|
||||
var filter *uint64set.Set
|
||||
if !isSingleMetricNameFilter(tfss) {
|
||||
filter, err := is.searchMetricIDsWithFiltersOnDate(qt, tfss, date, maxMetrics)
|
||||
var err error
|
||||
filter, err = is.searchMetricIDsWithFiltersOnDate(qt, tfss, date, maxMetrics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if filter != nil && filter.Len() <= 100e3 {
|
||||
if filter != nil && filter.Len() <= maxMetricIDsForDirectLabelsLookup {
|
||||
// It is faster to obtain label names by metricIDs from the filter
|
||||
// instead of scanning the inverted index for the matching filters.
|
||||
// This should help https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2978
|
||||
@@ -844,11 +847,12 @@ func (is *indexSearch) searchLabelValuesOnDate(qt *querytracer.Tracer, labelName
|
||||
useCompositeScan := labelName != "" && isSingleMetricNameFilter(tfss)
|
||||
var filter *uint64set.Set
|
||||
if !useCompositeScan {
|
||||
filter, err := is.searchMetricIDsWithFiltersOnDate(qt, tfss, date, maxMetrics)
|
||||
var err error
|
||||
filter, err = is.searchMetricIDsWithFiltersOnDate(qt, tfss, date, maxMetrics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if filter != nil && filter.Len() <= 100e3 {
|
||||
if filter != nil && filter.Len() <= maxMetricIDsForDirectLabelsLookup {
|
||||
// It is faster to obtain label values by metricIDs from the filter
|
||||
// instead of scanning the inverted index for the matching filters.
|
||||
// This should help https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2978
|
||||
|
||||
@@ -1423,6 +1423,9 @@ func TestMatchTagFilters(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSearchTSIDWithTimeRange(t *testing.T) {
|
||||
// TODO: @f41gh7 refactor this test:
|
||||
// create a new test for LabelNames
|
||||
// move exist LabelVales tests into TestSearchLabelValues
|
||||
const path = "TestSearchTSIDWithTimeRange"
|
||||
// Create a bunch of per-day time series
|
||||
const days = 5
|
||||
@@ -2008,3 +2011,139 @@ func TestIndexSearchLegacyContainsTimeRange_Concurrent(t *testing.T) {
|
||||
t.Fatalf("unexpected min timestamp: got %v, want %v", time.UnixMilli(got).UTC(), time.UnixMilli(want).UTC())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchLabelValues(t *testing.T) {
|
||||
const path = "TestSearchLabelValues"
|
||||
// Create a bunch of per-day time series
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
|
||||
baseDate := uint64(timestamp) / msecPerDay
|
||||
var metricNameBuf []byte
|
||||
perDayMetricIDs := make(map[uint64]*uint64set.Set)
|
||||
var allMetricIDs uint64set.Set
|
||||
uniqLabelNames := make(map[string]struct{})
|
||||
|
||||
newMN := func(name string, day, metric int) MetricName {
|
||||
var mn MetricName
|
||||
metricName := fmt.Sprintf("%s_%d", name, metric)
|
||||
if _, ok := uniqLabelNames[metricName]; !ok {
|
||||
uniqLabelNames[metricName] = struct{}{}
|
||||
}
|
||||
mn.MetricGroup = []byte(metricName)
|
||||
mn.AddTag(
|
||||
"constant",
|
||||
"const",
|
||||
)
|
||||
mn.AddTag(
|
||||
"day",
|
||||
fmt.Sprintf("%v", day),
|
||||
)
|
||||
mn.AddTag(
|
||||
"UniqueId",
|
||||
fmt.Sprintf("%v", metric),
|
||||
)
|
||||
mn.AddTag(
|
||||
"some_unique_id",
|
||||
fmt.Sprintf("%v", day),
|
||||
)
|
||||
mn.sortTags()
|
||||
return mn
|
||||
}
|
||||
|
||||
s := MustOpenStorage(path, OpenOptions{})
|
||||
ptw := s.tb.MustGetPartition(timestamp)
|
||||
db := ptw.pt.idb
|
||||
is := db.getIndexSearch(noDeadline)
|
||||
|
||||
for day := range days {
|
||||
date := baseDate - uint64(day)
|
||||
var metricIDs uint64set.Set
|
||||
for metric := range metricsPerDay {
|
||||
mn := newMN("testMetric", day, metric)
|
||||
metricNameBuf = mn.Marshal(metricNameBuf[:0])
|
||||
var tsid TSID
|
||||
if !is.getTSIDByMetricName(&tsid, metricNameBuf, date) {
|
||||
generateTSID(&tsid, &mn)
|
||||
createAllIndexesForMetricName(db, &mn, &tsid, date)
|
||||
}
|
||||
metricIDs.Add(tsid.MetricID)
|
||||
}
|
||||
|
||||
allMetricIDs.Union(&metricIDs)
|
||||
perDayMetricIDs[date] = &metricIDs
|
||||
}
|
||||
db.putIndexSearch(is)
|
||||
|
||||
labelValues := sortedSlice(uniqLabelNames)
|
||||
|
||||
// Flush index to disk, so it becomes visible for search
|
||||
db.tb.DebugFlush()
|
||||
|
||||
is2 := db.getIndexSearch(noDeadline)
|
||||
|
||||
// Check that all the metrics are found for all the days.
|
||||
for date := baseDate - days + 1; date <= baseDate; date++ {
|
||||
metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if !perDayMetricIDs[date].Equal(metricIDs) {
|
||||
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
|
||||
}
|
||||
}
|
||||
|
||||
// Check that all the metrics are found in global index
|
||||
metricIDs, err := is2.getMetricIDsForDate(0, metricsPerDay*days)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if !allMetricIDs.Equal(metricIDs) {
|
||||
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), allMetricIDs.AppendTo(nil))
|
||||
}
|
||||
db.putIndexSearch(is2)
|
||||
|
||||
// Check SearchLabelNames with the specified time range.
|
||||
tr := TimeRange{
|
||||
MinTimestamp: timestamp - msecPerDay,
|
||||
MaxTimestamp: timestamp,
|
||||
}
|
||||
|
||||
// Check SearchLabelValues with the specified time range.
|
||||
lvs, err := db.SearchLabelValues(nil, "", nil, tr, 10000, 1e9, noDeadline)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error in SearchLabelValues(timeRange=%s): %s", &tr, err)
|
||||
}
|
||||
got := sortedSlice(lvs)
|
||||
if !reflect.DeepEqual(got, labelValues) {
|
||||
t.Fatalf("unexpected labelValues; got\n%s\nwant\n%s", got, labelValues)
|
||||
}
|
||||
|
||||
tfsMetricNameRe := NewTagFilters()
|
||||
if err := tfsMetricNameRe.Add([]byte("constant"), []byte("const"), false, false); err != nil {
|
||||
t.Fatalf("cannot add filter on label: %s", err)
|
||||
}
|
||||
if err := tfsMetricNameRe.Add(nil, []byte("testMetric_99.*"), false, true); err != nil {
|
||||
t.Fatalf("cannot add filter on metric name: %s", err)
|
||||
}
|
||||
// Check SearchLabelValues with the specified time range and tfs matches correct results
|
||||
// if filter result exceeds quick search limit
|
||||
originValue := maxMetricIDsForDirectLabelsLookup
|
||||
maxMetricIDsForDirectLabelsLookup = 10
|
||||
defer func() {
|
||||
maxMetricIDsForDirectLabelsLookup = originValue
|
||||
}()
|
||||
lvs, err = db.SearchLabelValues(nil, "__name__", []*TagFilters{tfsMetricNameRe}, tr, 10000, 1e9, noDeadline)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error in SearchLabelValues(timeRange=%s): %s", &tr, err)
|
||||
}
|
||||
got = sortedSlice(lvs)
|
||||
labelValuesReMatch := []string{"testMetric_99", "testMetric_990", "testMetric_991", "testMetric_992", "testMetric_993", "testMetric_994", "testMetric_995", "testMetric_996", "testMetric_997", "testMetric_998", "testMetric_999"}
|
||||
if !reflect.DeepEqual(got, labelValuesReMatch) {
|
||||
t.Fatalf("unexpected labelValues; got\n%s\nwant\n%s", got, labelValuesReMatch)
|
||||
}
|
||||
|
||||
s.tb.PutPartition(ptw)
|
||||
s.MustClose()
|
||||
fs.MustRemoveDir(path)
|
||||
}
|
||||
|
||||
@@ -441,6 +441,7 @@ func TestStorageAddRows_nextDayIndexPrefill(t *testing.T) {
|
||||
MaxTimestamp: time.Now().Add(+15 * time.Minute).UnixMilli(),
|
||||
})
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{})
|
||||
defer s.MustClose()
|
||||
s.AddRows(mrs0, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
if got, want := countMetricIDs(t, s, "metric0", today), numSeries; got != want {
|
||||
@@ -462,12 +463,6 @@ func TestStorageAddRows_nextDayIndexPrefill(t *testing.T) {
|
||||
t.Fatalf("unexpected metric id count for next day: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
// Close the storage and reopen it 15m later instead of keeping it open
|
||||
// and waiting. This is to make the test faster. Storage has a lot of
|
||||
// background tasks that are activated every 1-10 seconds and synctest's
|
||||
// time.Sleep() will wake them up many times. Closing storage before
|
||||
// sleeping seems to eliminate this.
|
||||
//
|
||||
// At 23:15 the prefill must work.
|
||||
//
|
||||
// However, the mrs1 timestamps are not within the current hour and
|
||||
@@ -476,9 +471,7 @@ func TestStorageAddRows_nextDayIndexPrefill(t *testing.T) {
|
||||
//
|
||||
// The mrs2 timestamps are within the current hour so some next day index
|
||||
// entries will be created.
|
||||
s.MustClose()
|
||||
time.Sleep(15 * time.Minute) // 2000-01-01T23:15:00Z
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{})
|
||||
mrs1 := testGenerateMetricRowsWithPrefix(rng, numSeries, "metric1", TimeRange{
|
||||
MinTimestamp: time.Now().Add(-30 * time.Minute).UnixMilli(),
|
||||
MaxTimestamp: time.Now().Add(-15 * time.Minute).UnixMilli(),
|
||||
@@ -504,13 +497,7 @@ func TestStorageAddRows_nextDayIndexPrefill(t *testing.T) {
|
||||
t.Fatalf("unexpected metric id count for next day: got 0, want > 0")
|
||||
}
|
||||
|
||||
// Close the storage and reopen it at 23:30.
|
||||
//
|
||||
// Since we are now closer to midnight than we were at 23:15, more next
|
||||
// day entries must be created.
|
||||
s.MustClose()
|
||||
time.Sleep(15 * time.Minute) // 2000-01-01T23:30:00Z
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{})
|
||||
mrs3 := testGenerateMetricRowsWithPrefix(rng, numSeries, "metric3", TimeRange{
|
||||
MinTimestamp: time.Now().Add(-15 * time.Minute).UnixMilli(),
|
||||
MaxTimestamp: time.Now().UnixMilli(),
|
||||
@@ -525,13 +512,7 @@ func TestStorageAddRows_nextDayIndexPrefill(t *testing.T) {
|
||||
t.Fatalf("unexpected metric id count for next day: got %d, want > %d", got30min, got15min)
|
||||
}
|
||||
|
||||
// Close the storage and reopen it at 23:45.
|
||||
//
|
||||
// Since we are now closer to midnight than we were at 23:30, more next
|
||||
// day entries must be created.
|
||||
s.MustClose()
|
||||
time.Sleep(15 * time.Minute) // 2000-01-01T23:45:00Z
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{})
|
||||
mrs4 := testGenerateMetricRowsWithPrefix(rng, numSeries, "metric4", TimeRange{
|
||||
MinTimestamp: time.Now().Add(-15 * time.Minute).UnixMilli(),
|
||||
MaxTimestamp: time.Now().UnixMilli(),
|
||||
@@ -543,7 +524,33 @@ func TestStorageAddRows_nextDayIndexPrefill(t *testing.T) {
|
||||
t.Fatalf("unexpected metric id count for next day: got %d, want > %d", got45min, got30min)
|
||||
}
|
||||
|
||||
s.MustClose()
|
||||
// Sleep until the next day
|
||||
// do not close storage, it resets dataMetricID cache and it will result into slow inserts
|
||||
// since dateMetricID cache is not persisted on-disk
|
||||
|
||||
time.Sleep(35 * time.Minute) // 2000-01-02T00:20:00Z
|
||||
synctest.Wait()
|
||||
|
||||
// Ingest data for the next day, it must hit dateMetricID cache and
|
||||
// do not result into significant amount of slow inserts.
|
||||
var m Metrics
|
||||
s.UpdateMetrics(&m)
|
||||
currDaySlowInserts := m.SlowPerDayIndexInserts
|
||||
mrs3NextDay := testGenerateMetricRowsWithPrefix(rng, numSeries, "metric3", TimeRange{
|
||||
MinTimestamp: time.Now().Add(-5 * time.Minute).UnixMilli(),
|
||||
MaxTimestamp: time.Now().UnixMilli(),
|
||||
})
|
||||
|
||||
s.AddRows(mrs3NextDay, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
m.Reset()
|
||||
s.UpdateMetrics(&m)
|
||||
nextDaySlowInserts := m.SlowPerDayIndexInserts
|
||||
slowInserts := nextDaySlowInserts - currDaySlowInserts
|
||||
if slowInserts >= numSeries {
|
||||
t.Errorf("unexpected amount of slow inserts: got %d, want < %d", slowInserts, numSeries)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -201,29 +201,28 @@ func (das *dedupAggrShard) pushSamples(samples []pushSample, isGreen bool) {
|
||||
state.sizeBytes.Add(uint64(len(key)) + uint64(unsafe.Sizeof(key)+unsafe.Sizeof(s)+unsafe.Sizeof(*s)))
|
||||
continue
|
||||
}
|
||||
if !isDuplicate(s, sample) {
|
||||
s.value = sample.value
|
||||
s.timestamp = sample.timestamp
|
||||
}
|
||||
s.timestamp, s.value = deduplicateSamples(s.timestamp, sample.timestamp, s.value, sample.value)
|
||||
}
|
||||
state.samplesBuf = samplesBuf
|
||||
}
|
||||
|
||||
// isDuplicate returns true if b is duplicate of a
|
||||
// deduplicateSamples returns deduplicated timestamp and value results.
|
||||
// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication
|
||||
func isDuplicate(a *dedupAggrSample, b pushSample) bool {
|
||||
if b.timestamp > a.timestamp {
|
||||
return false
|
||||
func deduplicateSamples(oldT, newT int64, oldV, newV float64) (int64, float64) {
|
||||
if newT > oldT {
|
||||
return newT, newV
|
||||
}
|
||||
if b.timestamp == a.timestamp {
|
||||
if decimal.IsStaleNaN(b.value) {
|
||||
return false
|
||||
// if both samples have the same timestamp, choose the maximum value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3333;
|
||||
// always prefer a non-decimal.StaleNaN value, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196
|
||||
if newT == oldT {
|
||||
if decimal.IsStaleNaN(oldV) {
|
||||
return newT, newV
|
||||
}
|
||||
if b.value > a.value {
|
||||
return false
|
||||
if newV > oldV {
|
||||
return newT, newV
|
||||
}
|
||||
}
|
||||
return true
|
||||
return oldT, oldV
|
||||
}
|
||||
|
||||
func (das *dedupAggrShard) flush(ctx *dedupFlushCtx, f aggrPushFunc) {
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
||||
)
|
||||
|
||||
func TestDedupAggrSerial(t *testing.T) {
|
||||
@@ -77,3 +79,22 @@ func TestDedupAggrConcurrent(_ *testing.T) {
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestDeduplicateSamples(t *testing.T) {
|
||||
f := func(oldT, newT int64, oldV, newV float64, expectedT int64, expectedV float64) {
|
||||
t.Helper()
|
||||
dedupT, dedupV := deduplicateSamples(oldT, newT, oldV, newV)
|
||||
if dedupT != expectedT || dedupV != expectedV {
|
||||
t.Fatalf("unexpected deduplicated result for oldT=%d, newT=%d, oldV=%f, newV=%f; got dedupT=%d, dedupV=%f; want dedupT=%d, dedupV=%f",
|
||||
oldT, newT, oldV, newV, dedupT, dedupV, expectedT, expectedV)
|
||||
}
|
||||
}
|
||||
|
||||
f(1000, 2000, 1.0, 2.0, 2000, 2.0)
|
||||
f(2000, 1000, 2.0, 1.0, 2000, 2.0)
|
||||
f(1000, 1000, 1.0, 2.0, 1000, 2.0)
|
||||
f(1000, 1000, 2.0, 1.0, 1000, 2.0)
|
||||
f(1000, 1000, 1.0, 1.0, 1000, 1.0)
|
||||
f(1000, 1000, 1.0, float64(decimal.StaleNaN), 1000, 1.0)
|
||||
f(1000, 1000, float64(decimal.StaleNaN), 2.0, 1000, 2.0)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user