Compare commits

..

6 Commits

Author SHA1 Message Date
Artem Fetishev
d82ad68f60 docs/changelog: cut release v1.144.0
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-05-22 17:05:50 +02:00
Artem Fetishev
dcbd8ef721 app/vmselect: run make vmui-update
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-05-22 17:01:56 +02:00
JAYICE
886c7762eb app/vmui: add timestamp and value column for csv data exported from table view (#10979)
Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975

before:
<img width="342" height="67" alt="image"
src="https://github.com/user-attachments/assets/a972857b-6a6c-4ec3-86fb-371b72a4d6d9"
/>

after:
<img width="426" height="115" alt="image"
src="https://github.com/user-attachments/assets/ddb540ed-5c1b-4431-a5c5-3576292dd21e"
/>

---------

Signed-off-by: JAYICE <1185430411@qq.com>
Co-authored-by: Max Kotliar <mkotlyar@victoriametrics.com>
2026-05-22 15:35:19 +03:00
Roman Khavronenko
d3006b25e6 app/vmui: display null/NaN/staleNaN values on RawQuery chart (#10986)
`null` values can be actual `NaN` or `null` values exposed by the
exporter, or stale markers

https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers

 Before, vmui Raw Query was silently dropping non-numeric values.
 Displaying such values on chart could improve debugging experience.

Screenshots: 
<img width="1487" height="833" alt="image"
src="https://github.com/user-attachments/assets/2c80cd52-9d37-41f7-ad73-a48335b8aef2"
/>

Decisions:
1. Since null value can't be mapped on Y axis - it is placed on the very
bottom as `X` sign. It is ok to have it placed on top or as a dotted
line
2. It has a tooltip explaining what it is
3. Its value is not accounted in cumulative min/max/avg tolltip of the
whole time series on the graph
4. Null value can't exactly state it is a stale marker - such a value
can come from many places

Values to import as example dataset:
```
{"metric":{"__name__":"cpu_usage","job":"test","instance":"127.0.0.1:8080","cpu":"1","exported_instance":"foo","exported_job":"foo"},"values":[null,0.2,0.2,null,0.2,null,0.2,0.2,null,0.2,null,0.2,null,0.2,null,0.2,null,0.2,null,0.2,null,0.2,null,0.2,0.2,0.2,0.2,null,0.2,0.2,null,0.2,0.2,null,0.2,0.2,null,0.2,null,0.2,null,0.2,null,0.2,null,0.2,null,0.2,0.2],"timestamps":[1779362323611,1779362323617,1779362333617,1779362343610,1779362343617,1779362353610,1779362353617,1779362363617,1779362373610,1779362373617,1779362383610,1779362383617,1779362393610,1779362393617,1779362403610,1779362403617,1779362413610,1779362413617,1779362423610,1779362423617,1779362433610,1779362433617,1779362443610,1779362443617,1779362453617,1779362463617,1779362473617,1779362483610,1779362483617,1779362493617,1779362503610,1779362503617,1779362513617,1779362523610,1779362523617,1779362533617,1779362543609,1779362543617,1779362553609,1779362553617,1779362563609,1779362563617,1779362573609,1779362573617,1779362583609,1779362583617,1779362593609,1779362593617,1779362603617]}
```
------------

Feature like that would significantly help in debugging issues like
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10893

@Loori-R I am sorry, but this PR is rather a PoC - it was mostly
generated with AI help.

---------

Signed-off-by: hagen1778 <roman@victoriametrics.com>
Co-authored-by: Max Kotliar <mkotlyar@victoriametrics.com>
2026-05-22 15:29:31 +03:00
andriibeee
c41e967ee1 vmstorage: vmselect RPC server must respect -futureRetention (#10881)
Flag `-denyQueriesOutsideRetention` now also rejects queries past future
retention limit. Default behavior is unchanged.

Note that HTTP error code has been changed from 503 to 400.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10879.

---------

Signed-off-by: Andrii Beskomornyi <andriibeee@gmail.com>
Co-authored-by: Artem Fetishev <rtm@victoriametrics.com>
2026-05-22 13:00:21 +02:00
andriibeee
0c9a011e0a lib/timeutil: handle int64 overflow in relative duration parsing (#10883)
Fix `int64` overflow in `ParseTimeAt()` in `lib/timeutil/time.go` when
converting time expressed as string into nanoseconds.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10880.

---------

Signed-off-by: Andrii Beskomornyi <andriibeee@gmail.com>
Co-authored-by: Artem Fetishev <rtm@victoriametrics.com>
2026-05-22 12:17:04 +02:00
29 changed files with 998 additions and 417 deletions

View File

@@ -20,6 +20,9 @@ func TestGetTime_Failure(t *testing.T) {
// negative time
f("-292273086-05-16T16:47:06Z")
// relative duration that resolves to a timestamp before 1970
f("-9223372036.855")
}
func TestGetTime_Success(t *testing.T) {
@@ -77,9 +80,6 @@ func TestGetTime_Success(t *testing.T) {
// float timestamp representation",
f("1562529662.324", time.Date(2019, 7, 7, 20, 01, 02, 324e6, time.UTC))
// negative timestamp
f("-9223372036.855", time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC))
// big timestamp
f("1223372036855", time.Date(2008, 10, 7, 9, 33, 56, 855e6, time.UTC))

View File

@@ -990,9 +990,6 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
return fmt.Errorf("timeout exceeded before starting data export: %s", deadline.String())
}
tr := sq.GetTimeRange()
if err := vmstorage.CheckTimeRange(tr); err != nil {
return err
}
tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
if err != nil {
return err
@@ -1098,9 +1095,6 @@ func SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline
// Setup search.
tr := sq.GetTimeRange()
if err := vmstorage.CheckTimeRange(tr); err != nil {
return nil, err
}
tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
if err != nil {
return nil, err
@@ -1127,9 +1121,6 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
// Setup search.
tr := sq.GetTimeRange()
if err := vmstorage.CheckTimeRange(tr); err != nil {
return nil, err
}
tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
if err != nil {
return nil, err

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -37,11 +37,11 @@
<meta property="og:title" content="UI for VictoriaMetrics">
<meta property="og:url" content="https://victoriametrics.com/">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<script type="module" crossorigin src="./assets/index-C7gvW_Zn.js"></script>
<script type="module" crossorigin src="./assets/index-BjJ7fDL7.js"></script>
<link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
<link rel="modulepreload" crossorigin href="./assets/vendor-C8Kwp93_.js">
<link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
<link rel="stylesheet" crossorigin href="./assets/index-D2OEy8Ra.css">
<link rel="stylesheet" crossorigin href="./assets/index-BL7jEFBa.css">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>

View File

@@ -56,8 +56,8 @@ var (
logNewSeries = flag.Bool("logNewSeries", false, "Whether to log new series. This option is for debug purposes only. It can lead to performance issues "+
"when big number of new series are ingested into VictoriaMetrics")
denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod. "+
"When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. "+
denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod and -futureRetention. "+
"When set, then /api/v1/query_range will return an error for queries with 'from' value outside -retentionPeriod or 'to' value beyond -futureRetention. "+
"This may be useful when multiple data sources with distinct retentions are hidden behind query-tee")
maxHourlySeries = flag.Int64("storage.maxHourlySeries", 0, "The maximum number of unique series can be added to the storage during the last hour. "+
"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . "+
@@ -103,21 +103,6 @@ var (
"If set to 0 or a negative value, defaults to 1% of allowed memory.")
)
// CheckTimeRange returns true if the given tr is denied for querying.
func CheckTimeRange(tr storage.TimeRange) error {
if !*denyQueriesOutsideRetention {
return nil
}
minAllowedTimestamp := int64(fasttime.UnixTimestamp()*1000) - retentionPeriod.Milliseconds()
if tr.MinTimestamp > minAllowedTimestamp {
return nil
}
return &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf("the given time range %s is outside the allowed -retentionPeriod=%s according to -denyQueriesOutsideRetention", &tr, retentionPeriod),
StatusCode: http.StatusServiceUnavailable,
}
}
// Init initializes vmstorage.
func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
@@ -151,14 +136,15 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
startTime := time.Now()
WG = syncwg.WaitGroup{}
opts := storage.OpenOptions{
Retention: retentionPeriod.Duration(),
FutureRetention: futureRetention.Duration(),
MaxHourlySeries: getMaxHourlySeries(),
MaxDailySeries: getMaxDailySeries(),
DisablePerDayIndex: *disablePerDayIndex,
TrackMetricNamesStats: *trackMetricNamesStats,
IDBPrefillStart: *idbPrefillStart,
LogNewSeries: *logNewSeries,
Retention: retentionPeriod.Duration(),
FutureRetention: futureRetention.Duration(),
DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
MaxHourlySeries: getMaxHourlySeries(),
MaxDailySeries: getMaxDailySeries(),
DisablePerDayIndex: *disablePerDayIndex,
TrackMetricNamesStats: *trackMetricNamesStats,
IDBPrefillStart: *idbPrefillStart,
LogNewSeries: *logNewSeries,
}
strg := storage.MustOpenStorage(*DataPath, opts)
Storage = strg

View File

@@ -3,6 +3,7 @@ export interface MetricBase {
metric: {
[key: string]: string;
};
nullTimestamps?: number[];
}
export interface MetricResult extends MetricBase {

View File

@@ -16,6 +16,7 @@ export interface ChartTooltipProps {
point: { top: number, left: number };
unit?: string;
statsFormatted?: SeriesItemStatsFormatted;
description?: ReactNode;
isSticky?: boolean;
info?: ReactNode;
marker?: string;
@@ -34,6 +35,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
unit = "",
info,
statsFormatted,
description,
isSticky,
marker,
duplicateCount = 0,
@@ -173,6 +175,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
))}
</table>
)}
{description && <p className="vm-chart-tooltip__description">{description}</p>}
{info && <p className="vm-chart-tooltip__info">{info}</p>}
</div>
), u.root);

View File

@@ -143,4 +143,10 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
word-break: break-all;
white-space: pre-wrap;
}
&__description {
word-break: break-word;
white-space: pre-wrap;
opacity: 0.85;
}
}

View File

@@ -15,19 +15,65 @@ interface LineTooltipHook {
unit?: string;
}
// Pixel proximity for detecting hover over null-timestamp X markers drawn at chart bottom.
const NULL_HOVER_PROX = 8;
// Half the visual marker height in CSS px (BASE_POINT_SIZE * 1.4 / 2 from scatter.ts).
// scatter.ts lifts the marker center by this amount above yMin so the icon sits inside
// the plot area; the hover y-anchor must match that offset.
const NULL_MARKER_HALF_CSS = 2.8;
interface NullHover {
seriesIdx: number;
timestamp: number;
}
const findNullHover = (u: uPlot): NullHover | null => {
const cursorLeft = u.cursor.left ?? -1;
const cursorTop = u.cursor.top ?? -1;
if (cursorLeft < 0 || cursorTop < 0) return null;
const scaleY = u.scales["1"];
if (!scaleY || scaleY.min == null) return null;
const yPos = u.valToPos(scaleY.min, "1") - NULL_MARKER_HALF_CSS;
if (Math.abs(cursorTop - yPos) > NULL_HOVER_PROX) return null;
let best: { seriesIdx: number; timestamp: number; dist: number } | null = null;
for (let s = 1; s < u.series.length; s++) {
const seriesItem = u.series[s] as SeriesItem;
if (!seriesItem.show) continue;
const nullTs = seriesItem.nullTimestamps;
if (!nullTs || !nullTs.length) continue;
for (let i = 0; i < nullTs.length; i++) {
const t = nullTs[i];
const xPos = u.valToPos(t, "x");
const dist = Math.abs(cursorLeft - xPos);
if (dist < NULL_HOVER_PROX && (best === null || dist < best.dist)) {
best = { seriesIdx: s, timestamp: t, dist };
}
}
}
return best ? { seriesIdx: best.seriesIdx, timestamp: best.timestamp } : null;
};
const NULL_DESCRIPTION = "\"null\" can be a staleness marker or an actual NaN/null value produced by exporter.";
const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
const [showTooltip, setShowTooltip] = useState(false);
const [tooltipIdx, setTooltipIdx] = useState({ seriesIdx: -1, dataIdx: -1 });
const [nullTooltip, setNullTooltip] = useState<NullHover | null>(null);
const [stickyTooltips, setStickyToolTips] = useState<ChartTooltipProps[]>([]);
const resetTooltips = () => {
setStickyToolTips([]);
setTooltipIdx({ seriesIdx: -1, dataIdx: -1 });
setNullTooltip(null);
};
const setCursor = (u: uPlot) => {
const dataIdx = u.cursor.idx ?? -1;
setTooltipIdx(prev => ({ ...prev, dataIdx }));
setNullTooltip(findNullHover(u));
};
const seriesFocus = (u: uPlot, sidx: (number | null)) => {
@@ -35,7 +81,36 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
setTooltipIdx(prev => ({ ...prev, seriesIdx }));
};
const getNullTooltipProps = (hit: NullHover): ChartTooltipProps => {
const { seriesIdx, timestamp } = hit;
const metricItem = metrics[seriesIdx - 1];
const seriesItem = series[seriesIdx] as SeriesItem;
const groups = new Set(metrics.map(m => m.group));
const group = metricItem?.group || 0;
const yMin = u?.scales?.[1]?.min ?? 0;
const point = {
top: u ? u.valToPos(yMin, seriesItem?.scale || "1") - NULL_MARKER_HALF_CSS : 0,
left: u ? u.valToPos(timestamp, "x") : 0,
};
return {
u,
id: `null_${seriesIdx}_${timestamp}`,
title: groups.size > 1 ? `Query ${group}` : "",
dates: [dayjs(timestamp * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT)],
value: "null",
info: getMetricName(metricItem, seriesItem),
description: NULL_DESCRIPTION,
marker: `${seriesItem?.stroke}`,
point,
};
};
const getTooltipProps = useCallback((): ChartTooltipProps => {
if (nullTooltip) return getNullTooltipProps(nullTooltip);
const { seriesIdx, dataIdx } = tooltipIdx;
const metricItem = metrics[seriesIdx - 1];
const seriesItem = series[seriesIdx] as SeriesItem;
@@ -84,7 +159,7 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
marker: `${seriesItem?.stroke}`,
duplicateCount,
};
}, [u, tooltipIdx, metrics, series, unit]);
}, [u, tooltipIdx, metrics, series, unit, nullTooltip]);
const handleClick = useCallback(() => {
if (!showTooltip) return;
@@ -99,8 +174,9 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
};
useEffect(() => {
setShowTooltip(tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1);
}, [tooltipIdx]);
const normalHit = tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1;
setShowTooltip(normalHit || nullTooltip !== null);
}, [tooltipIdx, nullTooltip]);
useEventListener("click", handleClick);

View File

@@ -20,7 +20,7 @@ describe("convertMetricsDataToCSV", () => {
},
];
const result = convertMetricsDataToCSV(data);
expect(result).toBe("header1,header2\n123,value2");
expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123");
});
it("should return a valid CSV string for multiple metric entries with values", () => {
@@ -43,7 +43,7 @@ describe("convertMetricsDataToCSV", () => {
},
];
const result = convertMetricsDataToCSV(data);
expect(result).toBe("header1,header2\n123,value2\n456,value4");
expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123\n456,value4,1623949200,456");
});
it("should handle metric entries with multiple values field", () => {
@@ -58,7 +58,7 @@ describe("convertMetricsDataToCSV", () => {
},
];
const result = convertMetricsDataToCSV(data);
expect(result).toBe("header1,header2\n123-456,values");
expect(result).toBe("header1,header2,__timestamp__,__value__\n123-456,values,-,-");
});
it("should handle a combination of metric entries with value and values", () => {
@@ -81,6 +81,19 @@ describe("convertMetricsDataToCSV", () => {
},
];
const result = convertMetricsDataToCSV(data);
expect(result).toBe("header1,header2\n123,first\n456-789,second");
expect(result).toBe("header1,header2,__timestamp__,__value__\n123,first,1623945600,123\n456-789,second,-,-");
});
it("should return value and timestamp if metric field is empty", () => {
const data: InstantMetricResult[] = [
{
value: [1623945600, "123"],
group: 0,
metric: {}
},
];
const result = convertMetricsDataToCSV(data);
expect(result).toBe("__timestamp__,__value__\n1623945600,123");
});
});

View File

@@ -3,16 +3,22 @@ import { getColumns, MetricCategory } from "../../hooks/useSortedCategories";
import { formatValueToCSV } from "../../utils/csv";
const getHeaders = (data: InstantMetricResult[]): string => {
return getColumns(data).map(({ key }) => key).join(",");
const metricHeaders = getColumns(data).map(({ key }) => key);
return [...metricHeaders, "__timestamp__", "__value__"].join(",");
};
const getRows = (data: InstantMetricResult[], headers: MetricCategory[]) => {
return data?.map(d => headers.map(c => formatValueToCSV(d.metric[c.key] || "-")).join(","));
return data?.map(d => {
const metricPart = headers.map(c => formatValueToCSV(d.metric[c.key] || "-"));
const timestamp = d.value ? formatValueToCSV(String(d.value[0])) : "-";
const value = d.value ? formatValueToCSV(d.value[1]) : "-";
return [...metricPart, timestamp, value].join(",");
});
};
export const convertMetricsDataToCSV = (data: InstantMetricResult[]): string => {
if (!data.length) return "";
const headers = getHeaders(data);
if (!headers.length) return "";
const rows = getRows(data, getColumns(data));
return [headers, ...rows].join("\n");
};

View File

@@ -149,15 +149,21 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
const pointsToTake = shouldDownsample ? maxPointsPerSeries : totalPoints;
const step = shouldDownsample ? totalPoints / maxPointsPerSeries : 1;
const values: [number, number][] = Array.from({ length: pointsToTake }, (_, i) => {
const values: [number, number][] = new Array(pointsToTake);
const nullTimestamps: number[] = [];
for (let i = 0; i < pointsToTake; i++) {
const idx = shouldDownsample ? Math.floor(i * step) : i;
return [rawTimestamps[idx] / 1000, rawValues[idx]];
});
const ts = rawTimestamps[idx] / 1000;
const raw = rawValues[idx];
if (raw === null) nullTimestamps.push(ts);
values[i] = [ts, raw as number];
}
tempData.push({
group: counter,
metric: jsonLine.metric,
values,
nullTimestamps,
} as MetricBase);
}

View File

@@ -11,6 +11,7 @@ export interface SeriesItem extends Series {
statsFormatted: SeriesItemStatsFormatted;
median: number;
hasAlias?: boolean;
nullTimestamps?: number[];
}
export interface HideSeriesArgs {

View File

@@ -103,6 +103,28 @@ export const drawPoints = (u: uPlot, seriesIdx: number) => {
u.ctx.lineWidth = 1.4 * uPlot.pxRatio;
u.ctx.strokeStyle = u.ctx.fillStyle;
u.ctx.stroke(squaresPath);
const nullTs = (series as unknown as { nullTimestamps?: number[] }).nullTimestamps;
if (nullTs && nullTs.length) {
const xSize = BASE_POINT_SIZE * 1.4 * uPlot.pxRatio;
const xHalf = xSize / 2;
// Lift the marker by half its size so the entire icon sits inside the plot area
// (yMin maps to the plot's bottom edge, so centering on it would clip the lower half).
const cy = valToPosY(yMin, scaleY, yDim, yOff) - xHalf;
const xPath = new Path2D();
for (let i = 0; i < nullTs.length; i++) {
const t = nullTs[i];
if (t < xMin || t > xMax) continue;
const cx = valToPosX(t, scaleX, xDim, xOff);
xPath.moveTo(cx - xHalf, cy - xHalf);
xPath.lineTo(cx + xHalf, cy + xHalf);
xPath.moveTo(cx + xHalf, cy - xHalf);
xPath.lineTo(cx - xHalf, cy + xHalf);
}
u.ctx.lineWidth = 1.6 * uPlot.pxRatio;
u.ctx.strokeStyle = u.ctx.fillStyle;
u.ctx.stroke(xPath);
}
};
uPlot.orient(u, seriesIdx, orientCallback);

View File

@@ -38,6 +38,7 @@ export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[],
show: !includesHideSeries(label, hideSeries),
scale: "1",
paths: isRawQuery ? drawPoints : undefined,
nullTimestamps: d.nullTimestamps,
...getSeriesStatistics(d),
};
};

View File

@@ -26,6 +26,10 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
## [v1.144.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.144.0)
Released at 2026-05-22
* FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `clusternative.tls` `vminsert` configuration flags for [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). See [#10958](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10958).
@@ -34,6 +38,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Improve [slowness-based rerouting](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) to prevent rerouting storms under high cluster load. Previously, rerouting could cascade across storage nodes when the whole cluster was saturated, making the situation worse. Now rerouting only activates when the cluster p90 saturation is below 60%, and the slowest node is more than 20% slower than p90. See [#10876](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10876).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `{{.MetricsAccountID}}` and `{{.MetricsProjectID}}` [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating) for use in `headers` and `url_prefix` config fields. Previously, only the combined `{{.MetricsTenant}}` (`accountID:projectID`) JWT placeholder was supported, making it impossible to configure [multitenancy via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers). See [#10927](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10927). Thanks to @Vinayak9769 for the contribution.
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): display `null` values on `Raw Query` chart. `null` values can be actual `NaN` or `null` values exposed by the exporter, or [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). Before, vmui Raw Query was silently dropping non-numeric values. Displaying such values on the chart could improve the debugging experience. See [#10986](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10986).
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. See [#10918](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918). Thanks to @alexei38 for the contribution.
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
@@ -47,6 +52,9 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): hide values passed to `vmalert.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve exact series values in graph tooltips instead of rounding them by significant digits. See [#10952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10952).
* BUGFIX: all VictoriaMetrics components: fix int64 overflow when parsing [timestamp parameters](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats) with relative durations. See [#10880](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10880).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `-denyQueriesOutsideRetention` now also rejects queries whose end time is beyond `-futureRetention`. See [#10879](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10879).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add missing `__timestamp__` and `__value__` columns to CSV exported from the table view on the Query tab. See [#10975](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975).
## [v1.143.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.143.0)

View File

@@ -59,10 +59,6 @@ Once connected, you can build graphs and dashboards using [PromQL](https://prome
_Creating a datasource may require [specific permissions](https://grafana.com/docs/grafana/latest/administration/data-source-management/).
If you don't see an option to create a data source - try contacting system administrator._
If you run [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) and want to see its rules in [Grafana Alerting UI](https://grafana.com/docs/grafana/latest/alerting/),
then set configure `-vmalert.proxyURL` on VictoriaMetrics [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
or [vmselect in cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert).
## Multi-tenant access with vmauth and OIDC
[vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) can proxy Grafana datasource requests and enforce

View File

@@ -21,9 +21,23 @@ Recording rules results are persisted via remote write protocols and require `-r
`vmalert` is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
implementation and aims to be compatible with its syntax.
Configure `-vmalert.proxyURL` on VictoriaMetrics [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
or [vmselect in cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert)
to proxy requests to `vmalert`. Proxying is needed for the following cases:
* to proxy requests from [Grafana Alerting UI](https://grafana.com/docs/grafana/latest/alerting/);
* to access `vmalert`'s UI through [vmui](https://docs.victoriametrics.com/victoriametrics/#vmui).
[VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_vmalert_intro)
provides out-of-the-box alerting functionality based on `vmalert`. This service simplifies the setup
and management of alerting and recording rules as well as the integration with Alertmanager. For more details,
please refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriametrics.com/victoriametrics-cloud/alertmanager-setup-for-deployment/).
## Features
* Integration with VictoriaMetrics, VictoriaLogs, VictoriaTraces, Graphite and Prometheus compatible storages. See [Integrations](https://docs.victoriametrics.com/victoriametrics/vmalert/#integrations) for details;
* Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) and [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/);
* Integration with [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) and [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/);
* Integration with [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) which also uses [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victoriatraces/vmalert/);
* Prometheus [alerting rules definition format](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#defining-alerting-rules)
support;
* Integration with [Alertmanager](https://github.com/prometheus/alertmanager) starting from [Alertmanager v0.16.0-alpha](https://github.com/prometheus/alertmanager/releases/tag/v0.16.0-alpha.0);
@@ -48,8 +62,8 @@ implementation and aims to be compatible with its syntax.
To start using `vmalert` you will need the following things:
* list of rules - PromQL/MetricsQL/LogsQL/GraphiteQL expressions to execute;
* datasource address - a storage that [vmalert integrates with](https://docs.victoriametrics.com/victoriametrics/vmalert/#integrations) for executing queries;
* list of rules - PromQL/MetricsQL expressions to execute;
* datasource address - reachable endpoint with [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/#http-api) support for running queries against;
* notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
[config file](https://docs.victoriametrics.com/victoriametrics/vmalert/#notifier-configuration-file).
@@ -59,7 +73,7 @@ To start using `vmalert` you will need the following things:
* remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.
You can use the existing [docker-compose environment](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#victoriametrics-single-server)
as an example. It already contains vmalert configured with the list of alerting rules and integrated with Alert Manager and VictoriaMetrics.
as example. It already contains vmalert configured with list of alerting rules and integrated with Alert Manager and VictoriaMetrics.
Alternatively, build `vmalert` from sources:
@@ -73,7 +87,7 @@ Then run `vmalert`:
```sh
./bin/vmalert -rule=alert.rules \ # Path to the file with rules configuration. Supports wildcard and HTTP URL (S3/GCS are available in Enterprise).
-datasource.url=http://localhost:8428 \ # VictoriaMetrics URL to query for rules evaluation. See other available Integrations above.
-datasource.url=http://localhost:8428 \ # Prometheus HTTP API compatible datasource
-notifier.url=http://localhost:9093 \ # AlertManager URL (required if alerting rules are used)
-notifier.url=http://127.0.0.1:9093 \ # AlertManager replica URL
-remoteWrite.url=http://localhost:8428 \ # Remote write compatible storage to persist rules and alerts state info (required if recording rules are used)
@@ -93,7 +107,7 @@ See also [stream aggregation](https://docs.victoriametrics.com/victoriametrics/s
See the full list of configuration flags in [configuration](#configuration) section.
If you run multiple `vmalert` services for the same datasource or AlertManager and need to distinguish the results or alerts,
If you run multiple `vmalert` services on the same datastore or AlertManager and need to distinguish the results or alerts,
specify different `-external.label` command-line flags to indicate which `vmalert` generated them.
If rule result metrics have label that conflict with `-external.label`, `vmalert` will automatically rename
it with prefix `exported_`.
@@ -102,7 +116,6 @@ Configuration for [recording](https://prometheus.io/docs/prometheus/latest/confi
and [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) rules is very
similar to Prometheus rules and configured using YAML. Configuration examples may be found
in [testdata](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/config/testdata) folder.
Every `rule` belongs to a `group` and every configuration file may contain arbitrary number of groups:
```yaml
@@ -110,6 +123,12 @@ groups:
[ - <rule_group> ]
```
> Explore how to integrate `vmalert` with [VictoriaMetrics Anomaly Detection](https://docs.victoriametrics.com/anomaly-detection/) in the following [guide](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
> For users of [VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_vmalert_config),
> many of the configuration steps (including highly available setup of `vmalert` for cluster deployments) are handled automatically.
> Please, refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriametrics.com/victoriametrics-cloud/alertmanager-setup-for-deployment/) for more details.
### Groups
Each group has the following attributes:
@@ -211,11 +230,9 @@ rules:
### Rules
Every rule contains `expr` field for the expression to evaluate against configured datasource.
Depending on `group.type` value or `-rule.defaultRuleType` cmd-line flag expression can be one of the following types:
- `prometheus` (default) - [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) or [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expression.
- `vlogs` - [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/vmalert/) expression.
- `graphite` - [Graphite](https://graphite.readthedocs.io/en/stable/render_api.html) expression.
Every rule contains `expr` field for [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/)
or [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expression. `vmalert` will execute the configured
expression and then act according to the Rule type.
There are two types of Rules:
@@ -227,7 +244,8 @@ There are two types of Rules:
`-remoteWrite.url`. Recording rules are used to precompute frequently needed or computationally
expensive expressions and save their result as a new set of time series ([Prometheus recording rules docs](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)).
> `vmalert` forbids defining duplicates - rules with the same combination of name, expression and labels within one group.
`vmalert` forbids defining duplicates - rules with the same combination of name, expression, and labels
within one group.
#### Alerting rules
@@ -238,8 +256,8 @@ The syntax for alerting rule is the following:
alert: <string>
# The expression to evaluate. The expression language depends on the type value.
# By default, PromQL/MetricsQL expression is used. Other available types are "graphite" and "vlogs".
# See https://docs.victoriametrics.com/victoriametrics/vmalert/#integrations
# By default, PromQL/MetricsQL expression is used. If group.type="graphite", then the expression
# must contain valid Graphite expression.
expr: <string>
# Alerts are considered firing once they have been returned for this long.
@@ -285,110 +303,7 @@ annotations:
[ <labelname>: <tmpl_string> ]
```
#### Recording rules
The syntax for recording rules is the following:
```yaml
# The name of the time series to output to. Must be a valid metric name.
record: <string>
# The expression to evaluate. The expression language depends on the type value.
# By default, PromQL/MetricsQL expression is used. Other available types are "graphite" and "vlogs".
# See https://docs.victoriametrics.com/victoriametrics/vmalert/#integrations
expr: <string>
# Labels to add or overwrite labels from other external label sources, such as group labels, before storing the result.
#
# In case of conflicts, original labels are kept with prefix `exported_`.
# As a special case, specifying a label with an empty string value removes the label from the result if it exists
# in the original query result; otherwise, it is ignored.
#
# Labels do not support templating in https://docs.victoriametrics.com/victoriametrics/vmalert/#templating due to cardinality concerns. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8171.
labels:
[ <labelname>: <labelvalue> ]
# Whether to print debug information into logs.
# Information includes requests sent to the datasource.
# information - it will be printed to logs.
# Logs are printed with INFO level, so make sure that -loggerLevel=INFO to see the output.
[ debug: <bool> | default = false ]
# Defines the number of rule's updates entries stored in memory
# and available for view on rule's Details page.
# Overrides `rule.updateEntriesLimit` value for this specific rule.
[ update_entries_limit: <integer> | default 0 ]
```
For recording rules to work `-remoteWrite.url` must be specified.
## Integrations
vmalert can be integrated with different data sources for alerting and recording rules. But it deliberately allows
configuring only one `datasource.url`. We recommend running separate instances of vmalert for each datasource type
with specified `-rule.defaultRuleType=<datasource_type>` command-line flag.
### VictoriaMetrics
vmalert natively integrates with [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/) for alerting and
recording rules.
### VictoriaLogs
vmalert integrates with [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) and allows configuring alerting and recording rules using [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/).
Results of recording rules and alerting state should be persisted to the remote-write compatible storage, such as VictoriaMetrics.
To enable VictoriaLogs compatibility set `-rule.defaultRuleType=vlogs` commmand-line flag.
See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/) for details.
### VictoriaTraces
vmalert integrates with [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) in exactly the same way as
with [VictoriaLogs](https://docs.victoriametrics.com/victoriametrics/vmalert/#victorialogs).
### Graphite
vmalert integrates with [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) and allows configuring alerting and recording rules.
During evaluation, vmalert will send requests to `<-datasource.url>/render?format=json`.
To enable Graphite compatibility set `-rule.defaultRuleType=graphite` commmand-line flag.
Since VictoriaMetrics supports both Graphite and Prometheus APIs, it is possible to mix Graphite and VictoriaMetrics rules.
On the group level, set `type` field to specify to which datasource type it should belong: `prometheus` (MetricsQL) or `graphite` (GraphiteQL).
When using vmalert with both `graphite` and `prometheus` rules configured against cluster version of VictoriaMetrics dont forget
to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on the query type.
### Prometheus
vmalert uses [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/#http-api) for querying
and [Prometheus Remote Write v1 protocol](https://prometheus.io/docs/specs/prw/remote_write_spec/) for persisting
recording rules results and alerting state. Hence, it can be integrated with any Prometheus compatible storage
that supports these protocols.
### Grafana
To proxy requests from [Grafana Alerting UI](https://grafana.com/docs/grafana/latest/alerting/) configure `-vmalert.proxyURL`
on VictoriaMetrics [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
or [vmselect in cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert).
### vmui
To access rules UI through [vmui](https://docs.victoriametrics.com/victoriametrics/#vmui) configure `-vmalert.proxyURL`
on VictoriaMetrics [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
or [vmselect in cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert).
### vmanomaly
See how to integrate vmalert with [VictoriaMetrics Anomaly Detection](https://docs.victoriametrics.com/anomaly-detection/)
in the following [guide](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
### VictoriaMetrics Cloud
For users of [VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_vmalert_config),
many of the configuration steps (including highly available setup of `vmalert` for cluster deployments) are handled automatically.
Please refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriametrics.com/victoriametrics-cloud/alertmanager-setup-for-deployment/) for more details.
## Templating
#### Templating
It is allowed to use [Go templating](https://golang.org/pkg/text/template/) in annotations and labels(with limited support) to format data, iterate over
or execute expressions.
@@ -410,7 +325,7 @@ The following variables are available in templating:
Additionally, `vmalert` provides some extra templating functions listed in [template functions](#template-functions) and [reusable templates](#reusable-templates).
### Template functions
#### Template functions
`vmalert` provides the following template functions, which can be used during [templating](#templating):
@@ -434,7 +349,7 @@ Additionally, `vmalert` provides some extra templating functions listed in [temp
* `parseDurationTime` - parses the input string into [time.Duration](https://pkg.go.dev/time#Duration).
* `pathEscape` - escapes the input string, so it can be safely put inside path part of URL.
* `pathPrefix` - returns the path part of the `-external.url` command-line flag.
* `query` - executes query against `-datasource.url` and returns the query result.
* `query` - executes the [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) query against `-datasource.url` and returns the query result.
For example, `{{ query "sort_desc(process_resident_memory_bytes)" | first | value }}` executes the `sort_desc(process_resident_memory_bytes)`
query at `-datasource.url` and returns the first result.
* `queryEscape` - escapes the input string, so it can be safely put inside [query arg](https://en.wikipedia.org/wiki/Percent-encoding) part of URL.
@@ -454,7 +369,7 @@ Additionally, `vmalert` provides some extra templating functions listed in [temp
* `toUpper` - converts all the chars in the input string to uppercase.
* `value` - returns the numeric value from the input query result.
### Reusable templates
#### Reusable templates
Like in Alertmanager you can define [reusable templates](https://prometheus.io/docs/prometheus/latest/configuration/template_examples/#defining-reusable-templates)
to share same templates across annotations. Just define the templates in a file and
@@ -492,8 +407,44 @@ groups:
The `-rule.templates` flag supports wildcards so multiple files with templates can be loaded.
The content of `-rule.templates` can be also [hot reloaded](#hot-config-reload).
#### Recording rules
## Alerts state on restarts
The syntax for recording rules is following:
```yaml
# The name of the time series to output to. Must be a valid metric name.
record: <string>
# The expression to evaluate. The expression language depends on the type value.
# By default, MetricsQL expression is used. If group.type="graphite", then the expression
# must contain valid Graphite expression.
expr: <string>
# Labels to add or overwrite labels from other external label sources, such as group labels, before storing the result.
#
# In case of conflicts, original labels are kept with prefix `exported_`.
# As a special case, specifying a label with an empty string value removes the label from the result if it exists
# in the original query result; otherwise, it is ignored.
#
# Labels do not support templating in https://docs.victoriametrics.com/victoriametrics/vmalert/#templating due to cardinality concerns. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8171.
labels:
[ <labelname>: <labelvalue> ]
# Whether to print debug information into logs.
# Information includes requests sent to the datasource.
# information - it will be printed to logs.
# Logs are printed with INFO level, so make sure that -loggerLevel=INFO to see the output.
[ debug: <bool> | default = false ]
# Defines the number of rule's updates entries stored in memory
# and available for view on rule's Details page.
# Overrides `rule.updateEntriesLimit` value for this specific rule.
[ update_entries_limit: <integer> | default 0 ]
```
For recording rules to work `-remoteWrite.url` must be specified.
### Alerts state on restarts
`vmalert` holds alerts state in the memory. Restart of the `vmalert` process will reset the state of all active alerts
in the memory. To prevent `vmalert` from losing the state on restarts configure it to persist the state
@@ -514,7 +465,7 @@ in configured `-remoteRead.url`, weren't updated in the last `1h` (controlled by
or received state doesn't match current `vmalert` rules configuration. `vmalert` marks successfully restored rules
with `restored` label in [web UI](#web).
## Link to alert source
### Link to alert source
Alerting notifications sent by vmalert always contain a `source` link. By default, the link format
is the following `http://<vmalert-addr>/vmalert/alert?group_id=<group_id>&alert_id=<alert_id>`. On click, it opens
@@ -550,7 +501,7 @@ In addition to `source` link, some extra links could be added to alert's [annota
field. See [how we use them](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/9751ea10983d42068487624849cac7ad6fd7e1d8/deployment/docker/rules/alerts-cluster.yml#L44)
to link alerting rule and the corresponding panel on Grafana dashboard.
## Multitenancy
### Multitenancy
There are the following approaches exist for alerting and recording rules across
[multiple tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy):
@@ -608,7 +559,7 @@ The enterprise version of vmalert is available in `vmutils-*-enterprise.tar.gz`
at [release page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) and in `*-enterprise`
tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags) and [Quay](https://quay.io/repository/victoriametrics/vmalert?tab=tags).
## Reading rules from object storage
### Reading rules from object storage
[Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
from object storage:
@@ -627,7 +578,7 @@ The following [command-line flags](#flags) can be used for fine-tuning access to
* `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set.
* `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default.
## Topology examples
### Topology examples
The following sections are showing how `vmalert` may be used and configured
for different scenarios.
@@ -638,7 +589,7 @@ Please note, not all flags in examples are required:
you have recording rules or want to store [alerts state](#alerts-state-on-restarts) on `vmalert` restarts;
* `-notifier.url` is optional and is needed only if you have alerting rules.
### Single-node VictoriaMetrics
#### Single-node VictoriaMetrics
The simplest configuration where one single-node VM server is used for
rules execution, storing recording rules results and alerts state.
@@ -656,7 +607,7 @@ rules execution, storing recording rules results and alerts state.
![vmalert single](vmalert_single.webp)
{width="500"}
### Cluster VictoriaMetrics
#### Cluster VictoriaMetrics
In [cluster mode](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/)
VictoriaMetrics has separate components for writing and reading path:
@@ -679,7 +630,7 @@ Cluster mode could have multiple `vminsert` and `vmselect` components.
In case when you want to spread the load on these components - add balancers before them and configure
`vmalert` with balancer addresses. Please, see more about [VictoriaMetrics cluster architecture](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#architecture-overview).
### HA vmalert
#### HA vmalert
For High Availability(HA) user can run multiple identically configured `vmalert` instances.
It means all of them will execute the same rules, write state and results to
@@ -723,12 +674,12 @@ to ensure [high availability](https://github.com/prometheus/alertmanager#high-av
This example uses single-node VM server for the sake of simplicity.
Check how to replace it with [cluster VictoriaMetrics](#cluster-victoriametrics) if needed.
### Downsampling and aggregation via vmalert
#### Downsampling and aggregation via vmalert
_Please note, [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) might be more efficient
for cases when downsampling or aggregation need to be applied **before data gets into the TSDB.**_
`vmalert` can't modify existing data. But it can run arbitrary queries
`vmalert` can't modify existing data. But it can run arbitrary PromQL/MetricsQL queries
via [recording rules](#recording-rules) and backfill results to the configured `-remoteWrite.url`.
This ability allows to aggregate data. For example, the following rule will calculate the average value for
metric `http_requests` on the `5m` interval:
@@ -781,7 +732,7 @@ Flags `-remoteRead.url` and `-notifier.url` are omitted since we assume only rec
See also [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) and [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling).
### Multiple remote writes
#### Multiple remote writes
For persisting recording or alerting rule results `vmalert` requires `-remoteWrite.url` to be set.
But this flag supports only one destination. To persist rule results to multiple destinations
@@ -795,7 +746,7 @@ Using `vmagent` as a proxy provides additional benefits such as
[data persisting when storage is unreachable](https://docs.victoriametrics.com/victoriametrics/vmagent/#replication-and-high-availability),
or time series modification via [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/).
## Web
### Web
`vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
@@ -820,11 +771,28 @@ This may be used for better integration with Grafana unified alerting system. Se
* [How to query vmalert from single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert)
* [How to query vmalert from VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert)
## Graphite
vmalert sends requests to `<-datasource.url>/render?format=json` during evaluation of alerting and recording rules
if the corresponding group or rule contains `type: "graphite"` config option. It is expected that the `<-datasource.url>/render`
implements [Graphite Render API](https://graphite.readthedocs.io/en/stable/render_api.html) for `format=json`.
When using vmalert with both `graphite` and `prometheus` rules configured against cluster version of VM do not forget
to set `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust URL prefix automatically based on the query type.
## VictoriaLogs
vmalert supports [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) as a datasource for writing alerting and recording rules using [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/) for details.
## VictoriaTraces
vmalert supports [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) as a (`vlogs`) datasource for writing alerting and recording rules using [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/). See [this doc](https://docs.victoriametrics.com/victoriatraces/vmalert/) for details.
## Rules backfilling
vmalert supports alerting and recording rules backfilling (aka `replay`). In replay mode vmalert
can read the same rules configuration as normal, evaluate them on the given time range and backfill
results via remote write to the configured storage. vmalert supports only `prometheus` datasource type for backfilling.
results via remote write to the configured storage. vmalert supports any PromQL/MetricsQL compatible
data source for backfilling.
Please note, that response caching may lead to unexpected results during and after backfilling process.
In order to avoid this you need to reset cache contents or disable caching when using backfilling
@@ -884,9 +852,13 @@ vmalert respects `evaluationInterval` value set by flag or per-group during the
vmalert automatically disables caching on VictoriaMetrics side by sending `nocache=1` param. It allows
to prevent cache pollution and unwanted time range boundaries adjustment during backfilling.
Results of recording rules `replay` should match with results of normal rules evaluation.
#### Recording rules
Results of alerting rules `replay` are time series reflecting [alert's state](#alerts-state-on-restarts).
The result of recording rules `replay` should match with results of normal rules evaluation.
#### Alerting rules
The result of alerting rules `replay` is time series reflecting [alert's state](#alerts-state-on-restarts).
To see if `replayed` alert has fired in the past use the following PromQL/MetricsQL expression:
```

View File

@@ -51,8 +51,6 @@ func TestGetTimeSuccess(t *testing.T) {
f("-292273086-05-16T16:47:06Z", minTimeMsecs)
f("292277025-08-18T07:12:54.999999999Z", maxTimeMsecs)
f("1562529662.324", 1562529662324)
f("-9223372036.854", minTimeMsecs)
f("-9223372036.855", minTimeMsecs)
f("1223372036.855", 1223372036855)
}
@@ -85,4 +83,8 @@ func TestGetTimeError(t *testing.T) {
f("292277025-08-18T07:12:54.999999998Z")
f("123md")
f("-12.3md")
// relative duration that resolves to a timestamp before 1970
f("-9223372036.854")
f("-9223372036.855")
}

View File

@@ -228,6 +228,7 @@ func testAssertSearchResult(st *Storage, tr TimeRange, tfs *TagFilters, want []M
var s Search
s.Init(nil, st, []*TagFilters{tfs}, tr, 1e5, noDeadline)
defer s.MustClose()
var mbs []metricBlock
for s.NextMetricBlock() {
var b Block
@@ -241,7 +242,6 @@ func testAssertSearchResult(st *Storage, tr TimeRange, tfs *TagFilters, want []M
if err := s.Error(); err != nil {
return fmt.Errorf("search error: %w", err)
}
s.MustClose()
var got []MetricRow
var mn MetricName

View File

@@ -61,10 +61,11 @@ type Storage struct {
// indexdb rotation.
legacyNextRotationTimestamp atomic.Int64
path string
cachePath string
retentionMsecs int64
futureRetentionMsecs int64
path string
cachePath string
retentionMsecs int64
futureRetentionMsecs int64
denyQueriesOutsideRetention bool
// lock file for exclusive access to the storage on the given path.
flockF *os.File
@@ -161,14 +162,15 @@ type Storage struct {
// OpenOptions optional args for MustOpenStorage
type OpenOptions struct {
Retention time.Duration
FutureRetention time.Duration
MaxHourlySeries int
MaxDailySeries int
DisablePerDayIndex bool
TrackMetricNamesStats bool
IDBPrefillStart time.Duration
LogNewSeries bool
Retention time.Duration
FutureRetention time.Duration
DenyQueriesOutsideRetention bool
MaxHourlySeries int
MaxDailySeries int
DisablePerDayIndex bool
TrackMetricNamesStats bool
IDBPrefillStart time.Duration
LogNewSeries bool
}
// MustOpenStorage opens storage on the given path with the given retentionMsecs.
@@ -190,12 +192,13 @@ func MustOpenStorage(path string, opts OpenOptions) *Storage {
idbPrefillStart = time.Hour
}
s := &Storage{
path: path,
cachePath: filepath.Join(path, cacheDirname),
retentionMsecs: retention.Milliseconds(),
futureRetentionMsecs: futureRetention.Milliseconds(),
stopCh: make(chan struct{}),
idbPrefillStartSeconds: idbPrefillStart.Milliseconds() / 1000,
path: path,
cachePath: filepath.Join(path, cacheDirname),
retentionMsecs: retention.Milliseconds(),
futureRetentionMsecs: futureRetention.Milliseconds(),
denyQueriesOutsideRetention: opts.DenyQueriesOutsideRetention,
stopCh: make(chan struct{}),
idbPrefillStartSeconds: idbPrefillStart.Milliseconds() / 1000,
}
s.logNewSeries.Store(opts.LogNewSeries)
@@ -1225,6 +1228,25 @@ func searchAndMergeUniq(qt *querytracer.Tracer, s *Storage, tr TimeRange, search
return res, nil
}
// checkTimeRange returns an error if time range is outside the allowed
// -retentionPeriod or -futureRetention window when
// -denyQueriesOutsideRetention flag is set
func (s *Storage) checkTimeRange(tr TimeRange) error {
if !s.denyQueriesOutsideRetention {
return nil
}
minTimestamp, maxTimestamp := s.tb.getMinMaxTimestamps()
if minTimestamp <= tr.MinTimestamp && tr.MaxTimestamp <= maxTimestamp {
return nil
}
retention := time.Duration(s.retentionMsecs) * time.Millisecond
futureRetention := time.Duration(s.futureRetentionMsecs) * time.Millisecond
return fmt.Errorf("the given time range %s is outside the allowed -retentionPeriod=%s, -futureRetention=%s "+
"according to -denyQueriesOutsideRetention", &tr, retention, futureRetention)
}
// SearchTSIDs searches the TSIDs that correspond to filters within the given
// time range.
//
@@ -1236,6 +1258,10 @@ func (s *Storage) SearchTSIDs(qt *querytracer.Tracer, tfss []*TagFilters, tr Tim
qt = qt.NewChild("search TSIDs: filters=%s, timeRange=%s, maxMetrics=%d", tfss, &tr, maxMetrics)
defer qt.Done()
if err := s.checkTimeRange(tr); err != nil {
return nil, err
}
search := func(qt *querytracer.Tracer, idb *indexDB, tr TimeRange) ([]TSID, error) {
return idb.SearchTSIDs(qt, tfss, tr, maxMetrics, deadline)
}
@@ -1271,6 +1297,9 @@ func (s *Storage) SearchTSIDs(qt *querytracer.Tracer, tfss []*TagFilters, tr Tim
// MetricName.UnmarshalString().
func (s *Storage) SearchMetricNames(qt *querytracer.Tracer, tfss []*TagFilters, tr TimeRange, maxMetrics int, deadline uint64) ([]string, error) {
qt = qt.NewChild("search metric names: filters=%s, timeRange=%s, maxMetrics: %d", tfss, &tr, maxMetrics)
if err := s.checkTimeRange(tr); err != nil {
return nil, err
}
search := func(qt *querytracer.Tracer, idb *indexDB, tr TimeRange) ([]string, error) {
return idb.SearchMetricNames(qt, tfss, tr, maxMetrics, deadline)
}

View File

@@ -6,6 +6,7 @@ import (
"fmt"
"math/rand"
"path/filepath"
"slices"
"testing"
"testing/synctest"
"time"
@@ -1144,3 +1145,268 @@ func TestStorage_partitionsOutsideRetentionAreRemoved(t *testing.T) {
s.MustClose()
})
}
func TestStorage_denyQueriesOutsideRetention(t *testing.T) {
defer testRemoveAll(t)
tfsAll := NewTagFilters()
if err := tfsAll.Add(nil, []byte(".*"), false, true); err != nil {
t.Fatalf("TagFilters.Add() failed unexpectedly: %v", err)
}
tfssAll := []*TagFilters{tfsAll}
assertData := func(t *testing.T, s *Storage, tr TimeRange, wantData []MetricRow, wantErr bool) {
t.Helper()
err := testAssertSearchResult(s, tr, tfsAll, wantData)
gotErr := err != nil
if gotErr != wantErr {
t.Fatalf("Search: unmet error expectation for timeRange=%v: got %t, want %t (err: %v)", &tr, gotErr, wantErr, err)
}
}
assertMetricNames := func(t *testing.T, s *Storage, tr TimeRange, wantData []string, wantErr bool) {
t.Helper()
metricNames, err := s.SearchMetricNames(nil, tfssAll, tr, 1e9, noDeadline)
gotErr := err != nil
if gotErr != wantErr {
t.Fatalf("SearchMetricNames(): unmet error expectation for timeRange=%v: got %t, want %t (err: %v)", &tr, gotErr, wantErr, err)
}
var gotData []string
for _, name := range metricNames {
var mn MetricName
if err := mn.UnmarshalString(name); err != nil {
t.Fatalf("Could not unmarshal metric name %q: %v", name, err)
}
gotData = append(gotData, string(mn.MetricGroup))
}
if diff := cmp.Diff(wantData, gotData); diff != "" {
t.Fatalf("unexpected metric names (-want, +got):\n%s", diff)
}
}
assertLabelNames := func(t *testing.T, s *Storage, tr TimeRange, wantData []string, wantErr bool) {
t.Helper()
gotData, err := s.SearchLabelNames(nil, nil, tr, 1e9, 1e9, noDeadline)
gotErr := err != nil
if gotErr != wantErr {
t.Fatalf("SearchLabelNames(): unmet error expectation for timeRange=%v: got %t, want %t (err: %v)", &tr, gotErr, wantErr, err)
}
slices.Sort(gotData)
slices.Sort(wantData)
if diff := cmp.Diff(wantData, gotData); diff != "" {
t.Fatalf("unexpected label names (-want, +got):\n%s", diff)
}
}
assertLabelValues := func(t *testing.T, s *Storage, tr TimeRange, wantData []string, wantErr bool) {
t.Helper()
gotData, err := s.SearchLabelValues(nil, "__name__", nil, tr, 1e9, 1e9, noDeadline)
gotErr := err != nil
if gotErr != wantErr {
t.Fatalf("SearchLabelValues(): unmet error expectation for timeRange=%v: got %t, want %t (err: %v)", &tr, gotErr, wantErr, err)
}
slices.Sort(gotData)
slices.Sort(wantData)
if diff := cmp.Diff(wantData, gotData); diff != "" {
t.Fatalf("unexpected label values (-want, +got):\n%s", diff)
}
}
assertTagValueSuffixes := func(t *testing.T, s *Storage, tr TimeRange, wantData []string, wantErr bool) {
t.Helper()
gotData, err := s.SearchTagValueSuffixes(nil, tr, "", "", '.', 1e9, noDeadline)
gotErr := err != nil
if gotErr != wantErr {
t.Fatalf("SearchTagValueSuffixes(): unmet error expectation for timeRange=%v: got %t, want %t (err: %v)", &tr, gotErr, wantErr, err)
}
slices.Sort(gotData)
if diff := cmp.Diff(wantData, gotData); diff != "" {
t.Errorf("unexpected tag value suffixes (-want, +got):\n%s", diff)
}
}
assertGraphitePaths := func(t *testing.T, s *Storage, tr TimeRange, wantData []string, wantErr bool) {
t.Helper()
gotData, err := s.SearchGraphitePaths(nil, tr, []byte("*"), 1e9, noDeadline)
gotErr := err != nil
if gotErr != wantErr {
t.Fatalf("SearchTagValueSuffixes(): unmet error expectation for timeRange=%v: got %t, want %t (err: %v)", &tr, gotErr, wantErr, err)
}
slices.Sort(gotData)
if diff := cmp.Diff(wantData, gotData); diff != "" {
t.Errorf("unexpected graphite paths (-want, +got):\n%s", diff)
}
}
synctest.Test(t, func(t *testing.T) {
// synctests start at 2000-01-01T00:00:00Z
now := time.Now().UTC()
retention := 30 * 24 * time.Hour
futureRetention := 30 * 24 * time.Hour
day := 24 * time.Hour
trMatches := TimeRange{
MinTimestamp: now.Add(-retention).UnixMilli(),
MaxTimestamp: now.Add(futureRetention).UnixMilli(),
}
trContains := TimeRange{
MinTimestamp: now.Add(-(retention + day)).UnixMilli(),
MaxTimestamp: now.Add(futureRetention + day).UnixMilli(),
}
trInside := TimeRange{
MinTimestamp: now.Add(-(retention - day)).UnixMilli(),
MaxTimestamp: now.Add(futureRetention - day).UnixMilli(),
}
trOverlapsLeft := TimeRange{
MinTimestamp: now.Add(-(retention + day)).UnixMilli(),
MaxTimestamp: now.Add(futureRetention - day).UnixMilli(),
}
trOverlapsRight := TimeRange{
MinTimestamp: now.Add(-(retention - day)).UnixMilli(),
MaxTimestamp: now.Add(futureRetention + day).UnixMilli(),
}
trOutsideLeft := TimeRange{
MinTimestamp: now.Add(-(retention + 2*day)).UnixMilli(),
MaxTimestamp: now.Add(-(retention + day)).UnixMilli(),
}
trOutsideRight := TimeRange{
MinTimestamp: now.Add(futureRetention + day).UnixMilli(),
MaxTimestamp: now.Add(futureRetention + 2*day).UnixMilli(),
}
mn := MetricName{
MetricGroup: []byte("metric"),
}
metricNameRaw := mn.marshalRaw(nil)
mr := MetricRow{
MetricNameRaw: metricNameRaw,
Timestamp: now.UnixMilli(),
Value: 123,
}
wantMRs := []MetricRow{mr}
wantMetricNames := []string{"metric"}
wantLabelNames := []string{"__name__"}
emptyLabelNames := []string{}
wantLabelValues := []string{"metric"}
emptyLabelValues := []string{}
wantTagValueSuffixes := []string{"metric"}
emptyTagValueSuffixes := []string{}
wantGraphitePaths := []string{"metric"}
emptyGraphitePaths := []string{}
s := MustOpenStorage(t.Name(), OpenOptions{
Retention: retention,
FutureRetention: futureRetention,
})
s.AddRows([]MetricRow{mr}, defaultPrecisionBits)
s.DebugFlush()
assertData(t, s, trInside, wantMRs, false)
assertData(t, s, trMatches, wantMRs, false)
assertData(t, s, trContains, wantMRs, false)
assertData(t, s, trOverlapsLeft, wantMRs, false)
assertData(t, s, trOverlapsRight, wantMRs, false)
assertData(t, s, trOutsideLeft, nil, false)
assertData(t, s, trOutsideRight, nil, false)
assertMetricNames(t, s, trInside, wantMetricNames, false)
assertMetricNames(t, s, trMatches, wantMetricNames, false)
assertMetricNames(t, s, trContains, wantMetricNames, false)
assertMetricNames(t, s, trOverlapsLeft, wantMetricNames, false)
assertMetricNames(t, s, trOverlapsRight, wantMetricNames, false)
assertMetricNames(t, s, trOutsideLeft, nil, false)
assertMetricNames(t, s, trOutsideRight, nil, false)
assertLabelNames(t, s, trInside, wantLabelNames, false)
assertLabelNames(t, s, trMatches, wantLabelNames, false)
assertLabelNames(t, s, trContains, wantLabelNames, false)
assertLabelNames(t, s, trOverlapsLeft, wantLabelNames, false)
assertLabelNames(t, s, trOverlapsRight, wantLabelNames, false)
assertLabelNames(t, s, trOutsideLeft, emptyLabelNames, false)
assertLabelNames(t, s, trOutsideRight, emptyLabelNames, false)
assertLabelValues(t, s, trInside, wantLabelValues, false)
assertLabelValues(t, s, trMatches, wantLabelValues, false)
assertLabelValues(t, s, trContains, wantLabelValues, false)
assertLabelValues(t, s, trOverlapsLeft, wantLabelValues, false)
assertLabelValues(t, s, trOverlapsRight, wantLabelValues, false)
assertLabelValues(t, s, trOutsideLeft, emptyLabelValues, false)
assertLabelValues(t, s, trOutsideRight, emptyLabelValues, false)
assertTagValueSuffixes(t, s, trInside, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trMatches, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trContains, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trOverlapsLeft, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trOverlapsRight, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trOutsideLeft, emptyTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trOutsideRight, emptyTagValueSuffixes, false)
assertGraphitePaths(t, s, trInside, wantGraphitePaths, false)
assertGraphitePaths(t, s, trMatches, wantGraphitePaths, false)
assertGraphitePaths(t, s, trContains, wantGraphitePaths, false)
assertGraphitePaths(t, s, trOverlapsLeft, wantGraphitePaths, false)
assertGraphitePaths(t, s, trOverlapsRight, wantGraphitePaths, false)
assertGraphitePaths(t, s, trOutsideLeft, emptyGraphitePaths, false)
assertGraphitePaths(t, s, trOutsideRight, emptyGraphitePaths, false)
// Restart storage with DenyQueriesOutsideRetention on.
s.MustClose()
s = MustOpenStorage(t.Name(), OpenOptions{
Retention: retention,
FutureRetention: futureRetention,
DenyQueriesOutsideRetention: true,
})
defer s.MustClose()
assertData(t, s, trInside, wantMRs, false)
assertData(t, s, trMatches, wantMRs, false)
assertData(t, s, trContains, nil, true)
assertData(t, s, trOverlapsLeft, nil, true)
assertData(t, s, trOverlapsRight, nil, true)
assertData(t, s, trOutsideLeft, nil, true)
assertData(t, s, trOutsideRight, nil, true)
assertMetricNames(t, s, trInside, wantMetricNames, false)
assertMetricNames(t, s, trMatches, wantMetricNames, false)
assertMetricNames(t, s, trContains, nil, true)
assertMetricNames(t, s, trOverlapsLeft, nil, true)
assertMetricNames(t, s, trOverlapsRight, nil, true)
assertMetricNames(t, s, trOutsideLeft, nil, true)
assertMetricNames(t, s, trOutsideRight, nil, true)
// DenyQueriesOutsideRetention does not apply to searching label names.
assertLabelNames(t, s, trInside, wantLabelNames, false)
assertLabelNames(t, s, trMatches, wantLabelNames, false)
assertLabelNames(t, s, trContains, wantLabelNames, false)
assertLabelNames(t, s, trOverlapsLeft, wantLabelNames, false)
assertLabelNames(t, s, trOverlapsRight, wantLabelNames, false)
assertLabelNames(t, s, trOutsideLeft, emptyLabelNames, false)
assertLabelNames(t, s, trOutsideRight, emptyLabelNames, false)
// DenyQueriesOutsideRetention does not apply to searching label values.
assertLabelValues(t, s, trInside, wantLabelValues, false)
assertLabelValues(t, s, trMatches, wantLabelValues, false)
assertLabelValues(t, s, trContains, wantLabelValues, false)
assertLabelValues(t, s, trOverlapsLeft, wantLabelValues, false)
assertLabelValues(t, s, trOverlapsRight, wantLabelValues, false)
assertLabelValues(t, s, trOutsideLeft, emptyLabelValues, false)
assertLabelValues(t, s, trOutsideRight, emptyLabelValues, false)
// DenyQueriesOutsideRetention does not apply to searching tag value suffixes.
assertTagValueSuffixes(t, s, trInside, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trMatches, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trContains, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trOverlapsLeft, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trOverlapsRight, wantTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trOutsideLeft, emptyTagValueSuffixes, false)
assertTagValueSuffixes(t, s, trOutsideRight, emptyTagValueSuffixes, false)
// DenyQueriesOutsideRetention does not apply to searching graphite paths.
assertGraphitePaths(t, s, trInside, wantGraphitePaths, false)
assertGraphitePaths(t, s, trMatches, wantGraphitePaths, false)
assertGraphitePaths(t, s, trContains, wantGraphitePaths, false)
assertGraphitePaths(t, s, trOverlapsLeft, wantGraphitePaths, false)
assertGraphitePaths(t, s, trOverlapsRight, wantGraphitePaths, false)
assertGraphitePaths(t, s, trOutsideLeft, emptyGraphitePaths, false)
assertGraphitePaths(t, s, trOutsideRight, emptyGraphitePaths, false)
})
}

View File

@@ -1,16 +1,25 @@
package timeutil
import (
"fmt"
"time"
"github.com/VictoriaMetrics/metricsql"
)
var (
minDuration = time.Duration(minValidMilli * time.Millisecond)
maxDuration = time.Duration(maxValidMilli * time.Millisecond)
)
// ParseDuration parses duration string in Prometheus format
func ParseDuration(s string) (time.Duration, error) {
ms, err := metricsql.DurationValue(s, 0)
if err != nil {
return 0, err
}
if ms < minValidMilli || maxValidMilli < ms {
return 0, fmt.Errorf("duration %q must be in the range [%v, %v]", s, minDuration, maxDuration)
}
return time.Duration(ms) * time.Millisecond, nil
}

View File

@@ -1,6 +1,7 @@
package timeutil
import (
"fmt"
"testing"
"time"
)
@@ -27,3 +28,126 @@ func TestParseDuration(t *testing.T) {
f("-1m30s", -(time.Minute + time.Second*30))
f("1d-4h", time.Hour*20)
}
func TestParseDurationLimits(t *testing.T) {
f := func(s string, want time.Duration) {
t.Helper()
got, err := ParseDuration(s)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != want {
t.Fatalf("unexpected result: got %v, want %v", got, want)
}
}
var s string
var want time.Duration
s = fmt.Sprintf("%dms", int64(minValidMilli))
f(s, minDuration)
s = fmt.Sprintf("%dms", int64(maxValidMilli))
f(s, maxDuration)
s = fmt.Sprintf("%ds", int64(minValidSecond))
want = minValidSecond * time.Second
f(s, want)
s = fmt.Sprintf("%ds", int64(maxValidSecond))
want = maxValidSecond * time.Second
f(s, want)
// When no unit is specified, seconds are assumed.
s = fmt.Sprintf("%d", int64(minValidSecond))
want = minValidSecond * time.Second
f(s, want)
s = fmt.Sprintf("%d", int64(maxValidSecond))
want = maxValidSecond * time.Second
f(s, want)
minValidMinute := int64(minValidSecond) / 60
maxValidMinute := int64(maxValidSecond) / 60
s = fmt.Sprintf("%dm", minValidMinute)
want = time.Duration(minValidMinute) * time.Minute
f(s, want)
s = fmt.Sprintf("%dm", maxValidMinute)
want = time.Duration(maxValidMinute) * time.Minute
f(s, want)
minValidHour := minValidMinute / 60
maxValidHour := maxValidMinute / 60
s = fmt.Sprintf("%dh", minValidHour)
want = time.Duration(minValidHour) * time.Hour
f(s, want)
s = fmt.Sprintf("%dh", maxValidHour)
want = time.Duration(maxValidHour) * time.Hour
f(s, want)
minValidDay := minValidHour / 24
maxValidDay := maxValidHour / 24
s = fmt.Sprintf("%dd", minValidDay)
want = time.Duration(minValidDay) * 24 * time.Hour
f(s, want)
s = fmt.Sprintf("%dd", maxValidDay)
want = time.Duration(maxValidDay) * 24 * time.Hour
f(s, want)
minValidWeek := minValidDay / 7
maxValidWeek := maxValidDay / 7
s = fmt.Sprintf("%dw", minValidWeek)
want = time.Duration(minValidWeek) * 7 * 24 * time.Hour
f(s, want)
s = fmt.Sprintf("%dw", maxValidWeek)
want = time.Duration(maxValidWeek) * 7 * 24 * time.Hour
f(s, want)
minValidYear := minValidDay / 365
maxValidYear := maxValidDay / 365
s = fmt.Sprintf("%dy", minValidYear)
want = time.Duration(minValidYear) * 365 * 24 * time.Hour
f(s, want)
s = fmt.Sprintf("%dy", maxValidYear)
want = time.Duration(maxValidYear) * 365 * 24 * time.Hour
f(s, want)
}
func TestParseDurationOutsideLimits(t *testing.T) {
f := func(s string) {
t.Helper()
got, err := ParseDuration(s)
gotDuration := time.Duration(got) * time.Millisecond
if err == nil {
t.Fatalf("ParseDuration(%s) unexpected result: got %d (%s), want error", s, got, gotDuration)
}
}
f(fmt.Sprintf("%dms", int64(minValidMilli)-1))
f(fmt.Sprintf("%dms", int64(maxValidMilli)+1))
f(fmt.Sprintf("%ds", int64(minValidSecond)-1))
f(fmt.Sprintf("%ds", int64(maxValidSecond)+1))
minValidMinute := int64(minValidSecond)/60 - 1
f(fmt.Sprintf("%dm", minValidMinute))
maxValidMinute := int64(maxValidSecond)/60 + 1
f(fmt.Sprintf("%dm", maxValidMinute))
minValidHour := minValidMinute/60 - 1
f(fmt.Sprintf("%dh", minValidHour))
maxValidHour := maxValidMinute/60 + 2
f(fmt.Sprintf("%dh", maxValidHour))
minValidDay := minValidHour/24 - 1
f(fmt.Sprintf("%dd", minValidDay))
maxValidDay := maxValidHour/24 + 1
f(fmt.Sprintf("%dd", maxValidDay))
minValidWeek := minValidDay/7 - 1
f(fmt.Sprintf("%dw", minValidWeek))
maxValidWeek := maxValidDay/7 + 1
f(fmt.Sprintf("%dw", maxValidWeek))
minValidYear := minValidDay/365 - 1
f(fmt.Sprintf("%dy", minValidYear))
maxValidYear := maxValidDay/365 + 1
f(fmt.Sprintf("%dy", maxValidYear))
}

View File

@@ -74,7 +74,11 @@ func ParseTimeAt(s string, currentTimestamp int64) (int64, error) {
if d > 0 {
d = -d
}
return currentTimestamp + int64(d), nil
nsec := currentTimestamp + int64(d)
if nsec < 0 {
return 0, fmt.Errorf("time %s (%v) must be in the range [%v, %v]", sOrig, time.Unix(0, nsec).UTC(), minTime, maxTime)
}
return nsec, nil
}
if len(s) == 4 {
// Parse YYYY

View File

@@ -1,6 +1,8 @@
package timeutil
import (
"fmt"
"math"
"strings"
"testing"
"time"
@@ -204,10 +206,11 @@ func TestParseTimeAtSuccess(t *testing.T) {
}
func TestParseTimeAtLimits(t *testing.T) {
now := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
f := func(s string, wantTime time.Time) {
t.Helper()
currentTimestamp := time.Now().UnixNano()
got, err := ParseTimeAt(s, currentTimestamp)
got, err := ParseTimeAt(s, now.UnixNano())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -226,6 +229,14 @@ func TestParseTimeAtLimits(t *testing.T) {
}
east := location(t, "Etc/GMT-14") // UTC+14:00
west := location(t, "Etc/GMT+12") // UTC-12:00
var s string
// min timestamp
f("0", time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC))
s = fmt.Sprintf("-%d", now.Unix())
f(s, time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC))
s = fmt.Sprintf("now-%d", now.Unix())
f(s, time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC))
// min year
f("1970Z", time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC))
@@ -286,13 +297,39 @@ func TestParseTimeAtLimits(t *testing.T) {
f("2262-04-11T23:47:16Z", time.Date(2262, 4, 11, 23, 47, 16, 0, time.UTC))
f("2262-04-12T13:47:16+14:00", time.Date(2262, 4, 12, 13, 47, 16, 0, east))
f("2262-04-11T11:47:16-12:00", time.Date(2262, 4, 11, 11, 47, 16, 0, west))
// max timestamp
s = fmt.Sprintf("%d", int64(maxValidSecond))
f(s, time.Date(2262, 4, 11, 23, 47, 16, 0, time.UTC))
s = fmt.Sprintf("%d", int64(maxValidMilli))
f(s, time.Date(2262, 4, 11, 23, 47, 16, 854_000_000, time.UTC))
s = fmt.Sprintf("%d", int64(maxValidMicro))
f(s, time.Date(2262, 4, 11, 23, 47, 16, 854_775_000, time.UTC))
s = fmt.Sprintf("%d", int64(math.MaxInt64))
f(s, time.Date(2262, 4, 11, 23, 47, 16, 854_775_807, time.UTC))
// timestamps beyond max valid second are still valid but are treated as
// milliseconds.
s = fmt.Sprintf("%d", int64(maxValidSecond)+1)
f(s, time.Date(1970, 4, 17, 18, 2, 52, 37_000_000, time.UTC))
// timestamps beyond max valid millisecond are still valid but are treated
// as microseconds.
s = fmt.Sprintf("%d", int64(maxValidMilli)+1)
f(s, time.Date(1970, 4, 17, 18, 2, 52, 36_855_000, time.UTC))
// timestamps beyond max valid microsecond are still valid but are treated
// as nanoseconds.
s = fmt.Sprintf("%d", int64(maxValidMicro)+1)
f(s, time.Date(1970, 4, 17, 18, 2, 52, 36_854_776, time.UTC))
}
func TestParseTimeAtOutsideLimits(t *testing.T) {
now := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
f := func(s string) {
t.Helper()
currentTimestamp := time.Now().UnixNano()
got, err := ParseTimeAt(s, currentTimestamp)
got, err := ParseTimeAt(s, now.UnixNano())
if err == nil {
t.Fatalf("expected error but got %d", got)
}
@@ -301,6 +338,10 @@ func TestParseTimeAtOutsideLimits(t *testing.T) {
}
}
// min timestamp
f(fmt.Sprintf("-%d", now.Unix()+1))
f(fmt.Sprintf("now-%d", now.Unix()+1))
// min year
f("1969Z")
f("1970+14:00")
@@ -362,6 +403,24 @@ func TestParseTimeAtOutsideLimits(t *testing.T) {
f("2262-04-11T11:47:17-12:00")
}
func TestParseTimeAtOutsideLimits_Nanos(t *testing.T) {
now := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
f := func(s string) {
t.Helper()
got, err := ParseTimeAt(s, now.UnixNano())
if err == nil {
t.Fatalf("expected error but got %d", got)
}
if !strings.Contains(err.Error(), "cannot parse numeric timestamp") {
t.Fatalf("expected error: %v", err)
}
}
// max unix nano
f(fmt.Sprintf("%d", uint64(math.MaxInt64+1)))
}
func TestParseTimeMsecFailure(t *testing.T) {
f := func(s string) {
t.Helper()