mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-05-17 08:36:55 +03:00
lib/flagutil: Add explicit month duration unit (M) for -retentionPeriod.
Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10181
This commit is contained in:
@@ -29,7 +29,8 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
|
||||
retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
|
||||
snapshotAuthKey = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
|
||||
forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
|
||||
forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
|
||||
|
||||
@@ -1542,9 +1542,9 @@ See also [Why IndexDB size is so large?](https://docs.victoriametrics.com/victor
|
||||
## Retention
|
||||
|
||||
Retention is configured with the `-retentionPeriod` command-line flag, which takes a number followed by a time unit
|
||||
character - `h(ours)`, `d(ays)`, `w(eeks)`, `y(ears)`. If the time unit is not specified, a month (31 days) is assumed.
|
||||
character - `h(ours)`, `d(ays)`, `w(eeks)`, `M(onth)`, `y(ears)`. If the time unit is not specified, a month (31 days) is assumed.
|
||||
For instance, `-retentionPeriod=3` means that the data will be stored for 3 months (93 days) and then deleted.
|
||||
The default retention period is one month. The **minimum retention** period is 24h or 1d.
|
||||
The default retention period is one month: 1M (31 days). The **minimum retention** period is 24h or 1d.
|
||||
|
||||
Data is split in per-month partitions inside `<-storageDataPath>/data/{small,big}` folders.
|
||||
**Data partitions** outside the configured retention are deleted **on the first day of the new month**.
|
||||
|
||||
@@ -31,6 +31,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `vm_rollup_result_cache_requests_total` which tracks the number of requests to the query rollup cache. See [#10117](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10117).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add `localStorage` availability checks with error reporting. See [#10085](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10085).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add `VMUI:`-prefixed `localStorage` keys and legacy key migration.
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add explicit month duration unit (`M`) for `-retentionPeriod` flag. This allows users to specify retention periods in months more explicitly. See [#10181](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10181).
|
||||
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix configuration reloading for `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` when vmagent is launched with empty files. Previously, if vmagent started with an empty config, subsequent config reloads were ignored. See [#10211](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10211).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Fixed a missing path error for `http://<victoriametrics-addr>:8428/zabbixconnector/api/v1/history`. See PR [10214](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10214)
|
||||
|
||||
@@ -372,7 +372,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
|
||||
Flag value can be read from the given file when using -reloadAuthKey=file:///abs/path/to/file or -reloadAuthKey=file://./relative/path/to/file.
|
||||
Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path
|
||||
-retentionPeriod value
|
||||
Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter
|
||||
Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter
|
||||
The following optional suffixes are supported: s (second), h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 1)
|
||||
-retentionTimezoneOffset duration
|
||||
The offset for performing indexdb rotation. If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)
|
||||
|
||||
@@ -161,7 +161,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
|
||||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
|
||||
-retentionPeriod value
|
||||
Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter
|
||||
Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter
|
||||
The following optional suffixes are supported: s (second), h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 1)
|
||||
-retentionTimezoneOffset duration
|
||||
The offset for performing indexdb rotation. If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)
|
||||
|
||||
@@ -76,7 +76,28 @@ func (d *RetentionDuration) Set(value string) error {
|
||||
d.valueString = ""
|
||||
return nil
|
||||
}
|
||||
// An attempt to parse value in months.
|
||||
|
||||
// An attempt to parse value as months with unit M(onth).
|
||||
if cutValue, found := strings.CutSuffix(value, "M"); found {
|
||||
months, err := strconv.ParseFloat(cutValue, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot parse months from %q: %w", value, err)
|
||||
}
|
||||
|
||||
if months > maxMonths {
|
||||
return fmt.Errorf("duration months must be smaller than %d; got %g", maxMonths, months)
|
||||
}
|
||||
if months < 0 {
|
||||
return fmt.Errorf("duration months cannot be negative; got %g", months)
|
||||
}
|
||||
d.msecs = int64(months * msecsPer31Days)
|
||||
d.valueString = value
|
||||
return nil
|
||||
}
|
||||
|
||||
// An attempt to parse value as a numeric month (without unit).
|
||||
// Such values should be treated as months for BC and historical reasons.
|
||||
// The format is deprecated, value with unit M should be used instead.
|
||||
months, err := strconv.ParseFloat(value, 64)
|
||||
if err == nil {
|
||||
if months > maxMonths {
|
||||
@@ -89,10 +110,11 @@ func (d *RetentionDuration) Set(value string) error {
|
||||
d.valueString = value
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parse duration.
|
||||
value = strings.ToLower(value)
|
||||
if strings.HasSuffix(value, "m") {
|
||||
return fmt.Errorf("duration in months must be set without `m` suffix due to ambiguity with duration in minutes; got %s", value)
|
||||
return fmt.Errorf("duration in months must be set with capital `M` suffix, lower case `m` means minutes and not allowed; got %s", value)
|
||||
}
|
||||
msecs, err := metricsql.PositiveDurationValue(value, 0)
|
||||
if err != nil {
|
||||
|
||||
@@ -31,12 +31,14 @@ func TestDurationSetFailure(t *testing.T) {
|
||||
f("-1")
|
||||
f("-34h")
|
||||
|
||||
f("1mM")
|
||||
|
||||
// RetentionDuration in minutes is confused with duration in months
|
||||
f("1m")
|
||||
}
|
||||
|
||||
func TestDurationSetSuccess(t *testing.T) {
|
||||
f := func(value string, expectedMsecs int64) {
|
||||
f := func(value string, expectedMsecs int64, expectedValueString string) {
|
||||
t.Helper()
|
||||
var d RetentionDuration
|
||||
if err := d.Set(value); err != nil {
|
||||
@@ -46,21 +48,21 @@ func TestDurationSetSuccess(t *testing.T) {
|
||||
t.Fatalf("unexpected result; got %d; want %d", d.Milliseconds(), expectedMsecs)
|
||||
}
|
||||
valueString := d.String()
|
||||
valueExpected := strings.ToLower(value)
|
||||
if valueString != valueExpected {
|
||||
t.Fatalf("unexpected valueString; got %q; want %q", valueString, valueExpected)
|
||||
if valueString != expectedValueString {
|
||||
t.Fatalf("unexpected valueString; got %q; want %q", valueString, expectedValueString)
|
||||
}
|
||||
}
|
||||
f("", 0)
|
||||
f("0", 0)
|
||||
f("1", msecsPer31Days)
|
||||
f("123.456", 123.456*msecsPer31Days)
|
||||
f("1h", 3600*1000)
|
||||
f("1.5d", 1.5*24*3600*1000)
|
||||
f("2.3W", 2.3*7*24*3600*1000)
|
||||
f("1w", 7*24*3600*1000)
|
||||
f("0.25y", 0.25*365*24*3600*1000)
|
||||
f("100y", 100*365*24*3600*1000)
|
||||
f("", 0, "")
|
||||
f("0", 0, "0")
|
||||
f("1", msecsPer31Days, "1")
|
||||
f("123.456", 123.456*msecsPer31Days, "123.456")
|
||||
f("1h", 3600*1000, "1h")
|
||||
f("1.5d", 1.5*24*3600*1000, "1.5d")
|
||||
f("2.3W", 2.3*7*24*3600*1000, "2.3w")
|
||||
f("1w", 7*24*3600*1000, "1w")
|
||||
f("0.25y", 0.25*365*24*3600*1000, "0.25y")
|
||||
f("3M", 93*24*3600*1000, "3M")
|
||||
f("100y", 100*365*24*3600*1000, "100y")
|
||||
}
|
||||
|
||||
func TestDurationDuration(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user