Merge branch 'master' into issue-11324-2

Signed-off-by: JAYICE <1185430411@qq.com>

app/vmselect/netstorage/tmp_blocks_file.go

@@ -62,9 +62,6 @@ type tmpBlocksFile struct {
 	r *fs.ReaderAt
 
 	offset uint64
-
-	// err stores the first error occurred while writing the temporary blocks file.
-	err error
 }
 
 func getTmpBlocksFile() *tmpBlocksFile {
@@ -85,7 +82,6 @@ func putTmpBlocksFile(tbf *tmpBlocksFile) {
 	tbf.f = nil
 	tbf.r = nil
 	tbf.offset = 0
-	tbf.err = nil
 	tmpBlocksFilePool.Put(tbf)
 }
 
@@ -113,10 +109,6 @@ var (
 // and this must be handled.
 func (tbf *tmpBlocksFile) WriteBlockRefData(b []byte) (tmpBlockAddr, error) {
 	var addr tmpBlockAddr
-	if tbf.err != nil {
-		// Do not write anything to the tbf after the first failed write
-		return addr, tbf.err
-	}
 	addr.offset = tbf.offset
 	addr.size = len(b)
 	tbf.offset += uint64(addr.size)
@@ -130,8 +122,7 @@ func (tbf *tmpBlocksFile) WriteBlockRefData(b []byte) (tmpBlockAddr, error) {
 	if tbf.f == nil {
 		f, err := os.CreateTemp(tmpBlocksDir, "")
 		if err != nil {
-			tbf.err = fmt.Errorf("cannot create temporary blocks file at %q: %w", tmpBlocksDir, err)
-			return addr, tbf.err
+			return addr, err
 		}
 		tbf.f = f
 		tmpBlocksFilesCreated.Inc()
@@ -139,9 +130,7 @@ func (tbf *tmpBlocksFile) WriteBlockRefData(b []byte) (tmpBlockAddr, error) {
 	_, err := tbf.f.Write(tbf.buf)
 	tbf.buf = append(tbf.buf[:0], b...)
 	if err != nil {
-		// The blocks buffered at tbf.buf could be partially lost, mark the tbf as unusable.
-		tbf.err = fmt.Errorf("cannot write block to %q: %w", tbf.f.Name(), err)
-		return addr, tbf.err
+		return addr, fmt.Errorf("cannot write block to %q: %w", tbf.f.Name(), err)
 	}
 	return addr, nil
 }
@@ -152,16 +141,12 @@ func (tbf *tmpBlocksFile) Len() uint64 {
 }
 
 func (tbf *tmpBlocksFile) Finalize() error {
-	if tbf.err != nil {
-		return tbf.err
-	}
 	if tbf.f == nil {
 		return nil
 	}
 	fname := tbf.f.Name()
 	if _, err := tbf.f.Write(tbf.buf); err != nil {
-		tbf.err = fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
-		return tbf.err
+		return fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
 	}
 	tbf.buf = tbf.buf[:0]
 	r := fs.NewReaderAt(tbf.f)
@@ -181,10 +166,6 @@ func (tbf *tmpBlocksFile) Finalize() error {
 }
 
 func (tbf *tmpBlocksFile) MustReadBlockRefAt(partRef storage.PartRef, addr tmpBlockAddr) storage.BlockRef {
-	if tbf.err != nil {
-		// This should never happen, since Finalize() already returns the error for such a tbf.
-		logger.Panicf("BUG: cannot read block at %s from the temporary blocks file with the failed write: %s", addr, tbf.err)
-	}
 	var buf []byte
 	if tbf.r == nil {
 		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]

deployment/docker/rules/alerts-vmalert.yml

@@ -126,10 +126,10 @@ groups:
           (
             vmalert_alerting_rules_last_evaluation_samples
               > on(group,file) group_left()
-            max by (group,file) (vmalert_group_rule_results_limit * 0.9)
+            (vmalert_group_rule_results_limit * 0.9)
           )
             and on(group,file)
-          (max by (group,file) (vmalert_group_rule_results_limit) > 0)
+          (vmalert_group_rule_results_limit > 0)
         for: 5m
         labels:
           severity: warning
@@ -144,10 +144,10 @@ groups:
           (
             vmalert_recording_rules_last_evaluation_samples
               > on(group,file) group_left()
-            max by (group,file) (vmalert_group_rule_results_limit * 0.9)
+            (vmalert_group_rule_results_limit * 0.9)
           )
             and on(group,file)
-          (max by (group,file) (vmalert_group_rule_results_limit) > 0)
+          (vmalert_group_rule_results_limit > 0)
         for: 5m
         labels:
           severity: warning

docs/victoriametrics/changelog/CHANGELOG.md

@@ -29,11 +29,11 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmstorage` and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose the `vm_app_prev_shutdown_unclean` gauge. It is set to `1` when the previous process run didn't shut down cleanly. Added the `UncleanShutdown` [alerting rule](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml), which fires for 10 minutes after an unclean shutdown is detected. See [#8443](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8443).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): show the selected time zone UTC offset next to the date/time controls and allow opening time zone settings from it. See [#11332](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11332).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/), and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): show how the default value is calculated for command-line flags which derive it from the number of available CPU cores. For example, `-maxConcurrentInserts` now prints `(default 16 = 2*cgroup.AvailableCPUs())` in `-help` output instead of `(default 16)`. Updated flags: `-search.maxConcurrentRequests`, `-search.maxWorkersPerQuery`, `-fs.maxConcurrency`, `-remoteWrite.concurrency`, `-remoteWrite.queues`. See [#9680](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680). Thanks to @Vandit1604 for contribution.
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): accept Unix timestamp values with negative scientific notation exponents in [Prometheus querying API](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-usage) time parameters such as `start` and `end` in `/api/v1/query_range`. Previously, values such as `1000e-1` were rejected. See [#11427](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11427).
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix infinite loop in the OpenTelemetry Firehose ingestion endpoint (`/opentelemetry/api/v1/push`) when receiving a malformed record with an incomplete varint in the `data` field. Previously this caused the goroutine to spin forever, permanently consuming CPU until the process was restarted.
 * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): reuse connections to `-remoteWrite.url` when writing the results of recording rules and alerts. Previously every series was sent over a new connection, which left a lot of sockets in `TIME_WAIT` state and could exhaust the ephemeral port range. The number of idle connections can be tuned via the new `-remoteWrite.maxIdleConnections` command-line flag. Thanks @evkuzin for contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent process crash in `sort_by_label_numeric()` and `sort_by_label_numeric_desc()` when a label value contains a number with 309 or more digits. See [#11423](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11423).
-* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fail the query request directly when there is not enough disk space to store temporary search results. Previously, such queries could lead to vmselect crash. See [#4688](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4688).
 
 ## [v1.150.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.150.0)
 

lib/timeutil/time.go

@@ -79,7 +79,7 @@ func ParseTimeAt(s string, currentTimestamp int64) (int64, error) {
 		// Parse YYYY
 		return parseTimeAt("2006", s, tzOffset, sOrig)
 	}
-	if !strings.Contains(sOrig, "-") {
+	if !strings.Contains(sOrig, "-") || getExpIndex(sOrig) >= 0 {
 		nsec, ok := TryParseUnixTimestamp(sOrig)
 		if !ok {
 			return 0, fmt.Errorf("cannot parse numeric timestamp %q", sOrig)
@@ -186,80 +186,84 @@ func getExpIndex(s string) int {
 }
 
 func tryParseScientificUnixTimestamp(s string, decimalExp int64) (int64, bool) {
-	if decimalExp < 0 {
-		// Negative exponents on a fractional mantissa are intentionally not
-		// supported. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11268
+	intStr, fracStr, ok := expandScientificUnixTimestamp(s, decimalExp)
+	if !ok {
 		return 0, false
 	}
-	dotIdx := strings.IndexByte(s, '.')
-	if dotIdx < 0 {
-		n, ok := tryParseInt64(s)
-		if !ok {
-			return 0, false
-		}
-		n, ok = multiplyByDecimalExp(n, decimalExp)
+	if fracStr == "" {
+		n, ok := tryParseInt64(intStr)
 		if !ok {
 			return 0, false
 		}
 		return getUnixTimestampNanoseconds(n), true
 	}
-
-	intStr := s[:dotIdx]
-	fracStr := s[dotIdx+1:]
-	if decimalExp >= int64(len(fracStr)) {
-		// The exponent shifts the decimal point past every fractional digit.
-		n, ok := tryParseDecimalMantissaAsInt(intStr, fracStr)
-		if !ok {
-			return 0, false
-		}
-		decimalExp -= int64(len(fracStr))
-		n, ok = multiplyByDecimalExp(n, decimalExp)
-		if !ok {
-			return 0, false
-		}
-		return getUnixTimestampNanoseconds(n), true
-	}
-
-	// The exponent leaves fractional digits, e.g. 1.784144612388E9 == 1784144612.388
-	if decimalExp >= int64(len(decimalMultipliers)) {
-		return 0, false
-	}
-	decimalExpInt := int(decimalExp)
-	intStr = s[:dotIdx] + fracStr[:decimalExpInt]
-	fracStr = fracStr[decimalExpInt:]
 	return tryParseFractionalUnixTimestamp(intStr, fracStr)
 }
 
-func tryParseDecimalMantissaAsInt(intStr, fracStr string) (int64, bool) {
-	n, ok := tryParseInt64(intStr)
-	if !ok {
-		return 0, false
+func expandScientificUnixTimestamp(s string, decimalExp int64) (string, string, bool) {
+	dotIdx := strings.IndexByte(s, '.')
+	intStr := s
+	fracStr := ""
+	if dotIdx >= 0 {
+		intStr = s[:dotIdx]
+		fracStr = s[dotIdx+1:]
+	}
+	if _, ok := tryParseInt64(intStr); !ok {
+		return "", "", false
+	}
+	if !isDecimalString(fracStr) {
+		return "", "", false
 	}
 
-	decimalExp := int64(len(fracStr))
-	num, ok := multiplyByDecimalExp(n, decimalExp)
-	if !ok {
-		return 0, false
-	}
-
-	frac, ok := tryParseInt64(fracStr)
-	if !ok {
-		return 0, false
-	}
-
-	if num >= 0 {
-		if num > math.MaxInt64-frac {
-			return 0, false
+	isNegativeExp := decimalExp < 0
+	if isNegativeExp {
+		if decimalExp <= -int64(len(decimalMultipliers)) {
+			return "", "", false
 		}
-		num += frac
+		decimalExp = -decimalExp
+	}
+	if decimalExp > int64(math.MaxInt) || decimalExp < int64(math.MinInt) {
+		return "", "", false
+	}
+
+	intStr = strings.TrimPrefix(intStr, "+")
+	isNegative := strings.HasPrefix(intStr, "-")
+	if isNegative {
+		intStr = intStr[1:]
+	}
+
+	var shiftedIntStr, shiftedFracStr string
+	if isNegativeExp {
+		// e.g.
+		// 1. the integer and fractional part of 1.23e-5 should be 0 and 0000123 respectively.
+		// 2. the integer and fractional part of 123.4e-1 should be 12 and 34 respectively.
+		if decimalExp >= int64(len(intStr)) {
+			zerosToAdd := decimalExp - int64(len(intStr))
+			shiftedIntStr = "0"
+			shiftedFracStr = strings.Repeat("0", int(zerosToAdd)) + intStr + fracStr
+		} else {
+			decimalExpInt := int(decimalExp)
+			shiftedIntStr = intStr[:len(intStr)-decimalExpInt]
+			shiftedFracStr = intStr[len(intStr)-decimalExpInt:] + fracStr
+		}
+	} else if decimalExp >= int64(len(fracStr)) {
+		zerosToAdd := decimalExp - int64(len(fracStr))
+		if zerosToAdd >= int64(len(decimalMultipliers)) {
+			return "", "", false
+		}
+		// e.g. the integer part and fractional part of 1.23e5 should be 123000 and 0 respectively.
+		shiftedIntStr = intStr + fracStr + strings.Repeat("0", int(zerosToAdd))
+		shiftedFracStr = ""
 	} else {
-		if num < math.MinInt64+frac {
-			return 0, false
-		}
-		num -= frac
+		decimalExpInt := int(decimalExp)
+		shiftedIntStr = intStr + fracStr[:decimalExpInt]
+		shiftedFracStr = fracStr[decimalExpInt:]
 	}
 
-	return num, true
+	if isNegative {
+		shiftedIntStr = "-" + shiftedIntStr
+	}
+	return shiftedIntStr, shiftedFracStr, true
 }
 
 func tryParseFractionalUnixTimestamp(intStr, fracStr string) (int64, bool) {
@@ -270,15 +274,11 @@ func tryParseFractionalUnixTimestamp(intStr, fracStr string) (int64, bool) {
 	isNegative := n < 0 || n == 0 && strings.HasPrefix(intStr, "-")
 
 	multiplier, maxFracDigits := getUnixTimestampMultiplier(n)
+	if !isDecimalString(fracStr) {
+		return 0, false
+	}
 	// Truncate the fractional digits to valid length according to the unit precision.
 	if len(fracStr) > maxFracDigits {
-		// 1.123456789XXX is invalid.
-		tail := fracStr[maxFracDigits:]
-		for i := 0; i < len(tail); i++ {
-			if tail[i] < '0' || tail[i] > '9' {
-				return 0, false
-			}
-		}
 		fracStr = fracStr[:maxFracDigits]
 	}
 	if len(fracStr) == 0 {
@@ -309,26 +309,6 @@ func tryParseFractionalUnixTimestamp(intStr, fracStr string) (int64, bool) {
 	return n + frac, true
 }
 
-func multiplyByDecimalExp(n int64, decimalExp int64) (int64, bool) {
-	if decimalExp < 0 {
-		return 0, false
-	}
-	if decimalExp >= int64(len(decimalMultipliers)) {
-		return 0, false
-	}
-	if decimalExp == 0 {
-		return n, true
-	}
-
-	m := decimalMultipliers[decimalExp]
-
-	if n >= 0 && n > math.MaxInt64/m || n < 0 && n < math.MinInt64/m {
-		return 0, false
-	}
-
-	return n * m, true
-}
-
 var decimalMultipliers = [...]int64{0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18}
 
 const (
@@ -369,3 +349,12 @@ func tryParseInt64(s string) (int64, bool) {
 	}
 	return n, true
 }
+
+func isDecimalString(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] < '0' || s[i] > '9' {
+			return false
+		}
+	}
+	return true
+}

lib/timeutil/time_test.go

@@ -76,6 +76,7 @@ func TestTryParseUnixTimestamp_Success(t *testing.T) {
 	f("-1.23456789e9", -1234567890_000_000_000)
 	f("1.234567890123456789e18", 1234567890_123_456_789)
 	f("-1.234567890123456789e18", -1234567890_123_456_789)
+	f("0.0000000001e19", 1000000000_000_000_000)
 	f("0.23456789e9", 234567890_000_000_000)
 	f("123.456789123e9", 123456789_123_000_000)
 	f("-1234.5678912e9", -1234567891_200_000_000)
@@ -99,7 +100,19 @@ func TestTryParseUnixTimestamp_Success(t *testing.T) {
 	f("1.23456789e9", 1234567890_000_000_000)   // exponent consumes all frac digits (integer result)
 	f("1.23e1", 12_300_000_000)                 // == 12.3
 	f("1.234e0", 1_234_000_000)                 // == 1.234
+	f("-0.1e1", -1_000_000_000)                 // == -1
 	f("1234567890123456789.0e0", 1234567890_123_456_789)
+	f("1000e-1", 100_000_000_000)
+	f("1000E-1", 100_000_000_000)
+	f("-1000e-1", -100_000_000_000)
+	f("1e-1", 100_000_000)
+	f("-1e-1", -100_000_000)
+	f("1000.01e-1", 100_001_000_000)
+	f("-1000.01e-1", -100_001_000_000)
+	f("1.2E-1", 120_000_000)
+	f("1.2345678901e-1", 123_456_789)
+	f("0.12345678901", 123_456_789)
+	f("1.2345678901234567890123e-1", 123_456_789)
 
 }
 
@@ -138,7 +151,8 @@ func TestTryParseUnixTimestamp_Failure(t *testing.T) {
 	f("1.3e123456789090123")
 
 	// negative decimal exponent
-	f("1E-1")
+	f("1e-9223372036854775808")
+	f("1e-123456789090123")
 	f("1.3e-123456789090123")
 
 }
@@ -164,6 +178,8 @@ func TestParseTimeAtSuccess(t *testing.T) {
 	f("1562529662.678", now, 1562529662_678_000_000)
 	f("1562529662.678123", now, 1562529662_678_123_000)
 	f("1562529662.678123456", now, 1562529662_678_123_456)
+	f("1000e-1", now, 100_000_000_000)
+	f("1000.01e-1", now, 100_001_000_000)
 
 	// unix timestamp in milliseconds
 	f("1562529662678", now, 1562529662_678_000_000)