lib/logstorage: add support for parsing Unix timestamp in format pipe (#8767)

### Describe Your Changes This PR adds support for parsing Unix timestamps (both integer and float) in the format pipe using the `time:` prefix. The timestamp precision (seconds, milliseconds, microseconds, or nanoseconds) is automatically determined based on the value. It might be worth creating a new prefix for this rather than reusing `time:`, but I haven't found any compelling reason to extend the syntax. ### Checklist The following checks are **mandatory**: - [X] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). --------- Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com> Co-authored-by: Aliaksandr Valialkin <valyala@gmail.com>
2026-05-17 08:36:55 +03:00 · 2025-07-06 14:33:13 +04:00
parent 524f58c78c
commit 7294ffcdfc
10 changed files with 152 additions and 117 deletions
--- a/app/vlinsert/elasticsearch/elasticsearch.go
+++ b/app/vlinsert/elasticsearch/elasticsearch.go
@@ -231,7 +231,11 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
 	}
 	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
 		// Try parsing timestamp in seconds or milliseconds
-		return insertutil.ParseUnixTimestamp(s)
+		nsecs, ok := logstorage.TryParseUnixTimestamp(s)
+		if !ok {
+			return 0, fmt.Errorf("cannot parse unix timestamp %q", s)
+		}
+		return nsecs, nil
 	}
 	if len(s) == len("YYYY-MM-DD") {
 		t, err := time.Parse("2006-01-02", s)
--- a/app/vlinsert/insertutil/timestamp.go
+++ b/app/vlinsert/insertutil/timestamp.go
@@ -2,9 +2,6 @@ package insertutil

 import (
 	"fmt"
-	"math"
-	"strconv"
-	"strings"
 	"time"

 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
@@ -45,7 +42,11 @@ func parseTimestamp(s string) (int64, error) {
 		return time.Now().UnixNano(), nil
 	}
 	if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
-		return ParseUnixTimestamp(s)
+		nsecs, ok := logstorage.TryParseUnixTimestamp(s)
+		if !ok {
+			return 0, fmt.Errorf("cannot parse unix timestamp %q", s)
+		}
+		return nsecs, nil
 	}
 	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
 	if !ok {
@@ -53,54 +54,3 @@ func parseTimestamp(s string) (int64, error) {
 	}
 	return nsecs, nil
 }
-
-// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
-func ParseUnixTimestamp(s string) (int64, error) {
-	if strings.IndexByte(s, '.') >= 0 {
-		// Parse timestamp as floating-point value
-		f, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-		}
-		if f < (1<<31) && f >= (-1<<31) {
-			// The timestamp is in seconds.
-			return int64(f * 1e9), nil
-		}
-		if f < 1e3*(1<<31) && f >= 1e3*(-1<<31) {
-			// The timestamp is in milliseconds.
-			return int64(f * 1e6), nil
-		}
-		if f < 1e6*(1<<31) && f >= 1e6*(-1<<31) {
-			// The timestamp is in microseconds.
-			return int64(f * 1e3), nil
-		}
-		// The timestamp is in nanoseconds
-		if f > math.MaxInt64 {
-			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
-		}
-		if f < math.MinInt64 {
-			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
-		}
-		return int64(f), nil
-	}
-
-	// Parse timestamp as integer
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-	}
-	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds.
-		return n * 1e9, nil
-	}
-	if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
-		// The timestamp is in milliseconds.
-		return n * 1e6, nil
-	}
-	if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
-		// The timestamp is in microseconds.
-		return n * 1e3, nil
-	}
-	// The timestamp is in nanoseconds
-	return n, nil
-}
--- a/app/vlinsert/insertutil/timestamp_test.go
+++ b/app/vlinsert/insertutil/timestamp_test.go
@@ -6,63 +6,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )

-func TestParseUnixTimestamp_Success(t *testing.T) {
-	f := func(s string, timestampExpected int64) {
-		t.Helper()
-
-		timestamp, err := ParseUnixTimestamp(s)
-		if err != nil {
-			t.Fatalf("unexpected error in ParseUnixTimestamp(%q): %s", s, err)
-		}
-		if timestamp != timestampExpected {
-			t.Fatalf("unexpected timestamp returned from ParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
-		}
-	}
-
-	f("0", 0)
-
-	// nanoseconds
-	f("-1234567890123456789", -1234567890123456789)
-	f("1234567890123456789", 1234567890123456789)
-
-	// microseconds
-	f("-1234567890123456", -1234567890123456000)
-	f("1234567890123456", 1234567890123456000)
-	f("1234567890123456.789", 1234567890123456768)
-
-	// milliseconds
-	f("-1234567890123", -1234567890123000000)
-	f("1234567890123", 1234567890123000000)
-	f("1234567890123.456", 1234567890123456000)
-
-	// seconds
-	f("-1234567890", -1234567890000000000)
-	f("1234567890", 1234567890000000000)
-	f("-1234567890.123456", -1234567890123456000)
-}
-
-func TestParseUnixTimestamp_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := ParseUnixTimestamp(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error in ParseUnixTimestamp(%q)", s)
-		}
-	}
-
-	// non-numeric timestamp
-	f("")
-	f("foobar")
-	f("foo.bar")
-
-	// too big timestamp
-	f("12345678901234567890")
-	f("-12345678901234567890")
-	f("12345678901234567890.235424")
-	f("-12345678901234567890.235424")
-}
-
 func TestExtractTimestampFromFields_Success(t *testing.T) {
 	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
 		t.Helper()
--- a/app/vlinsert/loki/loki_json.go
+++ b/app/vlinsert/loki/loki_json.go
@@ -221,5 +221,9 @@ func parseLokiTimestamp(s string) (int64, error) {
 		// Special case - an empty timestamp must be substituted with the current time by the caller.
 		return 0, nil
 	}
-	return insertutil.ParseUnixTimestamp(s)
+	nsecs, ok := logstorage.TryParseUnixTimestamp(s)
+	if !ok {
+		return 0, fmt.Errorf("cannot parse unix timestamp %q", s)
+	}
+	return nsecs, nil
 }
--- a/docs/victorialogs/CHANGELOG.md
+++ b/docs/victorialogs/CHANGELOG.md
@@ -139,6 +139,7 @@ Released at 2025-04-25
 * FEATURE: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/): add an ability to force flush the recently ingested logs, so they become available for querying. See [these docs](https://docs.victoriametrics.com/victorialogs/#forced-flush).
 * FEATURE: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): add [`sample` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sample-pipe), which returns `1/Nth` random sample for the selected logs.
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add a toggle to handle ANSI escape sequences in log messages. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6614).
+* FEATURE: [`format` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#format-pipe): add support for parsing Unix timestamps with automatic precision detection. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8659).

 * BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): fix the Group tab to display raw JSON when `_msg` is missing. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8205).
 * BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): fix display of `Query history` icon in mobile view. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8788).
--- a/docs/victorialogs/LogsQL.md
+++ b/docs/victorialogs/LogsQL.md
@@ -2174,8 +2174,10 @@ String fields can be formatted with the following additional formatting rules:
 Numeric fields can be transformed into the following string representation at `format` pipe:

 - [RFC3339 time](https://www.rfc-editor.org/rfc/rfc3339) - by adding `time:` in front of the corresponding field name
-  containing [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) in nanoseconds.
-  For example, `format "time=<time:timestamp_nsecs>"`. The timestamp can be converted into nanoseconds with the [`math` pipe](#math-pipe).
+  containing [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time). 
+  The numeric timestamp can be in seconds, milliseconds, microseconds, or nanoseconds — the precision is automatically detected based on the value.
+  Both integer and floating-point values are supported.
+  For example, `format "time=<time:timestamp>"`.

 - Human-readable duration - by adding `duration:` in front of the corresponding numeric field name containing duration in nanoseconds.
  For example, `format "duration=<duration:duration_nsecs>"`. The duration can be converted into nanoseconds with the [`math` pipe](#math-pipe).
--- a/lib/logstorage/pipe_format.go
+++ b/lib/logstorage/pipe_format.go
@@ -226,7 +226,7 @@ func (shard *pipeFormatProcessorShard) formatRow(pf *pipeFormat, br *blockResult
 		case "lc":
 			b = appendLowercase(b, v)
 		case "time":
-			nsecs, ok := tryParseInt64(v)
+			nsecs, ok := TryParseUnixTimestamp(v)
 			if !ok {
 				b = append(b, v...)
 			} else {
--- a/lib/logstorage/pipe_format_test.go
+++ b/lib/logstorage/pipe_format_test.go
@@ -83,6 +83,29 @@ func TestPipeFormat(t *testing.T) {
 		},
 	})

+	// format Unix timestamp
+	f(`format 'a=<time:foo>, b=<time:bar>' as x`, [][]Field{
+		{
+			{"foo", "1717328141.123456789"},
+			{"bar", "1717328141.123456"},
+		},
+		{
+			{"foo", "-1717328141.123"},
+			{"bar", "-1717328141"},
+		},
+	}, [][]Field{
+		{
+			{"foo", "1717328141.123456789"},
+			{"bar", "1717328141.123456"},
+			{"x", "a=2024-06-02T11:35:41.123456768Z, b=2024-06-02T11:35:41.123456Z"},
+		},
+		{
+			{"foo", "-1717328141.123"},
+			{"bar", "-1717328141"},
+			{"x", "a=1915-08-01T12:24:18.877000192Z, b=1915-08-01T12:24:19Z"},
+		},
+	})
+
 	// skip_empty_results
 	f(`format '<foo><bar>' as x skip_empty_results`, [][]Field{
 		{
--- a/lib/logstorage/values_encoder.go
+++ b/lib/logstorage/values_encoder.go
@@ -380,6 +380,57 @@ func TryParseTimestampRFC3339Nano(s string) (int64, bool) {
 	return nsecs, true
 }

+// TryParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
+func TryParseUnixTimestamp(s string) (int64, bool) {
+	if strings.IndexByte(s, '.') >= 0 {
+		// Parse timestamp as floating-point value
+		f, err := strconv.ParseFloat(s, 64)
+		if err != nil {
+			return 0, false
+		}
+		if f < (1<<31) && f >= (-1<<31) {
+			// The timestamp is in seconds.
+			return int64(f * 1e9), true
+		}
+		if f < 1e3*(1<<31) && f >= 1e3*(-1<<31) {
+			// The timestamp is in milliseconds.
+			return int64(f * 1e6), true
+		}
+		if f < 1e6*(1<<31) && f >= 1e6*(-1<<31) {
+			// The timestamp is in microseconds.
+			return int64(f * 1e3), true
+		}
+		// The timestamp is in nanoseconds
+		if f > math.MaxInt64 {
+			return 0, false
+		}
+		if f < math.MinInt64 {
+			return 0, false
+		}
+		return int64(f), true
+	}
+
+	// Parse timestamp as integer
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return 0, false
+	}
+	if n < (1<<31) && n >= (-1<<31) {
+		// The timestamp is in seconds.
+		return n * 1e9, true
+	}
+	if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
+		// The timestamp is in milliseconds.
+		return n * 1e6, true
+	}
+	if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
+		// The timestamp is in microseconds.
+		return n * 1e3, true
+	}
+	// The timestamp is in nanoseconds
+	return n, true
+}
+
 func parseTimezoneOffset(s string) (int64, string, bool) {
 	if strings.HasSuffix(s, "Z") {
 		return 0, s[:len(s)-1], true
--- a/lib/logstorage/values_encoder_test.go
+++ b/lib/logstorage/values_encoder_test.go
@@ -239,6 +239,63 @@ func TestTryParseTimestampRFC3339Nano_Failure(t *testing.T) {
 	f("2023-01-23T23:33:ssZ")
 }

+func TestParseUnixTimestamp_Success(t *testing.T) {
+	f := func(s string, timestampExpected int64) {
+		t.Helper()
+
+		timestamp, ok := TryParseUnixTimestamp(s)
+		if !ok {
+			t.Fatalf("cannot parse timestamp %q", s)
+		}
+		if timestamp != timestampExpected {
+			t.Fatalf("unexpected timestamp returned from TryParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
+		}
+	}
+
+	f("0", 0)
+
+	// nanoseconds
+	f("-1234567890123456789", -1234567890123456789)
+	f("1234567890123456789", 1234567890123456789)
+
+	// microseconds
+	f("-1234567890123456", -1234567890123456000)
+	f("1234567890123456", 1234567890123456000)
+	f("1234567890123456.789", 1234567890123456768)
+
+	// milliseconds
+	f("-1234567890123", -1234567890123000000)
+	f("1234567890123", 1234567890123000000)
+	f("1234567890123.456", 1234567890123456000)
+
+	// seconds
+	f("-1234567890", -1234567890000000000)
+	f("1234567890", 1234567890000000000)
+	f("-1234567890.123456", -1234567890123456000)
+}
+
+func TestParseUnixTimestamp_Failure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+
+		_, ok := TryParseUnixTimestamp(s)
+		if ok {
+			t.Fatalf("expecting failure when parsing %q", s)
+		}
+	}
+
+	// non-numeric timestamp
+	f("")
+	f("foobar")
+	f("foo.bar")
+
+	// too big timestamp
+	f("12345678901234567890")
+	f("-12345678901234567890")
+	f("12345678901234567890.235424")
+	f("-12345678901234567890.235424")
+}
+
 func TestTryParseTimestampISO8601String_Success(t *testing.T) {
 	f := func(s string) {
 		t.Helper()