mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-05-17 00:26:36 +03:00
lib/timeutil: put TryParseUnixTimestamp function here
This allows avoiding precision loss at VictoriaLogs query time when parsing fractional unix timestamps with millisecond, microsecond and nanosecond precisions.
This is a follow-up for 1d0e96c8d2
See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8767#discussion_r2051657518
This commit is contained in:
Aliaksandr Valialkin
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
)
|
||||
|
||||
@@ -231,7 +232,7 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
|
||||
}
|
||||
if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
|
||||
// Try parsing timestamp in seconds or milliseconds
|
||||
nsecs, ok := logstorage.TryParseUnixTimestamp(s)
|
||||
nsecs, ok := timeutil.TryParseUnixTimestamp(s)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("cannot parse unix timestamp %q", s)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
)
|
||||
|
||||
// ExtractTimestampFromFields extracts timestamp in nanoseconds from the first field the name from timeFields at fields.
|
||||
@@ -42,7 +43,7 @@ func parseTimestamp(s string) (int64, error) {
|
||||
return time.Now().UnixNano(), nil
|
||||
}
|
||||
if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
|
||||
nsecs, ok := logstorage.TryParseUnixTimestamp(s)
|
||||
nsecs, ok := timeutil.TryParseUnixTimestamp(s)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("cannot parse unix timestamp %q", s)
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
)
|
||||
|
||||
var maxRequestSize = flagutil.NewBytes("loki.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single Loki request")
|
||||
@@ -221,7 +222,7 @@ func parseLokiTimestamp(s string) (int64, error) {
|
||||
// Special case - an empty timestamp must be substituted with the current time by the caller.
|
||||
return 0, nil
|
||||
}
|
||||
nsecs, ok := logstorage.TryParseUnixTimestamp(s)
|
||||
nsecs, ok := timeutil.TryParseUnixTimestamp(s)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("cannot parse unix timestamp %q", s)
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prefixfilter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
)
|
||||
|
||||
// pipeFormat processes '| format ...' pipe.
|
||||
@@ -226,7 +227,7 @@ func (shard *pipeFormatProcessorShard) formatRow(pf *pipeFormat, br *blockResult
|
||||
case "lc":
|
||||
b = appendLowercase(b, v)
|
||||
case "time":
|
||||
nsecs, ok := TryParseUnixTimestamp(v)
|
||||
nsecs, ok := timeutil.TryParseUnixTimestamp(v)
|
||||
if !ok {
|
||||
b = append(b, v...)
|
||||
} else {
|
||||
|
||||
@@ -380,159 +380,6 @@ func TryParseTimestampRFC3339Nano(s string) (int64, bool) {
|
||||
return nsecs, true
|
||||
}
|
||||
|
||||
// TryParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
|
||||
func TryParseUnixTimestamp(s string) (int64, bool) {
|
||||
if expIdx := getExpIndex(s); expIdx >= 0 {
|
||||
// The timestamp is a scientific number such as 1.234e5
|
||||
decimalExp, ok := tryParseInt64(s[expIdx+1:])
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if decimalExp > math.MaxInt64 || decimalExp < math.MinInt64 {
|
||||
return 0, false
|
||||
}
|
||||
n, ok := tryParseScientificNumberForUnixTimestamp(s[:expIdx], int(decimalExp))
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return getUnixTimestampNanoseconds(n), true
|
||||
}
|
||||
|
||||
dotIdx := strings.IndexByte(s, '.')
|
||||
if dotIdx < 0 {
|
||||
// The timestamp is integer.
|
||||
n, ok := tryParseInt64(s)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return getUnixTimestampNanoseconds(n), true
|
||||
}
|
||||
|
||||
// The timestamp is fractional.
|
||||
intStr := s[:dotIdx]
|
||||
fracStr := s[dotIdx+1:]
|
||||
n, ok := tryParseFractionalNumberForUnixTimestamp(intStr, fracStr)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Adjust the n to multiples of thousands, since this is expected by getUnixTimestampNanoseconds.
|
||||
decimalExp := len(fracStr)
|
||||
for decimalExp%3 != 0 {
|
||||
if n >= 0 && n > math.MaxInt64/10 || n < 0 && n < math.MinInt64/10 {
|
||||
return 0, false
|
||||
}
|
||||
n *= 10
|
||||
decimalExp++
|
||||
}
|
||||
|
||||
return getUnixTimestampNanoseconds(n), true
|
||||
}
|
||||
|
||||
func getExpIndex(s string) int {
|
||||
if n := strings.IndexByte(s, 'e'); n >= 0 {
|
||||
return n
|
||||
}
|
||||
if n := strings.IndexByte(s, 'E'); n >= 0 {
|
||||
return n
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func tryParseScientificNumberForUnixTimestamp(s string, decimalExp int) (int64, bool) {
|
||||
dotIdx := strings.IndexByte(s, '.')
|
||||
if dotIdx < 0 {
|
||||
n, ok := tryParseInt64(s)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return multiplyByDecimalExp(n, decimalExp)
|
||||
}
|
||||
|
||||
intStr := s[:dotIdx]
|
||||
fracStr := s[dotIdx+1:]
|
||||
if decimalExp < len(fracStr) {
|
||||
return 0, false
|
||||
}
|
||||
n, ok := tryParseFractionalNumberForUnixTimestamp(intStr, fracStr)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
decimalExp -= len(fracStr)
|
||||
return multiplyByDecimalExp(n, decimalExp)
|
||||
}
|
||||
|
||||
func tryParseFractionalNumberForUnixTimestamp(intStr, fracStr string) (int64, bool) {
|
||||
n, ok := tryParseInt64(intStr)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
decimalExp := len(fracStr)
|
||||
num, ok := multiplyByDecimalExp(n, decimalExp)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
frac, ok := tryParseInt64(fracStr)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
if num >= 0 {
|
||||
if num > math.MaxInt64-frac {
|
||||
return 0, false
|
||||
}
|
||||
num += frac
|
||||
} else {
|
||||
if num < math.MinInt64+frac {
|
||||
return 0, false
|
||||
}
|
||||
num -= frac
|
||||
}
|
||||
|
||||
return num, true
|
||||
}
|
||||
|
||||
func multiplyByDecimalExp(n int64, decimalExp int) (int64, bool) {
|
||||
if decimalExp < 0 {
|
||||
return 0, false
|
||||
}
|
||||
if decimalExp >= len(decimalMultipliers) {
|
||||
return 0, false
|
||||
}
|
||||
if decimalExp == 0 {
|
||||
return n, true
|
||||
}
|
||||
|
||||
m := decimalMultipliers[decimalExp]
|
||||
|
||||
if n >= 0 && n > math.MaxInt64/m || n < 0 && n < math.MinInt64/m {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return n * m, true
|
||||
}
|
||||
|
||||
var decimalMultipliers = [...]int64{0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18}
|
||||
|
||||
func getUnixTimestampNanoseconds(n int64) int64 {
|
||||
if n < (1<<31) && n >= (-1<<31) {
|
||||
// The timestamp is in seconds.
|
||||
return n * 1e9
|
||||
}
|
||||
if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
|
||||
// The timestamp is in milliseconds.
|
||||
return n * 1e6
|
||||
}
|
||||
if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
|
||||
// The timestamp is in microseconds.
|
||||
return n * 1e3
|
||||
}
|
||||
// The timestamp is in nanoseconds
|
||||
return n
|
||||
}
|
||||
|
||||
func parseTimezoneOffset(s string) (int64, string, bool) {
|
||||
if strings.HasSuffix(s, "Z") {
|
||||
return 0, s[:len(s)-1], true
|
||||
|
||||
@@ -239,120 +239,6 @@ func TestTryParseTimestampRFC3339Nano_Failure(t *testing.T) {
|
||||
f("2023-01-23T23:33:ssZ")
|
||||
}
|
||||
|
||||
func TestTryParseUnixTimestamp_Success(t *testing.T) {
|
||||
f := func(s string, timestampExpected int64) {
|
||||
t.Helper()
|
||||
|
||||
timestamp, ok := TryParseUnixTimestamp(s)
|
||||
if !ok {
|
||||
t.Fatalf("cannot parse timestamp %q", s)
|
||||
}
|
||||
if timestamp != timestampExpected {
|
||||
t.Fatalf("unexpected timestamp returned from TryParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f("0", 0)
|
||||
|
||||
// nanoseconds
|
||||
f("-1234567890123456789", -1234567890123456789)
|
||||
f("1234567890123456789", 1234567890123456789)
|
||||
f("1234567890123456.789", 1234567890123456789)
|
||||
|
||||
// microseconds
|
||||
f("-1234567890123456", -1234567890123456000)
|
||||
f("1234567890123456", 1234567890123456000)
|
||||
f("1234567890123456.789", 1234567890123456789)
|
||||
|
||||
// milliseconds
|
||||
f("-1234567890123", -1234567890123000000)
|
||||
f("1234567890123", 1234567890123000000)
|
||||
f("1234567890123.456", 1234567890123456000)
|
||||
|
||||
// seconds
|
||||
f("-1234567890", -1234567890000000000)
|
||||
f("1234567890", 1234567890000000000)
|
||||
f("1234567890.123456789", 1234567890123456789)
|
||||
f("1234567890.12345678", 1234567890123456780)
|
||||
f("1234567890.1234567", 1234567890123456700)
|
||||
f("-1234567890.123456", -1234567890123456000)
|
||||
f("-1234567890.12345", -1234567890123450000)
|
||||
f("-1234567890.1234", -1234567890123400000)
|
||||
f("-1234567890.123", -1234567890123000000)
|
||||
f("-1234567890.12", -1234567890120000000)
|
||||
f("-1234567890.1", -1234567890100000000)
|
||||
|
||||
// scientific notation
|
||||
f("1e9", 1000000000000000000)
|
||||
f("1.234e9", 1234000000000000000)
|
||||
f("-1.23456789e9", -1234567890000000000)
|
||||
f("1.234567890123456789e18", 1234567890123456789)
|
||||
f("-1.234567890123456789e18", -1234567890123456789)
|
||||
f("0.23456789e9", 234567890000000000)
|
||||
f("123.456789123e9", 123456789123000000)
|
||||
f("-1234.5678912e9", -1234567891200000000)
|
||||
f("123.678912e7", 1236789120000000000)
|
||||
f("1.23e7", 12300000000000000)
|
||||
f("1.23e6", 1230000000000000)
|
||||
f("1.23e5", 123000000000000)
|
||||
f("1.23e4", 12300000000000)
|
||||
f("1.23e3", 1230000000000)
|
||||
f("1.23e2", 123000000000)
|
||||
f("1.2e1", 12000000000)
|
||||
f("1123.456789123456789E15", 1123456789123456789)
|
||||
}
|
||||
|
||||
func TestTryParseUnixTimestamp_Failure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
_, ok := TryParseUnixTimestamp(s)
|
||||
if ok {
|
||||
t.Fatalf("expecting failure when parsing %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
// non-numeric timestamp
|
||||
f("")
|
||||
f("foobar")
|
||||
f("foo.bar")
|
||||
f("1.12345671x34")
|
||||
f("1.3e12345678x0123")
|
||||
f("1xs.12345671")
|
||||
f("1xs.12345671e5")
|
||||
f("-1xs.12345671e5")
|
||||
|
||||
// missing fractional part
|
||||
f("1233344.")
|
||||
|
||||
// too big timestamp
|
||||
f("12345678901234567.891")
|
||||
f("12345678901234567890")
|
||||
f("12345678901234.567891")
|
||||
f("12345678901234567890e3")
|
||||
f("12345678901234567890.234e3")
|
||||
f("-12345678901234567890")
|
||||
f("12345678901234567890.235424")
|
||||
f("12345678901234567890.235424e3")
|
||||
f("-12345678901234567890.235424")
|
||||
f("12345678901234567.89")
|
||||
f("12345678901234567.8")
|
||||
|
||||
// too big fractional part
|
||||
f("0.1234567890123456789123")
|
||||
f("-0.1234567890123456789123")
|
||||
|
||||
// too big decimal exponent
|
||||
f("1e19")
|
||||
f("1.3e123456789090123")
|
||||
|
||||
// too small decimal exponent
|
||||
f("1.23e1")
|
||||
f("1.234e0")
|
||||
f("1E-1")
|
||||
f("1.3e-123456789090123")
|
||||
}
|
||||
|
||||
func TestTryParseTimestampISO8601String_Success(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -6,150 +6,6 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkTryParseUnixTimestamp(b *testing.B) {
|
||||
b.Run("seconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890",
|
||||
"1234567891",
|
||||
"1234567892",
|
||||
"1234567893",
|
||||
"1234567894",
|
||||
"1234567895",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("milliseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890123",
|
||||
"1234567891123",
|
||||
"1234567892123",
|
||||
"1234567893123",
|
||||
"1234567894123",
|
||||
"1234567895123",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("microseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890123456",
|
||||
"1234567891123456",
|
||||
"1234567892123456",
|
||||
"1234567893123456",
|
||||
"1234567894123456",
|
||||
"1234567895123456",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("nanoseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890123456789",
|
||||
"1234567891123456789",
|
||||
"1234567892123456789",
|
||||
"1234567893123456789",
|
||||
"1234567894123456789",
|
||||
"1234567895123456789",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
|
||||
b.Run("scientific-seconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1.234567890e9",
|
||||
"1.234567891e9",
|
||||
"1.234567892e9",
|
||||
"1.234567893e9",
|
||||
"1.234567894e9",
|
||||
"1.234567895e9",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("scientific-milliseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1.234567890123e12",
|
||||
"1.234567891123e12",
|
||||
"1.234567892123e12",
|
||||
"1.234567893123e12",
|
||||
"1.234567894123e12",
|
||||
"1.234567895123e12",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("scientific-microseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1.234567890123456e15",
|
||||
"1.234567891123456e15",
|
||||
"1.234567892123456e15",
|
||||
"1.234567893123456e15",
|
||||
"1.234567894123456e15",
|
||||
"1.234567895123456e15",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("scientific-nanoseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1.234567890123456789e18",
|
||||
"1.234567891123456789e18",
|
||||
"1.234567892123456789e18",
|
||||
"1.234567893123456789e18",
|
||||
"1.234567894123456789e18",
|
||||
"1.234567895123456789e18",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
|
||||
b.Run("fractional-milliseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890.123",
|
||||
"1234567891.123",
|
||||
"1234567892.123",
|
||||
"1234567893.123",
|
||||
"1234567894.123",
|
||||
"1234567895.123",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("fractional-microseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890.123456",
|
||||
"1234567891.123456",
|
||||
"1234567892.123456",
|
||||
"1234567893.123456",
|
||||
"1234567894.123456",
|
||||
"1234567895.123456",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("fractional-nanoseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890.123456789",
|
||||
"1234567891.123456789",
|
||||
"1234567892.123456789",
|
||||
"1234567893.123456789",
|
||||
"1234567894.123456789",
|
||||
"1234567895.123456789",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
}
|
||||
|
||||
func benchmarkTryParseUnixTimestamp(b *testing.B, a []string) {
|
||||
b.SetBytes(int64(len(a)))
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
nSum := int64(0)
|
||||
for pb.Next() {
|
||||
for _, s := range a {
|
||||
n, ok := TryParseUnixTimestamp(s)
|
||||
if !ok {
|
||||
panic(fmt.Errorf("cannot parse timestamp %q", s))
|
||||
}
|
||||
nSum += n
|
||||
}
|
||||
}
|
||||
GlobalSink.Add(uint64(nSum))
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkTryParseTimestampRFC3339Nano(b *testing.B) {
|
||||
a := []string{
|
||||
"2023-01-15T23:45:51Z",
|
||||
|
||||
@@ -94,7 +94,11 @@ func ParseTimeAt(s string, currentTimestamp int64) (int64, error) {
|
||||
return tzOffset + t.UnixNano(), nil
|
||||
}
|
||||
if !strings.Contains(sOrig, "-") {
|
||||
return parseNumericTimestamp(sOrig)
|
||||
nsec, ok := TryParseUnixTimestamp(sOrig)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("cannot parse numeric timestamp %q", sOrig)
|
||||
}
|
||||
return nsec, nil
|
||||
}
|
||||
if len(s) == 7 {
|
||||
// Parse YYYY-MM
|
||||
@@ -144,39 +148,169 @@ func ParseTimeAt(s string, currentTimestamp int64) (int64, error) {
|
||||
return t.UnixNano(), nil
|
||||
}
|
||||
|
||||
// parseNumericTimestamp parses timestamp at s in seconds, milliseconds, microseconds or nanoseconds.
|
||||
// TryParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
|
||||
//
|
||||
// It returns nanoseconds for the parsed timestamp.
|
||||
func parseNumericTimestamp(s string) (int64, error) {
|
||||
if strings.ContainsAny(s, ".eE") {
|
||||
// The timestamp is a floating-point number
|
||||
ts, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
// The supported formats for s:
|
||||
//
|
||||
// - Integer. For example, 1234567890
|
||||
// - Fractional. For example, 1234567890.123
|
||||
// - Scientific. For example, 1.23456789e9
|
||||
func TryParseUnixTimestamp(s string) (int64, bool) {
|
||||
if expIdx := getExpIndex(s); expIdx >= 0 {
|
||||
// The timestamp is a scientific number such as 1.234e5
|
||||
decimalExp, ok := tryParseInt64(s[expIdx+1:])
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if ts >= (1 << 32) {
|
||||
// The timestamp is in milliseconds
|
||||
return int64(ts * 1_000_000), nil
|
||||
if decimalExp > math.MaxInt64 || decimalExp < math.MinInt64 {
|
||||
return 0, false
|
||||
}
|
||||
return int64(math.Round(ts*1_000)) * 1_000_000, nil
|
||||
n, ok := tryParseScientificNumberForUnixTimestamp(s[:expIdx], int(decimalExp))
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return getUnixTimestampNanoseconds(n), true
|
||||
}
|
||||
|
||||
// The timestamp is an integer number
|
||||
ts, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
dotIdx := strings.IndexByte(s, '.')
|
||||
if dotIdx < 0 {
|
||||
// The timestamp is integer.
|
||||
n, ok := tryParseInt64(s)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return getUnixTimestampNanoseconds(n), true
|
||||
}
|
||||
switch {
|
||||
case ts >= (1<<32)*1e6:
|
||||
// The timestamp is in nanoseconds
|
||||
return ts, nil
|
||||
case ts >= (1<<32)*1e3:
|
||||
// The timestamp is in microseconds
|
||||
return ts * 1_000, nil
|
||||
case ts >= (1 << 32):
|
||||
// The timestamp is in milliseconds
|
||||
return ts * 1_000_000, nil
|
||||
default:
|
||||
return ts * 1_000_000_000, nil
|
||||
|
||||
// The timestamp is fractional.
|
||||
intStr := s[:dotIdx]
|
||||
fracStr := s[dotIdx+1:]
|
||||
n, ok := tryParseFractionalNumberForUnixTimestamp(intStr, fracStr)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Adjust the n to multiples of thousands, since this is expected by getUnixTimestampNanoseconds.
|
||||
decimalExp := len(fracStr)
|
||||
for decimalExp%3 != 0 {
|
||||
if n >= 0 && n > math.MaxInt64/10 || n < 0 && n < math.MinInt64/10 {
|
||||
return 0, false
|
||||
}
|
||||
n *= 10
|
||||
decimalExp++
|
||||
}
|
||||
|
||||
return getUnixTimestampNanoseconds(n), true
|
||||
}
|
||||
|
||||
func getExpIndex(s string) int {
|
||||
if n := strings.IndexByte(s, 'e'); n >= 0 {
|
||||
return n
|
||||
}
|
||||
if n := strings.IndexByte(s, 'E'); n >= 0 {
|
||||
return n
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func tryParseScientificNumberForUnixTimestamp(s string, decimalExp int) (int64, bool) {
|
||||
dotIdx := strings.IndexByte(s, '.')
|
||||
if dotIdx < 0 {
|
||||
n, ok := tryParseInt64(s)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return multiplyByDecimalExp(n, decimalExp)
|
||||
}
|
||||
|
||||
intStr := s[:dotIdx]
|
||||
fracStr := s[dotIdx+1:]
|
||||
if decimalExp < len(fracStr) {
|
||||
return 0, false
|
||||
}
|
||||
n, ok := tryParseFractionalNumberForUnixTimestamp(intStr, fracStr)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
decimalExp -= len(fracStr)
|
||||
return multiplyByDecimalExp(n, decimalExp)
|
||||
}
|
||||
|
||||
func tryParseFractionalNumberForUnixTimestamp(intStr, fracStr string) (int64, bool) {
|
||||
n, ok := tryParseInt64(intStr)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
decimalExp := len(fracStr)
|
||||
num, ok := multiplyByDecimalExp(n, decimalExp)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
frac, ok := tryParseInt64(fracStr)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
if num >= 0 {
|
||||
if num > math.MaxInt64-frac {
|
||||
return 0, false
|
||||
}
|
||||
num += frac
|
||||
} else {
|
||||
if num < math.MinInt64+frac {
|
||||
return 0, false
|
||||
}
|
||||
num -= frac
|
||||
}
|
||||
|
||||
return num, true
|
||||
}
|
||||
|
||||
func multiplyByDecimalExp(n int64, decimalExp int) (int64, bool) {
|
||||
if decimalExp < 0 {
|
||||
return 0, false
|
||||
}
|
||||
if decimalExp >= len(decimalMultipliers) {
|
||||
return 0, false
|
||||
}
|
||||
if decimalExp == 0 {
|
||||
return n, true
|
||||
}
|
||||
|
||||
m := decimalMultipliers[decimalExp]
|
||||
|
||||
if n >= 0 && n > math.MaxInt64/m || n < 0 && n < math.MinInt64/m {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return n * m, true
|
||||
}
|
||||
|
||||
var decimalMultipliers = [...]int64{0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18}
|
||||
|
||||
func getUnixTimestampNanoseconds(n int64) int64 {
|
||||
if n < (1<<31) && n >= (-1<<31) {
|
||||
// The timestamp is in seconds.
|
||||
return n * 1e9
|
||||
}
|
||||
if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
|
||||
// The timestamp is in milliseconds.
|
||||
return n * 1e6
|
||||
}
|
||||
if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
|
||||
// The timestamp is in microseconds.
|
||||
return n * 1e3
|
||||
}
|
||||
// The timestamp is in nanoseconds
|
||||
return n
|
||||
}
|
||||
|
||||
func tryParseInt64(s string) (int64, bool) {
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return n, true
|
||||
}
|
||||
|
||||
@@ -5,6 +5,120 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTryParseUnixTimestamp_Success(t *testing.T) {
|
||||
f := func(s string, timestampExpected int64) {
|
||||
t.Helper()
|
||||
|
||||
timestamp, ok := TryParseUnixTimestamp(s)
|
||||
if !ok {
|
||||
t.Fatalf("cannot parse timestamp %q", s)
|
||||
}
|
||||
if timestamp != timestampExpected {
|
||||
t.Fatalf("unexpected timestamp returned from TryParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f("0", 0)
|
||||
|
||||
// nanoseconds
|
||||
f("-1234567890123456789", -1234567890123456789)
|
||||
f("1234567890123456789", 1234567890123456789)
|
||||
f("1234567890123456.789", 1234567890123456789)
|
||||
|
||||
// microseconds
|
||||
f("-1234567890123456", -1234567890123456000)
|
||||
f("1234567890123456", 1234567890123456000)
|
||||
f("1234567890123456.789", 1234567890123456789)
|
||||
|
||||
// milliseconds
|
||||
f("-1234567890123", -1234567890123000000)
|
||||
f("1234567890123", 1234567890123000000)
|
||||
f("1234567890123.456", 1234567890123456000)
|
||||
|
||||
// seconds
|
||||
f("-1234567890", -1234567890000000000)
|
||||
f("1234567890", 1234567890000000000)
|
||||
f("1234567890.123456789", 1234567890123456789)
|
||||
f("1234567890.12345678", 1234567890123456780)
|
||||
f("1234567890.1234567", 1234567890123456700)
|
||||
f("-1234567890.123456", -1234567890123456000)
|
||||
f("-1234567890.12345", -1234567890123450000)
|
||||
f("-1234567890.1234", -1234567890123400000)
|
||||
f("-1234567890.123", -1234567890123000000)
|
||||
f("-1234567890.12", -1234567890120000000)
|
||||
f("-1234567890.1", -1234567890100000000)
|
||||
|
||||
// scientific notation
|
||||
f("1e9", 1000000000000000000)
|
||||
f("1.234e9", 1234000000000000000)
|
||||
f("-1.23456789e9", -1234567890000000000)
|
||||
f("1.234567890123456789e18", 1234567890123456789)
|
||||
f("-1.234567890123456789e18", -1234567890123456789)
|
||||
f("0.23456789e9", 234567890000000000)
|
||||
f("123.456789123e9", 123456789123000000)
|
||||
f("-1234.5678912e9", -1234567891200000000)
|
||||
f("123.678912e7", 1236789120000000000)
|
||||
f("1.23e7", 12300000000000000)
|
||||
f("1.23e6", 1230000000000000)
|
||||
f("1.23e5", 123000000000000)
|
||||
f("1.23e4", 12300000000000)
|
||||
f("1.23e3", 1230000000000)
|
||||
f("1.23e2", 123000000000)
|
||||
f("1.2e1", 12000000000)
|
||||
f("1123.456789123456789E15", 1123456789123456789)
|
||||
}
|
||||
|
||||
func TestTryParseUnixTimestamp_Failure(t *testing.T) {
|
||||
f := func(s string) {
|
||||
t.Helper()
|
||||
|
||||
_, ok := TryParseUnixTimestamp(s)
|
||||
if ok {
|
||||
t.Fatalf("expecting failure when parsing %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
// non-numeric timestamp
|
||||
f("")
|
||||
f("foobar")
|
||||
f("foo.bar")
|
||||
f("1.12345671x34")
|
||||
f("1.3e12345678x0123")
|
||||
f("1xs.12345671")
|
||||
f("1xs.12345671e5")
|
||||
f("-1xs.12345671e5")
|
||||
|
||||
// missing fractional part
|
||||
f("1233344.")
|
||||
|
||||
// too big timestamp
|
||||
f("12345678901234567.891")
|
||||
f("12345678901234567890")
|
||||
f("12345678901234.567891")
|
||||
f("12345678901234567890e3")
|
||||
f("12345678901234567890.234e3")
|
||||
f("-12345678901234567890")
|
||||
f("12345678901234567890.235424")
|
||||
f("12345678901234567890.235424e3")
|
||||
f("-12345678901234567890.235424")
|
||||
f("12345678901234567.89")
|
||||
f("12345678901234567.8")
|
||||
|
||||
// too big fractional part
|
||||
f("0.1234567890123456789123")
|
||||
f("-0.1234567890123456789123")
|
||||
|
||||
// too big decimal exponent
|
||||
f("1e19")
|
||||
f("1.3e123456789090123")
|
||||
|
||||
// too small decimal exponent
|
||||
f("1.23e1")
|
||||
f("1.234e0")
|
||||
f("1E-1")
|
||||
f("1.3e-123456789090123")
|
||||
}
|
||||
|
||||
func TestParseTimeAtSuccess(t *testing.T) {
|
||||
f := func(s string, currentTime, resultExpected int64) {
|
||||
t.Helper()
|
||||
@@ -21,13 +135,23 @@ func TestParseTimeAtSuccess(t *testing.T) {
|
||||
|
||||
// unix timestamp in seconds
|
||||
f("1562529662", now, 1562529662*1e9)
|
||||
f("1562529662.6", now, 1562529662600*1e6)
|
||||
f("1562529662.67", now, 1562529662670*1e6)
|
||||
f("1562529662.678", now, 1562529662678*1e6)
|
||||
f("1562529662.678123", now, 1562529662678123*1e3)
|
||||
f("1562529662.678123456", now, 1562529662678123456)
|
||||
|
||||
// unix timestamp in milliseconds
|
||||
f("1562529662678", now, 1562529662678*1e6)
|
||||
f("1562529662678.9", now, 1562529662678900*1e3)
|
||||
f("1562529662678.901", now, 1562529662678901*1e3)
|
||||
f("1562529662678.901324", now, 1562529662678901324)
|
||||
|
||||
// unix timestamp in microseconds
|
||||
f("1562529662678901", now, 1562529662678901*1e3)
|
||||
f("1562529662678901.3", now, 1562529662678901300)
|
||||
f("1562529662678901.32", now, 1562529662678901320)
|
||||
f("1562529662678901.321", now, 1562529662678901321)
|
||||
|
||||
// unix timestamp in nanoseconds
|
||||
f("1562529662678901234", now, 1562529662678901234)
|
||||
|
||||
153
lib/timeutil/time_timing_test.go
Normal file
153
lib/timeutil/time_timing_test.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package timeutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkTryParseUnixTimestamp(b *testing.B) {
|
||||
b.Run("seconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890",
|
||||
"1234567891",
|
||||
"1234567892",
|
||||
"1234567893",
|
||||
"1234567894",
|
||||
"1234567895",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("milliseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890123",
|
||||
"1234567891123",
|
||||
"1234567892123",
|
||||
"1234567893123",
|
||||
"1234567894123",
|
||||
"1234567895123",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("microseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890123456",
|
||||
"1234567891123456",
|
||||
"1234567892123456",
|
||||
"1234567893123456",
|
||||
"1234567894123456",
|
||||
"1234567895123456",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("nanoseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890123456789",
|
||||
"1234567891123456789",
|
||||
"1234567892123456789",
|
||||
"1234567893123456789",
|
||||
"1234567894123456789",
|
||||
"1234567895123456789",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
|
||||
b.Run("scientific-seconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1.234567890e9",
|
||||
"1.234567891e9",
|
||||
"1.234567892e9",
|
||||
"1.234567893e9",
|
||||
"1.234567894e9",
|
||||
"1.234567895e9",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("scientific-milliseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1.234567890123e12",
|
||||
"1.234567891123e12",
|
||||
"1.234567892123e12",
|
||||
"1.234567893123e12",
|
||||
"1.234567894123e12",
|
||||
"1.234567895123e12",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("scientific-microseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1.234567890123456e15",
|
||||
"1.234567891123456e15",
|
||||
"1.234567892123456e15",
|
||||
"1.234567893123456e15",
|
||||
"1.234567894123456e15",
|
||||
"1.234567895123456e15",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("scientific-nanoseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1.234567890123456789e18",
|
||||
"1.234567891123456789e18",
|
||||
"1.234567892123456789e18",
|
||||
"1.234567893123456789e18",
|
||||
"1.234567894123456789e18",
|
||||
"1.234567895123456789e18",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
|
||||
b.Run("fractional-milliseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890.123",
|
||||
"1234567891.123",
|
||||
"1234567892.123",
|
||||
"1234567893.123",
|
||||
"1234567894.123",
|
||||
"1234567895.123",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("fractional-microseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890.123456",
|
||||
"1234567891.123456",
|
||||
"1234567892.123456",
|
||||
"1234567893.123456",
|
||||
"1234567894.123456",
|
||||
"1234567895.123456",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
b.Run("fractional-nanoseconds", func(b *testing.B) {
|
||||
a := []string{
|
||||
"1234567890.123456789",
|
||||
"1234567891.123456789",
|
||||
"1234567892.123456789",
|
||||
"1234567893.123456789",
|
||||
"1234567894.123456789",
|
||||
"1234567895.123456789",
|
||||
}
|
||||
benchmarkTryParseUnixTimestamp(b, a)
|
||||
})
|
||||
}
|
||||
|
||||
func benchmarkTryParseUnixTimestamp(b *testing.B, a []string) {
|
||||
b.SetBytes(int64(len(a)))
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
nSum := int64(0)
|
||||
for pb.Next() {
|
||||
for _, s := range a {
|
||||
n, ok := TryParseUnixTimestamp(s)
|
||||
if !ok {
|
||||
panic(fmt.Errorf("cannot parse timestamp %q", s))
|
||||
}
|
||||
nSum += n
|
||||
}
|
||||
}
|
||||
GlobalSink.Add(uint64(nSum))
|
||||
})
|
||||
}
|
||||
|
||||
var GlobalSink atomic.Uint64
|
||||
Reference in New Issue
Block a user