Files
VictoriaMetrics/lib/logstorage/values_encoder_test.go
Aliaksandr Valialkin df723a4870 lib/logstorage: automatically detect columns with int64 values and store them as packed 8-byte int64 values
Previously columns with negative int64 values were stored either as float64 or string
depending on whether the negative int64 values are bigger or smaller than -2^53.
If the integer values are smaller than -2^53, then they are stored as string, since float64 cannot
hold such values without precision loss. Now such values are stored as int64.
This should improve compression ratio and query performance over columns with negative int64 values.
2025-01-12 03:01:46 +01:00

903 lines
19 KiB
Go

package logstorage
import (
"fmt"
"math"
"reflect"
"strconv"
"testing"
)
func TestValuesEncoder(t *testing.T) {
f := func(values []string, expectedValueType valueType, expectedMinValue, expectedMaxValue uint64) {
t.Helper()
ve := getValuesEncoder()
var dict valuesDict
vt, minValue, maxValue := ve.encode(values, &dict)
if vt != expectedValueType {
t.Fatalf("unexpected value type; got %d; want %d", vt, expectedValueType)
}
if minValue != expectedMinValue {
t.Fatalf("unexpected minValue; got %d; want %d", minValue, expectedMinValue)
}
if maxValue != expectedMaxValue {
t.Fatalf("unexpected maxValue; got %d; want %d", maxValue, expectedMaxValue)
}
encodedValues := append([]string{}, ve.values...)
putValuesEncoder(ve)
vd := getValuesDecoder()
if err := vd.decodeInplace(encodedValues, vt, dict.values); err != nil {
t.Fatalf("unexpected error in decodeInplace(): %s", err)
}
if len(values) == 0 {
values = []string{}
}
if !reflect.DeepEqual(values, encodedValues) {
t.Fatalf("unexpected values decoded\ngot\n%q\nwant\n%q", encodedValues, values)
}
putValuesDecoder(vd)
}
// An empty values list
f(nil, valueTypeString, 0, 0)
// string values
values := make([]string, maxDictLen+1)
for i := range values {
values[i] = fmt.Sprintf("value_%d", i)
}
f(values, valueTypeString, 0, 0)
// dict values
f([]string{"foobar"}, valueTypeDict, 0, 0)
f([]string{"foo", "bar"}, valueTypeDict, 0, 0)
f([]string{"1", "2foo"}, valueTypeDict, 0, 0)
// uint8 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1))
}
f(values, valueTypeUint8, 1, uint64(len(values)))
// uint16 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<8)
}
f(values, valueTypeUint16, 1<<8, uint64(len(values))<<8)
// uint32 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<16)
}
f(values, valueTypeUint32, 1<<16, uint64(len(values))<<16)
// uint64 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<32)
}
f(values, valueTypeUint64, 1<<32, uint64(len(values))<<32)
// float64 values
for i := range values {
values[i] = fmt.Sprintf("%g", math.Sqrt(float64(i+1)))
}
f(values, valueTypeFloat64, 4607182418800017408, 4613937818241073152)
// ipv4 values
for i := range values {
values[i] = fmt.Sprintf("1.2.3.%d", i)
}
f(values, valueTypeIPv4, 16909056, 16909064)
// iso8601 timestamps
for i := range values {
values[i] = fmt.Sprintf("2011-04-19T03:44:01.%03dZ", i)
}
f(values, valueTypeTimestampISO8601, 1303184641000000000, 1303184641008000000)
}
func TestTryParseIPv4String_Success(t *testing.T) {
f := func(s string) {
t.Helper()
n, ok := tryParseIPv4(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
data := marshalIPv4String(nil, n)
if string(data) != s {
t.Fatalf("unexpected ip; got %q; want %q", data, s)
}
}
f("0.0.0.0")
f("1.2.3.4")
f("255.255.255.255")
f("127.0.0.1")
}
func TestTryParseIPv4_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseIPv4(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
f("")
f("foo")
f("a.b.c.d")
f("127.0.0.x")
f("127.0.x.0")
f("127.x.0.0")
f("x.0.0.0")
// Too big octets
f("127.127.127.256")
f("127.127.256.127")
f("127.256.127.127")
f("256.127.127.127")
// Negative octets
f("-1.127.127.127")
f("127.-1.127.127")
f("127.127.-1.127")
f("127.127.127.-1")
}
func TestTryParseTimestampRFC3339NanoString_Success(t *testing.T) {
f := func(s, timestampExpected string) {
t.Helper()
nsecs, ok := TryParseTimestampRFC3339Nano(s)
if !ok {
t.Fatalf("cannot parse timestamp %q", s)
}
timestamp := marshalTimestampRFC3339NanoString(nil, nsecs)
if string(timestamp) != timestampExpected {
t.Fatalf("unexpected timestamp; got %q; want %q", timestamp, timestampExpected)
}
}
// No fractional seconds
f("2023-01-15T23:45:51Z", "2023-01-15T23:45:51Z")
// Different number of fractional seconds
f("2023-01-15T23:45:51.1Z", "2023-01-15T23:45:51.1Z")
f("2023-01-15T23:45:51.12Z", "2023-01-15T23:45:51.12Z")
f("2023-01-15T23:45:51.123Z", "2023-01-15T23:45:51.123Z")
f("2023-01-15T23:45:51.1234Z", "2023-01-15T23:45:51.1234Z")
f("2023-01-15T23:45:51.12345Z", "2023-01-15T23:45:51.12345Z")
f("2023-01-15T23:45:51.123456Z", "2023-01-15T23:45:51.123456Z")
f("2023-01-15T23:45:51.1234567Z", "2023-01-15T23:45:51.1234567Z")
f("2023-01-15T23:45:51.12345678Z", "2023-01-15T23:45:51.12345678Z")
f("2023-01-15T23:45:51.123456789Z", "2023-01-15T23:45:51.123456789Z")
// The minimum possible timestamp
f("1677-09-21T00:12:44Z", "1677-09-21T00:12:44Z")
// The maximum possible timestamp
f("2262-04-11T23:47:15.999999999Z", "2262-04-11T23:47:15.999999999Z")
// timestamp with timezone
f("2023-01-16T00:45:51+01:00", "2023-01-15T23:45:51Z")
f("2023-01-16T00:45:51.123-01:00", "2023-01-16T01:45:51.123Z")
// SQL datetime format
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6721
f("2023-01-16 00:45:51+01:00", "2023-01-15T23:45:51Z")
f("2023-01-16 00:45:51.123-01:00", "2023-01-16T01:45:51.123Z")
}
func TestTryParseTimestampRFC3339Nano_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := TryParseTimestampRFC3339Nano(s)
if ok {
t.Fatalf("expecting faulure when parsing %q", s)
}
}
// invalid length
f("")
f("foobar")
// missing fractional part after dot
f("2023-01-15T22:15:51.Z")
// too small year
f("1676-09-21T00:12:43Z")
// too big year
f("2263-04-11T23:47:17Z")
// too small timestamp
f("1677-09-21T00:12:43.999999999Z")
// too big timestamp
f("2262-04-11T23:47:16Z")
// invalid year
f("YYYY-04-11T23:47:17Z")
// invalid moth
f("2023-MM-11T23:47:17Z")
// invalid day
f("2023-01-DDT23:47:17Z")
// invalid hour
f("2023-01-23Thh:47:17Z")
// invalid minute
f("2023-01-23T23:mm:17Z")
// invalid second
f("2023-01-23T23:33:ssZ")
}
func TestTryParseTimestampISO8601String_Success(t *testing.T) {
f := func(s string) {
t.Helper()
nsecs, ok := tryParseTimestampISO8601(s)
if !ok {
t.Fatalf("cannot parse timestamp %q", s)
}
data := marshalTimestampISO8601String(nil, nsecs)
if string(data) != s {
t.Fatalf("unexpected timestamp; got %q; want %q", data, s)
}
}
// regular timestamp
f("2023-01-15T23:45:51.123Z")
// The minimum possible timestamp
f("1677-09-21T00:12:44.000Z")
// The maximum possible timestamp
f("2262-04-11T23:47:15.999Z")
}
func TestTryParseTimestampISO8601_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseTimestampISO8601(s)
if ok {
t.Fatalf("expecting faulure when parsing %q", s)
}
}
// invalid length
f("")
f("foobar")
// Missing Z at the end
f("2023-01-15T22:15:51.123")
f("2023-01-15T22:15:51.1234")
// timestamp with timezone
f("2023-01-16T00:45:51.123+01:00")
// too small year
f("1676-09-21T00:12:43.434Z")
// too big year
f("2263-04-11T23:47:17.434Z")
// too small timestamp
f("1677-09-21T00:12:43.999Z")
// too big timestamp
f("2262-04-11T23:47:16.000Z")
// invalid year
f("YYYY-04-11T23:47:17.123Z")
// invalid moth
f("2023-MM-11T23:47:17.123Z")
// invalid day
f("2023-01-DDT23:47:17.123Z")
// invalid hour
f("2023-01-23Thh:47:17.123Z")
// invalid minute
f("2023-01-23T23:mm:17.123Z")
// invalid second
f("2023-01-23T23:33:ss.123Z")
}
func TestTryParseDuration_Success(t *testing.T) {
f := func(s string, nsecsExpected int64) {
t.Helper()
nsecs, ok := tryParseDuration(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if nsecs != nsecsExpected {
t.Fatalf("unexpected value; got %d; want %d", nsecs, nsecsExpected)
}
}
// zero duration
f("0s", 0)
f("0.0w0d0h0s0.0ms", 0)
f("-0w", 0)
// positive duration
f("1s", nsecsPerSecond)
f("1.5ms", 1.5*nsecsPerMillisecond)
f("1µs", nsecsPerMicrosecond)
f("1ns", 1)
f("1h", nsecsPerHour)
f("1.5d", 1.5*nsecsPerDay)
f("1.5w", 1.5*nsecsPerWeek)
f("2.5y", 2.5*nsecsPerYear)
f("1m5.123456789s", nsecsPerMinute+5.123456789*nsecsPerSecond)
// composite duration
f("1h5m", nsecsPerHour+5*nsecsPerMinute)
f("1.1h5m2.5s3_456ns", 1.1*nsecsPerHour+5*nsecsPerMinute+2.5*nsecsPerSecond+3456)
// nedgative duration
f("-1h5m3s", -(nsecsPerHour + 5*nsecsPerMinute + 3*nsecsPerSecond))
// max int duration
f("9_223_372_036_854_775_807ns", 1<<63-1)
f("9223372036854775807ns", 1<<63-1)
f("-9223372036854775808ns", -1<<63+1)
// too big value is clapped to 1<<63-1
f("15_223_372_036_854_775_808ns", 1<<63-1)
f("-15_223_372_036_854_775_808ns", -1<<63+1)
}
func TestTryParseDuration_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseDuration(s)
if ok {
t.Fatalf("expecting error for parsing %q", s)
}
}
// empty string
f("")
// missing suffix
f("2")
f("2.5")
// invalid string
f("foobar")
f("1foo")
f("1soo")
f("3.43e")
f("3.43es")
// superflouous space
f(" 2s")
f("2s ")
f("2s 3ms")
}
func TestMarshalDurationString(t *testing.T) {
f := func(nsecs int64, resultExpected string) {
t.Helper()
result := marshalDurationString(nil, nsecs)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1ns")
f(-1, "-1ns")
f(12345, "12µs345ns")
f(123456789, "123ms456µs789ns")
f(12345678901, "12.345678901s")
f(1234567890143, "20m34.567890143s")
f(1234567890123457, "2w6h56m7.890123457s")
}
func TestTryParseBytes_Success(t *testing.T) {
f := func(s string, resultExpected int64) {
t.Helper()
result, ok := tryParseBytes(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected result; got %d; want %d", result, resultExpected)
}
}
f("1_500", 1_500)
f("2.5B", 2)
f("1.5K", 1_500)
f("1.5M", 1_500_000)
f("1.5G", 1_500_000_000)
f("1.5T", 1_500_000_000_000)
f("1.5KB", 1_500)
f("1.5MB", 1_500_000)
f("1.5GB", 1_500_000_000)
f("1.5TB", 1_500_000_000_000)
f("1.5Ki", 1.5*(1<<10))
f("1.5Mi", 1.5*(1<<20))
f("1.5Gi", 1.5*(1<<30))
f("1.5Ti", 1.5*(1<<40))
f("1.5KiB", 1.5*(1<<10))
f("1.5MiB", 1.5*(1<<20))
f("1.5GiB", 1.5*(1<<30))
f("1.5TiB", 1.5*(1<<40))
f("1MiB500KiB200B", (1<<20)+500*(1<<10)+200)
// The maximum bytes value
f("9_223_372_036_854_775_807", 1<<63-1)
f("9223372036854775807B", 1<<63-1)
f("-9223372036854775808B", -1<<63+1)
// too big value is clapped to 1<<63-1
f("15_223_372_036_854_775_808", 1<<63-1)
f("-15_223_372_036_854_775_808", -1<<63+1)
}
func TestTryParseBytes_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseBytes(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// empty string
f("")
// invalid number
f("foobar")
// invalid suffix
f("123q")
f("123qs")
f("123qsb")
f("123sqsb")
f("123s5qsb")
// invalid case for the suffix
f("1b")
f("1k")
f("1m")
f("1g")
f("1t")
f("1kb")
f("1mb")
f("1gb")
f("1tb")
f("1ki")
f("1mi")
f("1gi")
f("1ti")
f("1kib")
f("1mib")
f("1gib")
f("1tib")
f("1KIB")
f("1MIB")
f("1GIB")
f("1TIB")
// fractional number without suffix
f("123.456")
}
func TestTryParseFloat64_Success(t *testing.T) {
f := func(s string, resultExpected float64) {
t.Helper()
result, ok := tryParseFloat64(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if !float64Equal(result, resultExpected) {
t.Fatalf("unexpected value; got %f; want %f", result, resultExpected)
}
}
f("0", 0)
f("1", 1)
f("-1", -1)
f("1234567890", 1234567890)
f("1_234_567_890", 1234567890)
f("-1.234_567", -1.234567)
f("0.345", 0.345)
f("-0.345", -0.345)
// The maximum integer
f("9007199254740991", (1<<53)-1)
f("9_007_199_254_740_991", (1<<53)-1)
f("-9007199254740991", (-1<<53)+1)
// Too big integer (exceeds 2^53-1). It leads to precision loss.
f("9_007_199_254_740_992", 9007199254740992)
f("10_007_199_254_740_992", 10007199254740993)
f("-9007199254740992", -9007199254740992)
}
func float64Equal(a, b float64) bool {
return math.Abs(a-b)*math.Abs(max(a, b)) < 1e-15
}
func TestTryParseFloat64_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseFloat64(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// Empty value
f("")
// Plus in the value isn't allowed, since it cannot be convered back to the same string representation
f("+123")
// Dot at the beginning and the end of value isn't allowed, since it cannot converted back to the same string representation
f(".123")
f("123.")
// Multiple dots aren't allowed
f("123.434.55")
// Invalid dots
f("-.123")
f(".")
// Scientific notation isn't allowed, since it cannot be converted back to the same string representation
f("12e5")
// Minus in the middle of string isn't allowed
f("12-5")
}
func TestTryParseFloat64Exact_Success(t *testing.T) {
f := func(s string, resultExpected float64) {
t.Helper()
result, ok := tryParseFloat64Exact(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if !float64Equal(result, resultExpected) {
t.Fatalf("unexpected value; got %f; want %f", result, resultExpected)
}
}
f("0", 0)
f("1", 1)
f("-1", -1)
f("1234567890", 1234567890)
f("1_234_567_890", 1234567890)
f("-1.234_567", -1.234567)
f("0.345", 0.345)
f("-0.345", -0.345)
// The maximum integer
f("9007199254740991", (1<<53)-1)
f("9_007_199_254_740_991", (1<<53)-1)
f("-9007199254740991", (-1<<53)+1)
}
func TestTryParseFloat64Exact_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseFloat64Exact(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// Empty value
f("")
// Plus in the value isn't allowed, since it cannot be convered back to the same string representation
f("+123")
// Dot at the beginning and the end of value isn't allowed, since it cannot converted back to the same string representation
f(".123")
f("123.")
// Multiple dots aren't allowed
f("123.434.55")
// Invalid dots
f("-.123")
f(".")
// Scientific notation isn't allowed, since it cannot be converted back to the same string representation
f("12e5")
// Minus in the middle of string isn't allowed
f("12-5")
// Too big integer (exceeds 2^53-1)
f("9_007_199_254_740_992")
f("-9007199254740992")
}
func TestMarshalFloat64String(t *testing.T) {
f := func(f float64, resultExpected string) {
t.Helper()
result := marshalFloat64String(nil, f)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1234, "1234")
f(-12345678, "-12345678")
f(1.234, "1.234")
f(-1.234567, "-1.234567")
}
func TestTryParseUint64_Success(t *testing.T) {
f := func(s string, resultExpected uint64) {
t.Helper()
result, ok := tryParseUint64(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected value; got %d; want %d", result, resultExpected)
}
}
f("0", 0)
f("123", 123)
f("123456", 123456)
f("123456789", 123456789)
f("123456789012", 123456789012)
f("123456789012345", 123456789012345)
f("123456789012345678", 123456789012345678)
f("12345678901234567890", 12345678901234567890)
f("12_345_678_901_234_567_890", 12345678901234567890)
// the maximum possible value
f("18446744073709551615", 18446744073709551615)
}
func TestTryParseUint64_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseUint64(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// empty value
f("")
// too big value
f("18446744073709551616")
// invalid value
f("foo")
f("1.2")
f("1e3")
}
func TestTryParseInt64_Success(t *testing.T) {
f := func(s string, resultExpected int64) {
t.Helper()
result, ok := tryParseInt64(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected value; got %d; want %d", result, resultExpected)
}
}
f("0", 0)
f("-0", 0)
f("123", 123)
f("-123", -123)
f("1345678901234567890", 1345678901234567890)
f("-1_345_678_901_234_567_890", -1345678901234567890)
// the maximum possible value
f("9223372036854775807", 9223372036854775807)
// the minimum possible value
f("-9223372036854775808", -9223372036854775808)
}
func TestTryParseInt64_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseInt64(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// empty value
f("")
// too big value
f("9223372036854775808")
// too small value
f("-9223372036854775809")
// invalid value
f("foo")
f("1.2")
f("1e3")
}
func TestMarshalUint8String(t *testing.T) {
f := func(n uint8, resultExpected string) {
t.Helper()
result := marshalUint8String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
for i := 0; i < 256; i++ {
resultExpected := strconv.Itoa(i)
f(uint8(i), resultExpected)
}
// the maximum possible value
f(math.MaxUint8, "255")
}
func TestMarshalUint16String(t *testing.T) {
f := func(n uint16, resultExpected string) {
t.Helper()
result := marshalUint16String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1")
f(10, "10")
f(12, "12")
f(120, "120")
f(1203, "1203")
f(12345, "12345")
// the maximum possible value
f(math.MaxUint16, "65535")
}
func TestMarshalUint32String(t *testing.T) {
f := func(n uint32, resultExpected string) {
t.Helper()
result := marshalUint32String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1")
f(10, "10")
f(12, "12")
f(120, "120")
f(1203, "1203")
f(12034, "12034")
f(123456, "123456")
f(1234567, "1234567")
f(12345678, "12345678")
f(123456789, "123456789")
f(1234567890, "1234567890")
// the maximum possible value
f(math.MaxUint32, "4294967295")
}
func TestMarshalUint64String(t *testing.T) {
f := func(n uint64, resultExpected string) {
t.Helper()
result := marshalUint64String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(123456, "123456")
// the maximum possible value
f(math.MaxUint64, "18446744073709551615")
}
func TestTryParseIPv4Mask_Success(t *testing.T) {
f := func(s string, resultExpected uint64) {
t.Helper()
result, ok := tryParseIPv4Mask(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected result; got %d; want %d", result, resultExpected)
}
}
f("/0", 1<<32)
f("/1", 1<<31)
f("/8", 1<<24)
f("/24", 1<<8)
f("/32", 1)
}
func TestTryParseIPv4Mask_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseIPv4Mask(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// Empty mask
f("")
// Invalid prefix
f("foo")
// Non-numeric mask
f("/foo")
// Too big mask
f("/33")
// Negative mask
f("/-1")
}