rms/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2026-05-17 08:36:55 +03:00
Code Issues Packages Projects Releases Wiki Activity

all: add Pure Go build (pull request #114)

Browse Source 
Updates #94
This commit is contained in:
Roman Khavronenko
2019-07-23 17:26:39 +01:00
committed by Aliaksandr Valialkin
parent 8c3629a892
commit 5bf4e5ffb5
77 changed files with 14430 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/encoding/compress.go
View File

@@ -1,8 +1,8 @@
package encoding
import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
"github.com/VictoriaMetrics/metrics"
"github.com/valyala/gozstd"
)
// CompressZSTDLevel appends compressed src to dst and returns
@@ -13,7 +13,7 @@ func CompressZSTDLevel(dst, src []byte, compressLevel int) []byte {
compressCalls.Inc()
originalBytes.Add(len(src))
dstLen := len(dst)
dst = gozstd.CompressLevel(dst, src, compressLevel)
dst = zstd.CompressLevel(dst, src, compressLevel)
compressedBytes.Add(len(dst) - dstLen)
return dst
}
@@ -22,7 +22,7 @@ func CompressZSTDLevel(dst, src []byte, compressLevel int) []byte {
// the appended dst.
func DecompressZSTD(dst, src []byte) ([]byte, error) {
decompressCalls.Inc()
return gozstd.Decompress(dst, src)
return zstd.Decompress(dst, src)
}
var (

83
lib/encoding/encoding_cgo_test.go Normal file
View File

@@ -0,0 +1,83 @@
// +build cgo
package encoding
import (
"math/rand"
"testing"
)
func TestMarshalUnmarshalInt64Array(t *testing.T) {
var va []int64
var v int64
// Verify nearest delta encoding.
va = va[:0]
v = 0
for i := 0; i < 8*1024; i++ {
v += int64(rand.NormFloat64() * 1e6)
va = append(va, v)
}
for precisionBits := uint8(1); precisionBits < 23; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeZSTDNearestDelta)
}
for precisionBits := uint8(23); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta)
}
// Verify nearest delta2 encoding.
va = va[:0]
v = 0
for i := 0; i < 8*1024; i++ {
v += 30e6 + int64(rand.NormFloat64()*1e6)
va = append(va, v)
}
for precisionBits := uint8(1); precisionBits < 24; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeZSTDNearestDelta2)
}
for precisionBits := uint8(24); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta2)
}
// Verify nearest delta encoding.
va = va[:0]
v = 1000
for i := 0; i < 6; i++ {
v += int64(rand.NormFloat64() * 100)
va = append(va, v)
}
for precisionBits := uint8(1); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta)
}
// Verify nearest delta2 encoding.
va = va[:0]
v = 0
for i := 0; i < 6; i++ {
v += 3000 + int64(rand.NormFloat64()*100)
va = append(va, v)
}
for precisionBits := uint8(5); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta2)
}
}
func TestMarshalInt64ArraySize(t *testing.T) {
var va []int64
v := int64(rand.Float64() * 1e9)
for i := 0; i < 8*1024; i++ {
va = append(va, v)
v += 30e3 + int64(rand.NormFloat64()*1e3)
}
testMarshalInt64ArraySize(t, va, 1, 500, 1300)
testMarshalInt64ArraySize(t, va, 2, 600, 1400)
testMarshalInt64ArraySize(t, va, 3, 900, 1800)
testMarshalInt64ArraySize(t, va, 4, 1300, 2100)
testMarshalInt64ArraySize(t, va, 5, 2000, 3200)
testMarshalInt64ArraySize(t, va, 6, 3000, 4800)
testMarshalInt64ArraySize(t, va, 7, 4000, 6400)
testMarshalInt64ArraySize(t, va, 8, 6000, 8000)
testMarshalInt64ArraySize(t, va, 9, 7000, 8800)
testMarshalInt64ArraySize(t, va, 10, 8000, 10000)
}

83
lib/encoding/encoding_pure_test.go Normal file
View File

@@ -0,0 +1,83 @@
// +build !cgo
package encoding
import (
"math/rand"
"testing"
)
func TestMarshalUnmarshalInt64Array(t *testing.T) {
var va []int64
var v int64
// Verify nearest delta encoding.
va = va[:0]
v = 0
for i := 0; i < 8*1024; i++ {
v += int64(rand.NormFloat64() * 1e6)
va = append(va, v)
}
for precisionBits := uint8(1); precisionBits < 17; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeZSTDNearestDelta)
}
for precisionBits := uint8(23); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta)
}
// Verify nearest delta2 encoding.
va = va[:0]
v = 0
for i := 0; i < 8*1024; i++ {
v += 30e6 + int64(rand.NormFloat64()*1e6)
va = append(va, v)
}
for precisionBits := uint8(1); precisionBits < 15; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeZSTDNearestDelta2)
}
for precisionBits := uint8(24); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta2)
}
// Verify nearest delta encoding.
va = va[:0]
v = 1000
for i := 0; i < 6; i++ {
v += int64(rand.NormFloat64() * 100)
va = append(va, v)
}
for precisionBits := uint8(1); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta)
}
// Verify nearest delta2 encoding.
va = va[:0]
v = 0
for i := 0; i < 6; i++ {
v += 3000 + int64(rand.NormFloat64()*100)
va = append(va, v)
}
for precisionBits := uint8(5); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta2)
}
}
func TestMarshalInt64ArraySize(t *testing.T) {
var va []int64
v := int64(rand.Float64() * 1e9)
for i := 0; i < 8*1024; i++ {
va = append(va, v)
v += 30e3 + int64(rand.NormFloat64()*1e3)
}
testMarshalInt64ArraySize(t, va, 1, 500, 1700)
testMarshalInt64ArraySize(t, va, 2, 600, 1800)
testMarshalInt64ArraySize(t, va, 3, 900, 2100)
testMarshalInt64ArraySize(t, va, 4, 1300, 2200)
testMarshalInt64ArraySize(t, va, 5, 2000, 3300)
testMarshalInt64ArraySize(t, va, 6, 3000, 5000)
testMarshalInt64ArraySize(t, va, 7, 4000, 6500)
testMarshalInt64ArraySize(t, va, 8, 6000, 8000)
testMarshalInt64ArraySize(t, va, 9, 7000, 8800)
testMarshalInt64ArraySize(t, va, 10, 8000, 17000)
}

95
lib/encoding/encoding_test.go
View File

@@ -89,73 +89,6 @@ func testEnsureNonDecreasingSequence(t *testing.T, a []int64, vMin, vMax int64,
}
}
func TestMarshalUnmarshalInt64Array(t *testing.T) {
testMarshalUnmarshalInt64Array(t, []int64{1, 20, 234}, 4, MarshalTypeNearestDelta2)
testMarshalUnmarshalInt64Array(t, []int64{1, 20, -2345, 678934, 342}, 4, MarshalTypeNearestDelta)
testMarshalUnmarshalInt64Array(t, []int64{1, 20, 2345, 6789, 12342}, 4, MarshalTypeNearestDelta2)
// Constant encoding
testMarshalUnmarshalInt64Array(t, []int64{1}, 4, MarshalTypeConst)
testMarshalUnmarshalInt64Array(t, []int64{1, 2}, 4, MarshalTypeDeltaConst)
testMarshalUnmarshalInt64Array(t, []int64{-1, 0, 1, 2, 3, 4, 5}, 4, MarshalTypeDeltaConst)
testMarshalUnmarshalInt64Array(t, []int64{-10, -1, 8, 17, 26}, 4, MarshalTypeDeltaConst)
testMarshalUnmarshalInt64Array(t, []int64{0, 0, 0, 0, 0, 0}, 4, MarshalTypeConst)
testMarshalUnmarshalInt64Array(t, []int64{100, 100, 100, 100}, 4, MarshalTypeConst)
var va []int64
var v int64
// Verify nearest delta encoding.
va = va[:0]
v = 0
for i := 0; i < 8*1024; i++ {
v += int64(rand.NormFloat64() * 1e6)
va = append(va, v)
}
for precisionBits := uint8(1); precisionBits < 23; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeZSTDNearestDelta)
}
for precisionBits := uint8(23); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta)
}
// Verify nearest delta2 encoding.
va = va[:0]
v = 0
for i := 0; i < 8*1024; i++ {
v += 30e6 + int64(rand.NormFloat64()*1e6)
va = append(va, v)
}
for precisionBits := uint8(1); precisionBits < 24; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeZSTDNearestDelta2)
}
for precisionBits := uint8(24); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta2)
}
// Verify nearest delta encoding.
va = va[:0]
v = 1000
for i := 0; i < 6; i++ {
v += int64(rand.NormFloat64() * 100)
va = append(va, v)
}
for precisionBits := uint8(1); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta)
}
// Verify nearest delta2 encoding.
va = va[:0]
v = 0
for i := 0; i < 6; i++ {
v += 3000 + int64(rand.NormFloat64()*100)
va = append(va, v)
}
for precisionBits := uint8(5); precisionBits < 65; precisionBits++ {
testMarshalUnmarshalInt64Array(t, va, precisionBits, MarshalTypeNearestDelta2)
}
}
func testMarshalUnmarshalInt64Array(t *testing.T, va []int64, precisionBits uint8, mtExpected MarshalType) {
t.Helper()
@@ -259,24 +192,18 @@ func TestMarshalUnmarshalValues(t *testing.T) {
}
}
func TestMarshalInt64ArraySize(t *testing.T) {
var va []int64
v := int64(rand.Float64() * 1e9)
for i := 0; i < 8*1024; i++ {
va = append(va, v)
v += 30e3 + int64(rand.NormFloat64()*1e3)
}
func TestMarshalUnmarshalInt64ArrayConstant(t *testing.T) {
testMarshalUnmarshalInt64Array(t, []int64{1, 20, 234}, 4, MarshalTypeNearestDelta2)
testMarshalUnmarshalInt64Array(t, []int64{1, 20, -2345, 678934, 342}, 4, MarshalTypeNearestDelta)
testMarshalUnmarshalInt64Array(t, []int64{1, 20, 2345, 6789, 12342}, 4, MarshalTypeNearestDelta2)
testMarshalInt64ArraySize(t, va, 1, 500, 1300)
testMarshalInt64ArraySize(t, va, 2, 600, 1400)
testMarshalInt64ArraySize(t, va, 3, 900, 1800)
testMarshalInt64ArraySize(t, va, 4, 1300, 2100)
testMarshalInt64ArraySize(t, va, 5, 2000, 3200)
testMarshalInt64ArraySize(t, va, 6, 3000, 4800)
testMarshalInt64ArraySize(t, va, 7, 4000, 6400)
testMarshalInt64ArraySize(t, va, 8, 6000, 8000)
testMarshalInt64ArraySize(t, va, 9, 7000, 8800)
testMarshalInt64ArraySize(t, va, 10, 8000, 10000)
// Constant encoding
testMarshalUnmarshalInt64Array(t, []int64{1}, 4, MarshalTypeConst)
testMarshalUnmarshalInt64Array(t, []int64{1, 2}, 4, MarshalTypeDeltaConst)
testMarshalUnmarshalInt64Array(t, []int64{-1, 0, 1, 2, 3, 4, 5}, 4, MarshalTypeDeltaConst)
testMarshalUnmarshalInt64Array(t, []int64{-10, -1, 8, 17, 26}, 4, MarshalTypeDeltaConst)
testMarshalUnmarshalInt64Array(t, []int64{0, 0, 0, 0, 0, 0}, 4, MarshalTypeConst)
testMarshalUnmarshalInt64Array(t, []int64{100, 100, 100, 100}, 4, MarshalTypeConst)
}
func testMarshalInt64ArraySize(t *testing.T, va []int64, precisionBits uint8, minSizeExpected, maxSizeExpected int) {

19
lib/encoding/zstd/zstd_cgo.go Normal file
View File

@@ -0,0 +1,19 @@
// +build cgo
package zstd
import (
"github.com/valyala/gozstd"
)
// Decompress appends decompressed src to dst and returns the result.
func Decompress(dst, src []byte) ([]byte, error) {
return gozstd.Decompress(dst, src)
}
// CompressLevel appends compressed src to dst and returns the result.
//
// The given compressionLevel is used for the compression.
func CompressLevel(dst, src []byte, compressionLevel int) []byte {
return gozstd.CompressLevel(dst, src, compressionLevel)
}

69
lib/encoding/zstd/zstd_pure.go Normal file
View File

@@ -0,0 +1,69 @@
// +build !cgo
package zstd
import (
"sync"
"sync/atomic"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/klauspost/compress/zstd"
)
var (
decoder *zstd.Decoder
mu sync.Mutex
av atomic.Value
)
type registry map[int]*zstd.Encoder
func init() {
r := make(registry)
av.Store(r)
var err error
decoder, err = zstd.NewReader(nil)
if err != nil {
logger.Panicf("BUG: failed to create ZSTD reader: %s", err)
}
}
// Decompress appends decompressed src to dst and returns the result.
func Decompress(dst, src []byte) ([]byte, error) {
return decoder.DecodeAll(src, dst)
}
// CompressLevel appends compressed src to dst and returns the result.
//
// The given compressionLevel is used for the compression.
func CompressLevel(dst, src []byte, compressionLevel int) []byte {
e := getEncoder(compressionLevel)
return e.EncodeAll(src, dst)
}
func getEncoder(compressionLevel int) *zstd.Encoder {
r := av.Load().(registry)
if e, ok := r[compressionLevel]; ok {
return e
}
level := zstd.EncoderLevelFromZstd(compressionLevel)
e, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(level))
if err != nil {
logger.Panicf("BUG: failed to create ZSTD writer: %s", err)
}
mu.Lock()
r1 := av.Load().(registry)
r2 := make(registry)
for k, v := range r1 {
r2[k] = v
}
r2[compressionLevel] = e
av.Store(r2)
mu.Unlock()
return e
}

94
lib/encoding/zstd/zstd_test.go Normal file
View File

@@ -0,0 +1,94 @@
// +build cgo
package zstd
import (
"math/rand"
"testing"
pure "github.com/klauspost/compress/zstd"
cgo "github.com/valyala/gozstd"
)
func TestCompressDecompress(t *testing.T) {
testCrossCompressDecompress(t, []byte("a"))
testCrossCompressDecompress(t, []byte("foobarbaz"))
var b []byte
for i := 0; i < 64*1024; i++ {
b = append(b, byte(rand.Int31n(256)))
}
testCrossCompressDecompress(t, b)
}
func testCrossCompressDecompress(t *testing.T, b []byte) {
testCompressDecompress(t, pureCompress, pureDecompress, b)
testCompressDecompress(t, cgoCompress, cgoDecompress, b)
testCompressDecompress(t, pureCompress, cgoDecompress, b)
testCompressDecompress(t, cgoCompress, pureDecompress, b)
}
func testCompressDecompress(t *testing.T, compress compressFn, decompress decompressFn, b []byte) {
bc, err := compress(nil, b, 5)
if err != nil {
t.Fatalf("unexpected error when compressing b=%x: %s", b, err)
}
bNew, err := decompress(nil, bc)
if err != nil {
t.Fatalf("unexpected error when decompressing b=%x from bc=%x: %s", b, bc, err)
}
if string(bNew) != string(b) {
t.Fatalf("invalid bNew; got\n%x; expecting\n%x", bNew, b)
}
prefix := []byte{1, 2, 33}
bcNew, err := compress(prefix, b, 5)
if err != nil {
t.Fatalf("unexpected error when compressing b=%x: %s", bcNew, err)
}
if string(bcNew[:len(prefix)]) != string(prefix) {
t.Fatalf("invalid prefix for b=%x; got\n%x; expecting\n%x", b, bcNew[:len(prefix)], prefix)
}
if string(bcNew[len(prefix):]) != string(bc) {
t.Fatalf("invalid prefixed bcNew for b=%x; got\n%x; expecting\n%x", b, bcNew[len(prefix):], bc)
}
bNew, err = decompress(prefix, bc)
if err != nil {
t.Fatalf("unexpected error when decompressing b=%x from bc=%x with prefix: %s", b, bc, err)
}
if string(bNew[:len(prefix)]) != string(prefix) {
t.Fatalf("invalid bNew prefix when decompressing bc=%x; got\n%x; expecting\n%x", bc, bNew[:len(prefix)], prefix)
}
if string(bNew[len(prefix):]) != string(b) {
t.Fatalf("invalid prefixed bNew; got\n%x; expecting\n%x", bNew[len(prefix):], b)
}
}
type compressFn func(dst, src []byte, compressionLevel int) ([]byte, error)
func pureCompress(dst, src []byte, _ int) ([]byte, error) {
w, err := pure.NewWriter(nil, pure.WithEncoderLevel(pure.SpeedBestCompression))
if err != nil {
return nil, err
}
return w.EncodeAll(src, dst), nil
}
func cgoCompress(dst, src []byte, compressionLevel int) ([]byte, error) {
return cgo.CompressLevel(dst, src, compressionLevel), nil
}
type decompressFn func(dst, src []byte) ([]byte, error)
func pureDecompress(dst, src []byte) ([]byte, error) {
decoder, err := pure.NewReader(nil)
if err != nil {
return nil, err
}
return decoder.DecodeAll(src, dst)
}
func cgoDecompress(dst, src []byte) ([]byte, error) {
return cgo.Decompress(dst, src)
}