Files
VictoriaMetrics/lib/bytesutil/bytebuffer_test.go
Aliaksandr Valialkin 13ff9a8ebd lib/{mergeset,storage,logstorage}: use chunked buffer instead of bytesutil.ByteBuffer as a storage for in-memory parts
This commit adds lib/chunkedbuffer.Buffer - an in-memory chunked buffer
optimized for random access via MustReadAt() function.
It is better than bytesutil.ByteBuffer for storing large volumes of data,
since it stores the data in chunks of a fixed size (4KiB at the moment)
instead of using a contiguous memory region. This has the following benefits over bytesutil.ByteBuffer:

- reduced memory fragmentation
- reduced memory re-allocations when new data is written to the buffer
- reduced memory usage, since the allocated chunks can be re-used
  by other Buffer instances after Buffer.Reset() call

Performance tests show up to 2x memory reduction for VictoriaLogs
when ingesting logs with big number of fields (aka wide events) under high speed.
2025-03-15 20:58:33 +01:00

219 lines
5.7 KiB
Go

package bytesutil
import (
"bytes"
"fmt"
"io"
"testing"
)
func TestByteBuffer(t *testing.T) {
var bb ByteBuffer
n, err := bb.Write(nil)
if err != nil {
t.Fatalf("cannot write nil: %s", err)
}
if n != 0 {
t.Fatalf("unexpected n when writing nil; got %d; want %d", n, 0)
}
if len(bb.B) != 0 {
t.Fatalf("unexpected len(bb.B) after writing nil; got %d; want %d", len(bb.B), 0)
}
n, err = bb.Write([]byte{})
if err != nil {
t.Fatalf("cannot write empty slice: %s", err)
}
if n != 0 {
t.Fatalf("unexpected n when writing empty slice; got %d; want %d", n, 0)
}
if len(bb.B) != 0 {
t.Fatalf("unexpected len(bb.B) after writing empty slice; got %d; want %d", len(bb.B), 0)
}
data1 := []byte("123")
n, err = bb.Write(data1)
if err != nil {
t.Fatalf("cannot write %q: %s", data1, err)
}
if n != len(data1) {
t.Fatalf("unexpected n when writing %q; got %d; want %d", data1, n, len(data1))
}
if string(bb.B) != string(data1) {
t.Fatalf("unexpected bb.B; got %q; want %q", bb.B, data1)
}
data2 := []byte("1")
n, err = bb.Write(data2)
if err != nil {
t.Fatalf("cannot write %q: %s", data2, err)
}
if n != len(data2) {
t.Fatalf("unexpected n when writing %q; got %d; want %d", data2, n, len(data2))
}
if string(bb.B) != string(data1)+string(data2) {
t.Fatalf("unexpected bb.B; got %q; want %q", bb.B, string(data1)+string(data2))
}
bb.Reset()
if string(bb.B) != "" {
t.Fatalf("unexpected bb.B after reset; got %q; want %q", bb.B, "")
}
r := bb.NewReader().(*reader)
if r.readOffset != 0 {
t.Fatalf("unexpected r.readOffset after reset; got %d; want %d", r.readOffset, 0)
}
}
func TestByteBufferReadFrom(t *testing.T) {
var bbPool ByteBufferPool
t.Run("zero_bytes", func(t *testing.T) {
t.Parallel()
bb := bbPool.Get()
defer bbPool.Put(bb)
src := bytes.NewBufferString("")
n, err := bb.ReadFrom(src)
if err != nil {
t.Fatalf("error when reading empty string: %s", err)
}
if n != 0 {
t.Fatalf("unexpected number of bytes read; got %d; want %d", n, 0)
}
if len(bb.B) != 0 {
t.Fatalf("unexpejcted len(bb.B); got %d; want %d", len(bb.B), 0)
}
})
t.Run("non_zero_bytes", func(t *testing.T) {
t.Parallel()
bb := bbPool.Get()
defer bbPool.Put(bb)
s := "foobarbaz"
src := bytes.NewBufferString(s)
n, err := bb.ReadFrom(src)
if err != nil {
t.Fatalf("error when reading non-empty string: %s", err)
}
if n != int64(len(s)) {
t.Fatalf("unexpected number of bytes read; got %d; want %d", n, len(s))
}
if string(bb.B) != s {
t.Fatalf("unexpected value read; got %q; want %q", bb.B, s)
}
})
t.Run("big_number_of_bytes", func(t *testing.T) {
t.Parallel()
bb := bbPool.Get()
defer bbPool.Put(bb)
b := make([]byte, 1024*1024+234)
for i := range b {
b[i] = byte(i)
}
src := bytes.NewBuffer(b)
n, err := bb.ReadFrom(src)
if err != nil {
t.Fatalf("cannot read big value: %s", err)
}
if n != int64(len(b)) {
t.Fatalf("unexpected number of bytes read; got %d; want %d", n, len(b))
}
if string(bb.B) != string(b) {
t.Fatalf("unexpected value read; got %q; want %q", bb.B, b)
}
})
t.Run("non_empty_bb", func(t *testing.T) {
t.Parallel()
bb := bbPool.Get()
defer bbPool.Put(bb)
prefix := []byte("prefix")
bb.B = append(bb.B[:0], prefix...)
s := "aosdfdsafdjsf"
src := bytes.NewBufferString(s)
n, err := bb.ReadFrom(src)
if err != nil {
t.Fatalf("cannot read to non-empty bb: %s", err)
}
if n != int64(len(s)) {
t.Fatalf("unexpected number of bytes read; got %d; want %d", n, len(s))
}
if len(bb.B) != len(prefix)+len(s) {
t.Fatalf("unexpected bb.B len; got %d; want %d", len(bb.B), len(prefix)+len(s))
}
if string(bb.B[:len(prefix)]) != string(prefix) {
t.Fatalf("unexpected prefix; got %q; want %q", bb.B[:len(prefix)], prefix)
}
if string(bb.B[len(prefix):]) != s {
t.Fatalf("unexpected data read; got %q; want %q", bb.B[len(prefix):], s)
}
})
}
func TestByteBufferRead(t *testing.T) {
var bb ByteBuffer
n, err := fmt.Fprintf(&bb, "foo, %s, baz", "bar")
if err != nil {
t.Fatalf("unexpected error after fmt.Fprintf: %s", err)
}
if n != len(bb.B) {
t.Fatalf("unexpected len(bb.B); got %d; want %d", len(bb.B), n)
}
if string(bb.B) != "foo, bar, baz" {
t.Fatalf("unexpected bb.B; got %q; want %q", bb.B, "foo, bar, baz")
}
r := bb.NewReader().(*reader)
if r.readOffset != 0 {
t.Fatalf("unexpected r.readOffset; got %d; want %q", r.readOffset, 0)
}
rCopy := bb.NewReader().(*reader)
var bb1 ByteBuffer
n1, err := io.Copy(&bb1, r)
if err != nil {
t.Fatalf("unexpected error after io.Copy: %s", err)
}
if int64(r.readOffset) != n1 {
t.Fatalf("unexpected r.readOffset after io.Copy; got %d; want %d", r.readOffset, n1)
}
if n1 != int64(n) {
t.Fatalf("unexpected number of bytes copied; got %d; want %d", n1, n)
}
if string(bb1.B) != "foo, bar, baz" {
t.Fatalf("unexpected bb1.B; got %q; want %q", bb1.B, "foo, bar, baz")
}
// Make read returns io.EOF
buf := make([]byte, n)
n2, err := r.Read(buf)
if err != io.EOF {
t.Fatalf("unexpected error returned: got %q; want %q", err, io.EOF)
}
if n2 != 0 {
t.Fatalf("unexpected n1 returned; got %d; want %d", n2, 0)
}
// Read data from rCopy
if rCopy.readOffset != 0 {
t.Fatalf("unexpected rCopy.readOffset; got %d; want %d", rCopy.readOffset, 0)
}
buf = make([]byte, n+13)
n2, err = rCopy.Read(buf)
if err != io.EOF {
t.Fatalf("unexpected error when reading from rCopy: got %q; want %q", err, io.EOF)
}
if n2 != n {
t.Fatalf("unexpected number of bytes read from rCopy; got %d; want %d", n2, n)
}
if string(buf[:n2]) != "foo, bar, baz" {
t.Fatalf("unexpected data read: got %q; want %q", buf[:n2], "foo, bar, baz")
}
if rCopy.readOffset != n2 {
t.Fatalf("unexpected rCopy.readOffset; got %d; want %d", rCopy.readOffset, n2)
}
}