gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

perf(blobstore): optimize read buffer size calculation

Browse Source 
Previously, the read buffer was always allocated using the fixed
CrcBlockUnitSize, which wasted memory for small shards.

Now calculate buffer size as min(actual shard size, CrcBlockUnitSize)
to reduce memory usage.

Fixes: github-4033
. #1000824490

Signed-off-by: yanghonggang <yanghonggang@cmss.chinamobile.com>
This commit is contained in:
yanghonggang 2026-04-21 11:03:57 +08:00 committed by slasher
parent 07f397377e
commit a763740261
3 changed files with 35 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
blobstore/blobnode/core/storage/datafile.go
View File

@ -422,7 +422,8 @@ func (cd *datafile) Read(ctx context.Context, shard *core.Shard, from, to uint32
iosr := cd.qosReaderAt(ctx, ra)
// new buffer
buffer := bytespool.Alloc(core.CrcBlockUnitSize)
buffSize := crc32block.DecoderBufferSize(int64(shard.Size))
buffer := bytespool.Alloc(int(buffSize))
// decode crc
decoder, err := crc32block.NewDecoderWithBlock(iosr, pos, int64(shard.Size), buffer, cd.conf.BlockBufferSize)

5
blobstore/common/crc32block/util.go
View File

@ -37,6 +37,11 @@ var (
ErrReadOnClosed = errors.New("crc32block: read on closed")
)
func DecoderBufferSize(shardSize int64) int64 {
buffSize := util.AlignedFull(EncodeSize(shardSize, gBlockSize), baseBlockLen)
return util.Min(buffSize, gBlockSize)
}
func isValidBlockLen(blockLen int64) bool {
return blockLen > 0 && blockLen%baseBlockLen == 0
}

28
blobstore/common/crc32block/util_test.go
View File

@ -23,6 +23,34 @@ import (
"github.com/stretchr/testify/require"
)
func TestDecoderBufferSize(t *testing.T) {
testData := []struct {
shardSize int64
description string
expected int64
}{
{shardSize: 1, description: "very small shard size", expected: baseBlockLen},
{shardSize: int64(baseBlockLen - crc32Len), description: "shard smaller than baseBlockLen after accounting for CRC", expected: baseBlockLen},
{shardSize: int64(baseBlockLen - crc32Len + 1), description: "shard equal to baseBlockLen after accounting for CRC", expected: baseBlockLen * 2},
{shardSize: gBlockSize - crc32Len, description: "shard equal to gBlockSize after accounting for CRC (should be limited)", expected: gBlockSize},
{shardSize: gBlockSize - crc32Len + 1, description: "shard larger than gBlockSize after accounting for CRC (should be limited)", expected: gBlockSize},
}
for _, tc := range testData {
t.Run(tc.description, func(t *testing.T) {
result := DecoderBufferSize(tc.shardSize)
require.GreaterOrEqual(t, result, int64(baseBlockLen))
require.LessOrEqual(t, result, gBlockSize)
require.Equal(t, int64(0), result%int64(baseBlockLen))
require.Equal(t, tc.expected, result,
"Expected %d for shardSize %d, but got %d",
tc.expected, tc.shardSize, result)
})
}
}
func TestDecodeSize(t *testing.T) {
datas := []struct {
blockLen int64