diff --git a/blobstore/access/stream/stream.go b/blobstore/access/stream/stream.go index cbe8af7fb..cba7b5bb9 100644 --- a/blobstore/access/stream/stream.go +++ b/blobstore/access/stream/stream.go @@ -533,11 +533,6 @@ func (h *Handler) punishShardnodeDisk(ctx context.Context, clusterID proto.Clust } } -// blobCount blobSize > 0 is certain -func blobCount(size uint64, blobSize uint32) uint64 { - return (size + uint64(blobSize) - 1) / uint64(blobSize) -} - func errorTimeout(err error) bool { msg := err.Error() return strings.Contains(msg, "Timeout") || strings.Contains(msg, "timeout") diff --git a/blobstore/access/stream/stream_alloc.go b/blobstore/access/stream/stream_alloc.go index c70d7c1e6..8744da864 100644 --- a/blobstore/access/stream/stream_alloc.go +++ b/blobstore/access/stream/stream_alloc.go @@ -113,7 +113,7 @@ func (h *Handler) allocFromAllocator(ctx context.Context, args := proxy.AllocVolsArgs{ Fsize: size, CodeMode: codeMode, - BidCount: blobCount(size, blobSize), + BidCount: util.AlignedBlocks(size, uint64(blobSize)), } var allocRets []proxy.AllocRet @@ -183,7 +183,7 @@ func (h *Handler) allocFromAllocator(ctx context.Context, setCacheVidHost(clusterID, ret.Vid, allocHost) } - blobN := blobCount(size, blobSize) + blobN := util.AlignedBlocks(size, uint64(blobSize)) blobs := make([]proto.Slice, 0, blobN) for _, bidRet := range allocRets { if blobN <= 0 { diff --git a/blobstore/common/crc32block/util.go b/blobstore/common/crc32block/util.go index 8e481a391..c75c64ae8 100644 --- a/blobstore/common/crc32block/util.go +++ b/blobstore/common/crc32block/util.go @@ -58,7 +58,7 @@ func EncodeSize(size int64, blockLen int64) int64 { panic(ErrInvalidBlock) } payload := BlockPayload(blockLen) - blockCnt := (size + (payload - 1)) / payload + blockCnt := util.AlignedBlocks(size, payload) return size + crc32Len*blockCnt } @@ -66,7 +66,7 @@ func DecodeSize(totalSize int64, blockLen int64) int64 { if !isValidBlockLen(blockLen) { panic(ErrInvalidBlock) } - blockCnt := (totalSize + (blockLen - 1)) / blockLen + blockCnt := util.AlignedBlocks(totalSize, blockLen) return totalSize - crc32Len*blockCnt } diff --git a/blobstore/common/rpc2/checksum.go b/blobstore/common/rpc2/checksum.go index 2cc1876f9..dbec59371 100644 --- a/blobstore/common/rpc2/checksum.go +++ b/blobstore/common/rpc2/checksum.go @@ -94,8 +94,7 @@ func (cb *ChecksumBlock) EncodeSize(originalSize int64) int64 { if cb == nil || *cb == (ChecksumBlock{}) || originalSize == 0 { return originalSize } - payload := int64(cb.BlockSize) - blocks := (originalSize + (payload - 1)) / payload + blocks := util.AlignedBlocks(originalSize, int64(cb.BlockSize)) size := int64(algorithmSizes[cb.Algorithm]) if cb.Aligned { size = _checksumAlignment diff --git a/blobstore/util/alignment.go b/blobstore/util/alignment.go index 08cfa4bd4..bc2c6612a 100644 --- a/blobstore/util/alignment.go +++ b/blobstore/util/alignment.go @@ -14,7 +14,10 @@ package util -import "unsafe" +import ( + "fmt" + "unsafe" +) // AlignedBuffer address of first byte of buff wa aligned. // - - - - - - - - - - - - - - - - - - - - @@ -34,10 +37,36 @@ func AlignedBuffer(head, capacity, alignment int) []byte { // AlignedHead head with alignment. func AlignedHead[I Integer](size, alignment I) I { + assert(size, alignment) return size % alignment } // AlignedTail tail with alignment. func AlignedTail[I Integer](size, alignment I) I { + assert(size, alignment) return (alignment - size%alignment) % alignment } + +// AlignedFull full with alignment. +func AlignedFull[I Integer](size, alignment I) I { + assert(size, alignment) + if (alignment & (alignment - 1)) == 0 { + return (size + (alignment - 1)) & (^(alignment - 1)) + } + return AlignedBlocks(size, alignment) * alignment +} + +// AlignedBlocks number of blocks with alignment. +func AlignedBlocks[I Integer](size, alignment I) I { + assert(size, alignment) + return (size + (alignment - 1)) / alignment +} + +func assert[I Integer](size, alignment I) { + if size < 0 { + panic(fmt.Sprintf("util-align: size(%d) < 0", size)) + } + if alignment <= 0 { + panic(fmt.Sprintf("util-align: alignment(%d) <= 0", alignment)) + } +} diff --git a/blobstore/util/alignment_test.go b/blobstore/util/alignment_test.go index 962765553..ac3aa23cf 100644 --- a/blobstore/util/alignment_test.go +++ b/blobstore/util/alignment_test.go @@ -38,6 +38,10 @@ func TestUtilAlignmentBuffer(t *testing.T) { } func TestUtilAlignmentHeadTail(t *testing.T) { + require.Panics(t, func() { AlignedHead(-1, 128) }) + require.Panics(t, func() { AlignedHead(1, 0) }) + require.Panics(t, func() { AlignedTail(-1, 128) }) + require.Panics(t, func() { AlignedTail(1, 0) }) require.Equal(t, 0, AlignedHead(0, 128)) require.Equal(t, int64(1), AlignedHead[int64](513, 512)) require.Equal(t, uint64(0), AlignedTail[uint64](512, 512)) @@ -57,3 +61,46 @@ func TestUtilAlignmentHeadTail(t *testing.T) { } } } + +func TestUtilAlignmentBlock(t *testing.T) { + require.Equal(t, 0, AlignedFull(0, 128)) + require.Equal(t, 0, AlignedBlocks(0, 128)) + require.Equal(t, int64(1024), AlignedFull[int64](513, 512)) + require.Equal(t, int64(2), AlignedBlocks[int64](513, 512)) + require.Equal(t, uint64(512), AlignedFull[uint64](512, 512)) + require.Equal(t, uint64(1), AlignedBlocks[uint64](512, 512)) + for range [100]struct{}{} { + size := rand.Int63() + 1 + alignment := int64(32 << 10) + if rand.Int63()%7 == 0 { + size -= size % alignment + } + full := AlignedFull(size, alignment) + blocks := AlignedBlocks(size, alignment) + require.Equal(t, full, blocks*alignment) + if tail := AlignedTail(size, alignment); tail == 0 { + require.Equal(t, size, full) + require.Equal(t, size/alignment, blocks) + } else { + require.Equal(t, size+tail, full) + require.Equal(t, size/alignment+1, blocks) + } + } + for range [100]struct{}{} { + size := rand.Int63() + 1 + alignment := rand.Int63n(1<<20) + 1 + if rand.Int63()%7 == 0 { + size -= size % alignment + } + full := AlignedFull(size, alignment) + blocks := AlignedBlocks(size, alignment) + require.Equal(t, full, blocks*alignment) + if tail := AlignedTail(size, alignment); tail == 0 { + require.Equal(t, size, full) + require.Equal(t, size/alignment, blocks) + } else { + require.Equal(t, size+tail, full) + require.Equal(t, size/alignment+1, blocks) + } + } +}