gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 10:06:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(data): compatible json serialization for old datanode. #22883846

Browse Source 
Signed-off-by: clinx <chenlin1@oppo.com>
This commit is contained in:
clinx 2025-01-16 14:28:35 +08:00 committed by zhumingze1108
parent e3ae2ea3f1
commit 0eb05bd9f7
3 changed files with 39 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
datanode/data_partition_repair.go
View File

@ -242,12 +242,22 @@ func (dp *DataPartition) getRemoteExtentInfo(extentType uint8, tinyExtents []uin
err = errors.Trace(err, "getRemoteExtentInfo DataPartition(%v) read from host(%v)", dp.partitionID, target)
return
}
extentFiles, err = storage.UnmarshalBinarySlice(reply.Data[:reply.Size])
if err != nil {
err = errors.Trace(err, "getRemoteExtentInfo DataPartition(%v) unmarshal json(%v) from host(%v)",
dp.partitionID, string(reply.Data[:reply.Size]), target)
data := reply.Data[:reply.Size]
extentFiles, err = storage.UnmarshalBinarySlice(data)
if err != nil && err == storage.ErrNonBytecodeEncode {
extentFiles = make([]*storage.ExtentInfo, 0)
err = json.Unmarshal(data, &extentFiles)
if err != nil {
err = errors.Trace(err, "getRemoteExtentInfo DataPartition(%v) unmarshal json(%v) from host(%v)",
dp.partitionID, string(data), target)
return
}
} else if err != nil {
err = errors.Trace(err, "getRemoteExtentInfo DataPartition(%v) unmarshal bytes from host(%v)",
dp.partitionID, target)
return
}
return
}

16
datanode/storage/extent.go
View File

@ -17,6 +17,7 @@ package storage
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"hash/crc32"
"io"
@ -48,8 +49,11 @@ const (
ExtentMaxSize = 1024 * 1024 * 1024 * 1024 * 4 // 4TB
pageSize = 4096
alignSize = 4096
magicNumber = byte('M')
)
var ErrNonBytecodeEncode = errors.New("non-bytecode serialization method")
func alignment(block []byte, AlignSize int) int {
return int(uintptr(unsafe.Pointer(&block[0])) & uintptr(AlignSize-1))
}
@ -119,6 +123,9 @@ func (ei *ExtentInfo) String() (m string) {
func MarshalBinarySlice(eiSlice []*ExtentInfo) (v []byte, err error) {
buff := bytes.NewBuffer([]byte{})
if err := buff.WriteByte(magicNumber); err != nil {
return nil, err
}
if err := binary.Write(buff, binary.BigEndian, int32(len(eiSlice))); err != nil {
return nil, err
}
@ -173,8 +180,15 @@ func (ei *ExtentInfo) MarshalBinary() (v []byte, err error) {
}
func UnmarshalBinarySlice(data []byte) ([]*ExtentInfo, error) {
buff := bytes.NewBuffer(data)
if len(data) <= 0 {
return nil, ErrNonBytecodeEncode
}
magic := data[0]
if magic != magicNumber {
return nil, ErrNonBytecodeEncode
}
buff := bytes.NewBuffer(data[1:])
// Read the length of the slice
var length int32
if err := binary.Read(buff, binary.BigEndian, &length); err != nil {

10
datanode/storage/extent_test.go
View File

@ -16,6 +16,7 @@ package storage_test
import (
"bytes"
"encoding/json"
"fmt"
"os"
"syscall"
@ -359,4 +360,13 @@ func TestExtentSliceSerialize(t *testing.T) {
require.Equal(t, uint64(2000), deserializedSlice[1].SnapshotDataOff)
require.Equal(t, false, deserializedSlice[0].IsDeleted)
require.Equal(t, uint64(200), deserializedSlice[1].Size)
buf, err1 := json.Marshal(eiSlice)
require.NoError(t, err1)
_, err = storage.UnmarshalBinarySlice(buf)
require.Equal(t, err, storage.ErrNonBytecodeEncode)
extentFiles := make([]*storage.ExtentInfo, 0)
err = json.Unmarshal(buf, &extentFiles)
require.NoError(t, err)
require.Equal(t, 2, len(extentFiles))
}