gian/cubefs
1
0
Fork 0
mirror of https://github.com/cubefs/cubefs.git synced 2026-08-02 02:00:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(raftserver): the raft log is stored in rocksdb

Browse Source 
Signed-off-by: yuxiaobo <yxbstorm@gmail.com>
This commit is contained in:
yuxiaobo 2025-02-11 16:02:15 +08:00 committed by luckydog
parent f091672537
commit 2ba145511f
15 changed files with 589 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
blobstore/common/raftserver/config.go
View File

@ -26,6 +26,7 @@ type Config struct {
ListenPort int `json:"listen_port"`
WalDir string `json:"raft_wal_dir"`
WalSync bool `json:"raft_wal_sync"`
UseRocksdb bool `json:"use_rocksdb"`
TickIntervalMs int `json:"-"` // for test
// TickInterval is the interval of timer which check heartbeat and election timeout.

11
blobstore/common/raftserver/server.go
View File

@ -113,7 +113,7 @@ func NewRaftServer(cfg *Config) (RaftServer, error) {
rs.readNotifier.Store(newReadIndexNotifier())
begin := time.Now()
store, err := NewRaftStorage(cfg.WalDir, cfg.WalSync, cfg.NodeId, rs.sm, rs.shotter)
store, err := NewRaftStorage(cfg.WalDir, cfg.WalSync, cfg.UseRocksdb, cfg.NodeId, rs.sm, rs.shotter)
if err != nil {
return nil, err
}
@ -522,17 +522,10 @@ func (s *raftServer) raftStart() {
}
s.tr.Send(s.processMessages(rd.Messages))
if len(rd.Entries) > 0 {
err := s.store.SaveEntries(rd.Entries)
err := s.store.Save(rd.HardState, rd.Entries)
if err != nil {
log.Panicf("save raft entries error: %v", err)
}
}
if !raft.IsEmptyHardState(rd.HardState) {
if err := s.store.SaveHardState(rd.HardState); err != nil {
log.Panicf("save raft hardstate error: %v", err)
}
}
s.n.Advance()
}

4
blobstore/common/raftserver/server_test.go
View File

@ -55,6 +55,7 @@ var cfgs = [3]*Config{
NodeId: 1,
ListenPort: 9090,
WalDir: "/tmp/raftserver/wal1",
UseRocksdb: true,
TickIntervalMs: 100,
ElectionTick: 3,
Members: members,
@ -63,6 +64,7 @@ var cfgs = [3]*Config{
NodeId: 2,
ListenPort: 9091,
WalDir: "/tmp/raftserver/wal2",
UseRocksdb: true,
TickIntervalMs: 100,
ElectionTick: 3,
Members: members,
@ -71,6 +73,7 @@ var cfgs = [3]*Config{
NodeId: 3,
ListenPort: 9092,
WalDir: "/tmp/raftserver/wal3",
UseRocksdb: true,
TickIntervalMs: 100,
ElectionTick: 3,
Members: members,
@ -266,6 +269,7 @@ func TestRaftServer(t *testing.T) {
NodeId: 4,
ListenPort: 9093,
WalDir: "/tmp/raftserver/wal4",
UseRocksdb: true,
TickIntervalMs: 100,
ElectionTick: 3,
Members: members,

29
blobstore/common/raftserver/storage.go
View File

@ -29,7 +29,7 @@ import (
type raftStorage struct {
nodeId uint64
walMu sync.RWMutex
wal *wal.Wal
wal wal.Wal
shotter *snapshotter
sm StateMachine
cs pb.ConfState
@ -39,7 +39,7 @@ type raftStorage struct {
snapIndex uint64
}
func NewRaftStorage(walDir string, sync bool, nodeId uint64, sm StateMachine, shotter *snapshotter) (*raftStorage, error) {
func NewRaftStorage(walDir string, sync bool, use_rocksdb bool, nodeId uint64, sm StateMachine, shotter *snapshotter) (*raftStorage, error) {
rs := &raftStorage{
nodeId: nodeId,
shotter: shotter,
@ -47,11 +47,20 @@ func NewRaftStorage(walDir string, sync bool, nodeId uint64, sm StateMachine, sh
sm: sm,
}
wal, err := wal.OpenWal(walDir, sync)
var (
w wal.Wal
err error
)
if use_rocksdb {
w, err = wal.OpenRocksdbWal(walDir)
} else {
w, err = wal.OpenWal(walDir, sync)
}
if err != nil {
return nil, err
}
rs.wal = wal
rs.wal = w
return rs, nil
}
@ -140,16 +149,10 @@ func (s *raftStorage) Snapshot() (pb.Snapshot, error) {
}, nil
}
func (s *raftStorage) SaveEntries(entries []pb.Entry) error {
func (s *raftStorage) Save(hs pb.HardState, entries []pb.Entry) error {
s.walMu.Lock()
defer s.walMu.Unlock()
return s.wal.SaveEntries(entries)
}
func (s *raftStorage) SaveHardState(hs pb.HardState) error {
s.walMu.Lock()
defer s.walMu.Unlock()
return s.wal.SaveHardState(hs)
return s.wal.Save(hs, entries)
}
func (s *raftStorage) SetApplied(applied uint64) {
@ -227,5 +230,7 @@ func (s *raftStorage) GetMember(id uint64) (Member, bool) {
}
func (s *raftStorage) Close() {
s.walMu.Lock()
defer s.walMu.Unlock()
s.wal.Close()
}

8
blobstore/common/raftserver/storage_test.go
View File

@ -86,7 +86,7 @@ func (sm *storeSM) LeaderChange(leader uint64, host string) {
func TestStorage(t *testing.T) {
{
os.RemoveAll(walDir)
store, err := NewRaftStorage(walDir, true, nodeId, &storeSM{}, newSnapshotter(5, time.Second*10))
store, err := NewRaftStorage(walDir, true, true, nodeId, &storeSM{}, newSnapshotter(5, time.Second*10))
require.Nil(t, err)
hs, cs, _ := store.InitialState()
require.Equal(t, hs, pb.HardState{})
@ -97,7 +97,7 @@ func TestStorage(t *testing.T) {
{
os.RemoveAll(walDir)
store, err := NewRaftStorage(walDir, true, nodeId, &storeSM{}, newSnapshotter(5, time.Second*10))
store, err := NewRaftStorage(walDir, true, true, nodeId, &storeSM{}, newSnapshotter(5, time.Second*10))
require.Nil(t, err)
var entries []pb.Entry
for i := 0; i < 1000; i++ {
@ -109,7 +109,7 @@ func TestStorage(t *testing.T) {
}
entries = append(entries, entry)
}
err = store.SaveEntries(entries)
err = store.Save(pb.HardState{}, entries)
require.Nil(t, err)
lastIndex, err := store.LastIndex()
@ -143,7 +143,7 @@ func TestStorage(t *testing.T) {
Vote: 1,
Commit: 90,
}
err = store.SaveHardState(hs)
err = store.Save(hs, nil)
require.Nil(t, err)
store.SetApplied(uint64(90))

4
blobstore/common/raftserver/wal/fileutil.go
View File

@ -26,8 +26,10 @@ const (
TrashPath = ".trash"
)
func InitPath(dir string) error {
func InitPath(dir string, createTrush bool) error {
if createTrush {
dir = path.Join(dir, TrashPath)
}
info, err := os.Stat(dir)
if err != nil {
if pathErr, ok := err.(*os.PathError); ok {

2
blobstore/common/raftserver/wal/fileutil_test.go
View File

@ -27,7 +27,7 @@ func TestFileutil(t *testing.T) {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
err := InitPath(dir)
err := InitPath(dir, true)
if err != nil {
t.Fatal(err)
}

32
blobstore/common/raftserver/wal/log_storage.go
View File

@ -30,7 +30,7 @@ const (
logfileCacheNum = 4
)
func (w *Wal) reload(firstIndex uint64) error {
func (w *fileWal) reload(firstIndex uint64) error {
names, err := listLogFiles(w.dir)
if err != nil {
return err
@ -57,7 +57,7 @@ func (w *Wal) reload(firstIndex uint64) error {
return nil
}
func (w *Wal) term(i uint64) (term uint64, err error) {
func (w *fileWal) term(i uint64) (term uint64, err error) {
lf, err := w.locateFile(i)
if err != nil {
return
@ -66,7 +66,7 @@ func (w *Wal) term(i uint64) (term uint64, err error) {
return
}
func (wal *Wal) lastIndex() uint64 {
func (wal *fileWal) lastIndex() uint64 {
if wal.last.Len() == 0 {
if len(wal.logfiles) > 1 {
return wal.last.name.index - 1
@ -76,7 +76,7 @@ func (wal *Wal) lastIndex() uint64 {
return wal.last.LastIndex()
}
func (w *Wal) entries(lo, hi uint64, maxSize uint64) (entries []pb.Entry, err error) {
func (w *fileWal) entries(lo, hi uint64, maxSize uint64) (entries []pb.Entry, err error) {
if hi > w.lastIndex()+1 {
err = fmt.Errorf("entries's hi(%d) is out of bound lastindex(%d)", hi, w.lastIndex())
return
@ -115,7 +115,7 @@ func (w *Wal) entries(lo, hi uint64, maxSize uint64) (entries []pb.Entry, err er
return
}
func (w *Wal) saveEntries(ents []pb.Entry) error {
func (w *fileWal) saveEntries(ents []pb.Entry) error {
if len(ents) == 0 {
return nil
}
@ -137,11 +137,11 @@ func (w *Wal) saveEntries(ents []pb.Entry) error {
return nil
}
func (w *Wal) Sync() error {
func (w *fileWal) Sync() error {
return w.last.Sync()
}
func (w *Wal) truncateFront(index uint64) error {
func (w *fileWal) truncateFront(index uint64) error {
truncFIndex := -1
for i := 0; i < len(w.logfiles)-1; i++ {
if w.logfiles[i+1].index-1 <= index {
@ -167,7 +167,7 @@ func (w *Wal) truncateFront(index uint64) error {
return nil
}
func (w *Wal) truncateAll(firstIndex uint64) error {
func (w *fileWal) truncateAll(firstIndex uint64) error {
for _, f := range w.logfiles {
if err := w.remove(f); err != nil {
return err
@ -187,7 +187,7 @@ func (w *Wal) truncateAll(firstIndex uint64) error {
return nil
}
func (w *Wal) truncateBack(index uint64) error {
func (w *fileWal) truncateBack(index uint64) error {
if w.lastIndex() < index {
return nil
}
@ -232,7 +232,7 @@ func (w *Wal) truncateBack(index uint64) error {
return nil
}
func (w *Wal) createNew(index uint64) (*logFile, error) {
func (w *fileWal) createNew(index uint64) (*logFile, error) {
name := logName{sequence: w.nextFileSeq, index: index}
f, err := createLogFile(w.dir, name)
if err != nil {
@ -249,20 +249,20 @@ func (w *Wal) createNew(index uint64) (*logFile, error) {
return f, nil
}
func (w *Wal) get(name logName) (*logFile, error) {
func (w *fileWal) get(name logName) (*logFile, error) {
if name.sequence == w.last.Seq() {
return w.last, nil
}
return w.cache.Get(name)
}
func (w *Wal) remove(name logName) error {
func (w *fileWal) remove(name logName) error {
filename := name.String()
trashdir := path.Join(w.dir, TrashPath)
return os.Rename(path.Join(w.dir, filename), path.Join(trashdir, filename))
}
func (w *Wal) rotate() error {
func (w *fileWal) rotate() error {
prevLast := w.last.LastIndex()
if err := w.last.FinishWrite(); err != nil {
@ -281,7 +281,7 @@ func (w *Wal) rotate() error {
return nil
}
func (w *Wal) locate(logindex uint64) int {
func (w *fileWal) locate(logindex uint64) int {
fi := sort.Search(len(w.logfiles), func(i int) bool {
var nextIndex uint64
if i == len(w.logfiles)-1 {
@ -294,7 +294,7 @@ func (w *Wal) locate(logindex uint64) int {
return fi
}
func (w *Wal) locateFile(logindex uint64) (*logFile, error) {
func (w *fileWal) locateFile(logindex uint64) (*logFile, error) {
i := w.locate(logindex)
if i >= len(w.logfiles) {
panic("could not find log file")
@ -302,7 +302,7 @@ func (w *Wal) locateFile(logindex uint64) (*logFile, error) {
return w.get(w.logfiles[i])
}
func (w *Wal) saveEntry(ent *pb.Entry) error {
func (w *fileWal) saveEntry(ent *pb.Entry) error {
prevIndex := w.lastIndex()
if prevIndex != 0 {
if prevIndex+1 != ent.Index {

15
blobstore/common/raftserver/wal/log_storage_test.go
View File

@ -23,10 +23,11 @@ import (
"github.com/stretchr/testify/require"
"go.etcd.io/etcd/raft/v3/raftpb"
pb "go.etcd.io/etcd/raft/v3/raftpb"
)
func openLogStorage(dir string) *Wal {
return &Wal{
func openLogStorage(dir string) *fileWal {
return &fileWal{
dir: dir,
nextFileSeq: 1,
cache: newLogFileCache(logfileCacheNum,
@ -112,7 +113,7 @@ func TestLogStorageTruncateFront(t *testing.T) {
}
clear()
defer clear()
InitPath(dir)
InitPath(dir, true)
ls := openLogStorage(dir)
err := ls.reload(1)
@ -126,7 +127,7 @@ func TestLogStorageTruncateFront(t *testing.T) {
Type: raftpb.EntryNormal,
}
}
err = ls.SaveEntries(entries)
err = ls.Save(pb.HardState{}, entries)
require.Nil(t, err)
err = ls.rotate()
require.Nil(t, err)
@ -148,7 +149,7 @@ func TestLogStoragetruncateBack(t *testing.T) {
}
clear()
defer clear()
InitPath(dir)
InitPath(dir, true)
ls := openLogStorage(dir)
err := ls.reload(100)
@ -162,7 +163,7 @@ func TestLogStoragetruncateBack(t *testing.T) {
Type: raftpb.EntryNormal,
}
}
err = ls.SaveEntries(entries)
err = ls.Save(pb.HardState{}, entries)
require.Nil(t, err)
err = ls.rotate()
require.Nil(t, err)
@ -179,7 +180,7 @@ func TestLogStoragetruncateBack(t *testing.T) {
Type: raftpb.EntryNormal,
}
}
err = ls.SaveEntries(entries)
err = ls.Save(pb.HardState{}, entries)
require.Nil(t, err)
err = ls.truncateBack(100)
require.Nil(t, err)

4
blobstore/common/raftserver/wal/record_reader_test.go
View File

@ -35,7 +35,7 @@ func TestRecordReaderRead(t *testing.T) {
}
clear()
defer clear()
InitPath(dir)
InitPath(dir, true)
f, err := os.OpenFile(path.Join(dir, "test"), os.O_CREATE|os.O_RDWR, 0o644)
require.Nil(t, err)
@ -87,7 +87,7 @@ func TestRecordReaderReadAt(t *testing.T) {
}
clear()
defer clear()
InitPath(dir)
InitPath(dir, true)
f, err := os.OpenFile(path.Join(dir, "test"), os.O_CREATE|os.O_RDWR, 0o644)
require.Nil(t, err)

2
blobstore/common/raftserver/wal/record_writer_test.go
View File

@ -28,7 +28,7 @@ func TestRecordWriter(t *testing.T) {
file := path.Join(dir, name.String())
os.RemoveAll(dir)
err := InitPath(dir)
err := InitPath(dir, true)
if err != nil {
t.Fatal(err)
}

328
blobstore/common/raftserver/wal/rocksdb_wal.go Normal file
View File

@ -0,0 +1,328 @@
package wal
import (
"encoding/binary"
"errors"
"fmt"
"math"
"path"
"sync"
"github.com/cubefs/cubefs/blobstore/util/log"
"github.com/tecbot/gorocksdb"
"go.etcd.io/etcd/raft/v3"
pb "go.etcd.io/etcd/raft/v3/raftpb"
)
type rocksdbWal struct {
db *gorocksdb.DB
hs pb.HardState
st Snapshot
lastIndex uint64
once sync.Once
}
const (
kMeta byte = iota
kLogEntry
)
func dbKey(typ byte, key uint64) []byte {
var skey [9]byte
skey[0] = typ
binary.BigEndian.PutUint64(skey[1:], key)
return skey[:]
}
func hardStateKey() []byte {
return dbKey(kMeta, 0)
}
func snapshotKey() []byte {
return dbKey(kMeta, 1)
}
func logKey(index uint64) []byte {
return dbKey(kLogEntry, index)
}
func OpenRocksdbWal(dir string) (Wal, error) {
dir = path.Clean(dir)
if err := InitPath(dir, false); err != nil {
return nil, err
}
opts := gorocksdb.NewDefaultOptions()
opts.SetCreateIfMissing(true)
db, err := gorocksdb.OpenDb(opts, dir)
if err != nil {
return nil, err
}
wal := &rocksdbWal{
db: db,
}
defer func() {
if err != nil {
db.Close()
}
}()
if err = wal.loadHardState(); err != nil {
log.Errorf("load hardstate form db error: %v", err)
return nil, err
}
if err = wal.loadSnapshot(); err != nil {
log.Errorf("load snapshot form db error: %v", err)
return nil, err
}
if err = wal.loadLastIndex(); err != nil {
log.Errorf("load last index form db error: %v", err)
return nil, err
}
if wal.lastIndex < wal.st.Index {
wal.lastIndex = wal.st.Index
}
log.Infof("load wal success: lastindex=%d hs=%s snapshot=%v", wal.lastIndex, wal.hs.String(), wal.st)
return wal, nil
}
func (w *rocksdbWal) loadHardState() error {
ro := gorocksdb.NewDefaultReadOptions()
defer ro.Destroy()
val, err := w.db.GetBytes(ro, hardStateKey())
if err != nil {
return err
}
if len(val) == 0 {
return nil
}
return w.hs.Unmarshal(val)
}
func (w *rocksdbWal) loadSnapshot() error {
ro := gorocksdb.NewDefaultReadOptions()
defer ro.Destroy()
val, err := w.db.GetBytes(ro, snapshotKey())
if err != nil {
return err
}
if len(val) == 0 {
return nil
} else if len(val) != 16 {
return errors.New("invalid snapshot")
}
w.st.Index = binary.BigEndian.Uint64(val)
w.st.Term = binary.BigEndian.Uint64(val[8:])
return nil
}
func (w *rocksdbWal) loadLastIndex() error {
ro := gorocksdb.NewDefaultReadOptions()
ro.SetFillCache(false)
it := w.db.NewIterator(ro)
defer func() {
it.Close()
ro.Destroy()
}()
it.SeekToLast()
if err := it.Err(); err != nil {
return err
}
prefix := make([]byte, 1)
prefix[0] = kLogEntry
if it.ValidForPrefix(prefix) {
key := it.Key()
skey := key.Data()
key.Free()
if len(skey) != 9 && skey[0] != kLogEntry {
return errors.New("invalid log key")
}
w.lastIndex = binary.BigEndian.Uint64(skey[1:])
}
return nil
}
func (w *rocksdbWal) InitialState() pb.HardState {
return w.hs
}
func (w *rocksdbWal) Entries(lo, hi uint64, maxSize uint64) (entries []pb.Entry, err error) {
if lo >= hi {
return nil, raft.ErrUnavailable
} else if lo <= w.st.Index {
return nil, raft.ErrCompacted
}
if hi > w.lastIndex+1 {
return nil, fmt.Errorf("entries's hi(%d) is out of bound lastindex(%d)", hi, w.lastIndex)
}
ro := gorocksdb.NewDefaultReadOptions()
ro.SetFillCache(false)
ro.SetIterateUpperBound(logKey(hi))
it := w.db.NewIterator(ro)
defer func() {
it.Close()
ro.Destroy()
}()
for it.Seek(logKey(lo)); it.Valid(); it.Next() {
entry := pb.Entry{}
value := it.Value()
if err := entry.Unmarshal(value.Data()); err != nil {
value.Free()
return nil, err
}
value.Free()
entries = append(entries, entry)
}
if err := it.Err(); err != nil {
return nil, err
}
return entries, nil
}
func (w *rocksdbWal) Term(index uint64) (term uint64, err error) {
if index < w.st.Index {
return 0, raft.ErrCompacted
} else if index == w.st.Index {
return w.st.Term, nil
}
if index > w.lastIndex {
return 0, raft.ErrUnavailable
}
ro := gorocksdb.NewDefaultReadOptions()
defer ro.Destroy()
val, err := w.db.GetBytes(ro, logKey(index))
if err != nil {
return 0, err
}
ent := pb.Entry{}
if err = ent.Unmarshal(val); err != nil {
return 0, err
}
return ent.Term, nil
}
func (w *rocksdbWal) FirstIndex() uint64 {
return w.st.Index + 1
}
func (w *rocksdbWal) LastIndex() uint64 {
return w.lastIndex
}
func (w *rocksdbWal) Save(hs pb.HardState, entries []pb.Entry) error {
wb := gorocksdb.NewWriteBatch()
wo := gorocksdb.NewDefaultWriteOptions()
defer func() {
wb.Destroy()
wo.Destroy()
}()
n := len(entries)
if n > 0 {
li := entries[0].Index
if li < w.lastIndex {
wb.DeleteRange(logKey(entries[0].Index), logKey(w.lastIndex))
} else if li == w.lastIndex {
wb.Delete(logKey(entries[0].Index))
} else if li > w.lastIndex+1 {
return fmt.Errorf("log index(%d) is larger than last index(%d)", entries[0].Index, w.lastIndex)
}
for i := 0; i < n; i++ {
val, err := entries[i].Marshal()
if err != nil {
return err
}
wb.Put(logKey(entries[i].Index), val)
}
}
if !raft.IsEmptyHardState(hs) {
val, err := hs.Marshal()
if err != nil {
return err
}
wb.Put(hardStateKey(), val)
}
if err := w.db.Write(wo, wb); err != nil {
return err
}
if n > 0 {
w.lastIndex = entries[n-1].Index
}
if !raft.IsEmptyHardState(hs) {
w.hs = hs
}
return nil
}
func (w *rocksdbWal) Truncate(index uint64) error {
if index <= w.st.Index {
return raft.ErrCompacted
}
term, err := w.Term(index)
if err != nil {
return err
}
st := Snapshot{
Index: index,
Term: term,
}
wb := gorocksdb.NewWriteBatch()
wo := gorocksdb.NewDefaultWriteOptions()
defer func() {
wb.Destroy()
wo.Destroy()
}()
val := make([]byte, 16)
binary.BigEndian.PutUint64(val, st.Index)
binary.BigEndian.PutUint64(val[8:], st.Term)
wb.Put(snapshotKey(), val)
wb.DeleteRange(logKey(0), logKey(index))
if err = w.db.Write(wo, wb); err != nil {
return err
}
w.st = st
return nil
}
func (w *rocksdbWal) ApplySnapshot(st Snapshot) error {
hs := pb.HardState{
Commit: st.Index,
Term: st.Term,
}
wb := gorocksdb.NewWriteBatch()
wo := gorocksdb.NewDefaultWriteOptions()
defer func() {
wb.Destroy()
wo.Destroy()
}()
val := make([]byte, 16)
binary.BigEndian.PutUint64(val, st.Index)
binary.BigEndian.PutUint64(val[8:], st.Term)
wb.Put(snapshotKey(), val)
value, err := hs.Marshal()
if err != nil {
return err
}
wb.Put(hardStateKey(), value)
wb.DeleteRange(logKey(0), logKey(math.MaxUint64))
if err := w.db.Write(wo, wb); err != nil {
return err
}
w.st = st
w.hs = hs
w.lastIndex = st.Index
return nil
}
func (w *rocksdbWal) Close() {
w.once.Do(func() {
w.db.Close()
})
}

157
blobstore/common/raftserver/wal/rocksdb_wal_test.go Normal file
View File

@ -0,0 +1,157 @@
// Copyright 2022 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package wal
import (
"math"
"os"
"testing"
"github.com/stretchr/testify/require"
"go.etcd.io/etcd/raft/v3"
pb "go.etcd.io/etcd/raft/v3/raftpb"
)
func TestRocksDBWal(t *testing.T) {
dir := "/tmp/raftwal-" + string(genRandomBytes(8))
clear := func() {
err := os.RemoveAll(dir)
if err != nil {
t.Fatal(err)
}
}
clear()
defer clear()
wal, err := OpenRocksdbWal(dir)
require.Nil(t, err)
entries := make([]pb.Entry, 0, 100000)
for i := 0; i < 100000; i++ {
entry := pb.Entry{
Term: uint64((i + 1) % 1000),
Index: uint64(i + 1),
Type: pb.EntryNormal,
Data: genRandomBytes(1024),
}
entries = append(entries, entry)
}
err = wal.Save(pb.HardState{}, entries)
require.Nil(t, err)
err = wal.Truncate(1000)
require.Nil(t, err)
err = wal.Truncate(1000)
require.Equal(t, raft.ErrCompacted, err)
testCase := []getEntriesTest{
{1, 1, true, raft.ErrUnavailable},
{1000, 1, true, raft.ErrUnavailable},
{1, 2, true, raft.ErrCompacted},
{1000, 2000, true, raft.ErrCompacted},
{1001, 100002, false, nil},
{100001, 1000002, false, nil},
}
for _, tc := range testCase {
_, err = wal.Entries(tc.lo, tc.hi, 64*1024*1024)
if err != nil {
if tc.matchError && err != tc.err {
t.Fatal(err)
}
} else {
t.Fatalf("need return error, dir(%s) testCase(%v)", dir, tc)
}
}
entries, err = wal.Entries(1001, 100000, math.MaxUint64)
require.Nil(t, err)
for i, entry := range entries {
require.Equal(t, uint64(i+1001), entry.Index)
}
_, err = wal.Term(999)
require.Equal(t, err, raft.ErrCompacted)
term, _ := wal.Term(1001)
require.Equal(t, uint64(1), term)
term, _ = wal.Term(1000)
require.Equal(t, uint64(0), term)
_, err = wal.Term(10000000)
require.Equal(t, err, raft.ErrUnavailable)
firstIndex := wal.FirstIndex()
require.Equal(t, uint64(1001), firstIndex)
lastIndex := wal.LastIndex()
require.Equal(t, uint64(100000), lastIndex)
wal.Close()
// reload wal
wal, err = OpenRocksdbWal(dir)
require.Nil(t, err)
require.Equal(t, uint64(100000), wal.LastIndex())
entries = entries[0:0]
for i := 0; i < 1000; i++ {
entry := pb.Entry{
Term: uint64((i + 9990) % 1000),
Index: uint64(i + 9990),
Type: pb.EntryNormal,
Data: genRandomBytes(1024),
}
entries = append(entries, entry)
}
err = wal.Save(pb.HardState{}, entries)
require.Nil(t, err)
lastIndex = wal.LastIndex()
require.Equal(t, uint64(10989), lastIndex)
st := Snapshot{100, 1}
err = wal.ApplySnapshot(st)
require.Nil(t, err)
firstIndex = wal.FirstIndex()
require.Equal(t, uint64(101), firstIndex)
lastIndex = wal.LastIndex()
require.Equal(t, uint64(100), lastIndex)
err = wal.Save(pb.HardState{
Term: 3,
Vote: 1,
Commit: 100,
}, nil)
require.Nil(t, err)
hs := wal.InitialState()
require.Equal(t, uint64(3), hs.Term)
require.Equal(t, uint64(1), hs.Vote)
require.Equal(t, uint64(100), hs.Commit)
err = wal.Save(pb.HardState{
Term: 5,
Vote: 1,
Commit: 200,
}, nil)
require.Nil(t, err)
wal.Close()
wal, err = OpenRocksdbWal(dir)
require.Nil(t, err)
wal.Close()
}

51
blobstore/common/raftserver/wal/wal.go
View File

@ -29,8 +29,20 @@ type Snapshot struct {
Term uint64
}
type Wal interface {
InitialState() pb.HardState
Entries(lo, hi uint64, maxSize uint64) (entries []pb.Entry, err error)
Term(index uint64) (term uint64, err error)
FirstIndex() uint64
LastIndex() uint64
Save(hs pb.HardState, entries []pb.Entry) error
Truncate(index uint64) error
ApplySnapshot(st Snapshot) error
Close()
}
// Storage the storage
type Wal struct {
type fileWal struct {
// Log Entry
sync bool
dir string
@ -45,16 +57,16 @@ type Wal struct {
}
// OpenWal
func OpenWal(dir string, sync bool) (*Wal, error) {
func OpenWal(dir string, sync bool) (Wal, error) {
dir = path.Clean(dir)
if err := InitPath(dir); err != nil {
if err := InitPath(dir, true); err != nil {
return nil, err
}
mt, st, hs, err := NewMeta(dir)
if err != nil {
return nil, err
}
w := &Wal{
w := &fileWal{
sync: sync,
dir: dir,
nextFileSeq: 1,
@ -87,11 +99,11 @@ func OpenWal(dir string, sync bool) (*Wal, error) {
return w, nil
}
func (w *Wal) InitialState() pb.HardState {
func (w *fileWal) InitialState() pb.HardState {
return w.hs
}
func (w *Wal) Entries(lo, hi uint64, maxSize uint64) (entries []pb.Entry, err error) {
func (w *fileWal) Entries(lo, hi uint64, maxSize uint64) (entries []pb.Entry, err error) {
if lo >= hi {
return nil, raft.ErrUnavailable
} else if lo <= w.st.Index {
@ -100,7 +112,7 @@ func (w *Wal) Entries(lo, hi uint64, maxSize uint64) (entries []pb.Entry, err er
return w.entries(lo, hi, maxSize)
}
func (w *Wal) Term(index uint64) (term uint64, err error) {
func (w *fileWal) Term(index uint64) (term uint64, err error) {
if index < w.st.Index {
return 0, raft.ErrCompacted
} else if index == w.st.Index {
@ -109,11 +121,11 @@ func (w *Wal) Term(index uint64) (term uint64, err error) {
return w.term(index)
}
func (w *Wal) FirstIndex() uint64 {
func (w *fileWal) FirstIndex() uint64 {
return w.st.Index + 1
}
func (w *Wal) LastIndex() uint64 {
func (w *fileWal) LastIndex() uint64 {
index := w.lastIndex()
if index < w.st.Index {
index = w.st.Index
@ -121,23 +133,24 @@ func (w *Wal) LastIndex() uint64 {
return index
}
func (w *Wal) SaveEntries(entries []pb.Entry) error {
func (w *fileWal) Save(hs pb.HardState, entries []pb.Entry) error {
if !raft.IsEmptyHardState(hs) {
w.mt.SaveHardState(hs)
w.hs = hs
}
if len(entries) > 0 {
if err := w.saveEntries(entries); err != nil {
return err
}
if w.sync {
return w.Sync()
}
}
return nil
}
func (w *Wal) SaveHardState(hs pb.HardState) error {
w.mt.SaveHardState(hs)
w.hs = hs
return nil
}
func (w *Wal) Truncate(index uint64) error {
func (w *fileWal) Truncate(index uint64) error {
if index <= w.st.Index {
return raft.ErrCompacted
}
@ -165,7 +178,7 @@ func (w *Wal) Truncate(index uint64) error {
return nil
}
func (w *Wal) ApplySnapshot(st Snapshot) error {
func (w *fileWal) ApplySnapshot(st Snapshot) error {
w.st = st
w.hs = pb.HardState{
Commit: st.Index,
@ -182,7 +195,7 @@ func (w *Wal) ApplySnapshot(st Snapshot) error {
return nil
}
func (w *Wal) Close() {
func (w *fileWal) Close() {
w.once.Do(func() {
w.cache.Close()
w.last.Close()

12
blobstore/common/raftserver/wal/wal_test.go
View File

@ -68,7 +68,7 @@ func TestRaftWal(t *testing.T) {
entries = append(entries, entry)
}
err = wal.SaveEntries(entries)
err = wal.Save(pb.HardState{}, entries)
require.Nil(t, err)
err = wal.Truncate(1000)
@ -124,7 +124,7 @@ func TestRaftWal(t *testing.T) {
}
entries = append(entries, entry)
}
err = wal.SaveEntries(entries)
err = wal.Save(pb.HardState{}, entries)
require.Nil(t, err)
lastIndex = wal.LastIndex()
require.Equal(t, uint64(10989), lastIndex)
@ -139,11 +139,11 @@ func TestRaftWal(t *testing.T) {
lastIndex = wal.LastIndex()
require.Equal(t, uint64(100), lastIndex)
err = wal.SaveHardState(pb.HardState{
err = wal.Save(pb.HardState{
Term: 3,
Vote: 1,
Commit: 100,
})
}, nil)
require.Nil(t, err)
hs := wal.InitialState()
@ -151,11 +151,11 @@ func TestRaftWal(t *testing.T) {
require.Equal(t, uint64(1), hs.Vote)
require.Equal(t, uint64(100), hs.Commit)
err = wal.SaveHardState(pb.HardState{
err = wal.Save(pb.HardState{
Term: 5,
Vote: 1,
Commit: 200,
})
}, nil)
require.Nil(t, err)
wal.Close()