Enhancement: auto fix crc-mismatched raft log crc

The last Log file of Raft is in an unindexed state, and Raft will traverse
the file to rebuild the index during the startup process.

When the process was shut down abnormally last time (for example, shut down
by kill -9), the log file data may be incomplete.

This improvement allows raft to perform a truncation operation if an incomplete
log entry is encountered during the indexing process of the last log file,
and the log is truncated to the most recent complete and valid log entry.

Signed-off-by: Mofei Zhang <mofei2816@gmail.com>

vendor/github.com/tiglabs/raft/storage/wal/log_file.go

@@ -16,7 +16,6 @@ package wal
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"os"
 	"path"
@@ -57,7 +56,7 @@ func openLogEntryFile(dir string, name logFileName, isLastOne bool) (*logEntryFi
 	} else {
 		// 重建索引
 		toffset, err := lf.ReBuildIndex()
-		if err != nil {
+		if err != nil && err != io.ErrUnexpectedEOF && !IsErrCorrupt(err) {
 			return nil, err
 		}
 		// 打开写
@@ -65,7 +64,7 @@ func openLogEntryFile(dir string, name logFileName, isLastOne bool) (*logEntryFi
 			return nil, err
 		}
 		// 截断索引及后面的数据
-		if toffset > 0 {
+		if toffset >= 0 {
 			log.Warn("truncate last logfile's N@%d index at: %d", lf.name.seq, toffset)
 			if err := lf.w.Truncate(toffset); err != nil {
 				return nil, err
@@ -165,18 +164,9 @@ func (lf *logEntryFile) ReBuildIndex() (truncateOffset int64, err error) {
 			ent := &proto.Entry{}
 			ent.Decode(rec.data)
 			lf.index = lf.index.Append(uint32(offset), ent)
-		} else if rec.recType == recTypeIndex { // 处理写了index，但是没写footer或者下一个新日志文件没创建
-			var footer footerRecord
-			curIndexSize := int64(recordSize(lf.index))
-			footerSize := int64(recordSize(footer))
-			// index的大小+footer不大于文件大小，则截断
-			if filesize <= offset+curIndexSize+footerSize {
-				return offset, nil
-			} else {
-				return 0, NewCorruptError(lf.f.Name(), offset, "could not truncate last logfile's index")
-			}
 		} else {
-			return 0, NewCorruptError(lf.f.Name(), offset, fmt.Sprintf("wrong log entry record type: %s", rec.recType.String()))
+			// All valid log entries have been loaded
+			return offset, nil
 		}
 	}
 	if err == io.EOF {

vendor/github.com/tiglabs/raft/storage/wal/log_file_test.go

@@ -0,0 +1,540 @@
+// Copyright 2021 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package wal
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/tiglabs/raft/proto"
+)
+
+// This test case is used to test the process logic of LogEntryFile for file fault
+// tolerance when loading a damaged log file.
+func TestAutoFixLastIndexLogEntryFile_BrokenLogEntry(t *testing.T) {
+	var err error
+	var testPath = path.Join(os.TempDir(), "test_auto_fix_broken_log_entry_file")
+	if err = os.MkdirAll(testPath, os.ModePerm); err != nil {
+		t.Fatalf("prepare test path fail: %v", err)
+	}
+	defer func() {
+		_ = os.RemoveAll(testPath)
+	}()
+
+	type __sample struct {
+		Desc       string         // Sample description
+		FileData   []byte         // Sample log file data
+		Entries    []*proto.Entry // Expected valid entries
+		FirstIndex uint64         // Expected first index
+		LastIndex  uint64         // Expected last index
+	}
+
+	var (
+		samples = []*__sample{
+			{
+				Desc: "complete log file",
+				FileData: []byte{
+					0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x43, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a,
+					0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a,
+					0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0xf0,
+					0xdf, 0xd9, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x44, 0x7b, 0x22, 0x6f,
+					0x70, 0x22, 0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22,
+					0x76, 0x22, 0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d,
+					0x22, 0x7d, 0x05, 0x02, 0x5b, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x45,
+					0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x38, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c,
+					0x2c, 0x22, 0x76, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x27, 0x93, 0x59, 0x23, 0x01, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x46, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x32, 0x35,
+					0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a, 0x22, 0x41,
+					0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0x85, 0x81, 0x69,
+					0xa3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x47, 0x7b, 0x22, 0x6f, 0x70, 0x22,
+					0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22,
+					0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d,
+					0x45, 0x41, 0xe4, 0xac,
+				},
+				Entries: []*proto.Entry{
+					{Index: 165699, Term: 369},
+					{Index: 165700, Term: 369},
+					{Index: 165701, Term: 369},
+					{Index: 165702, Term: 369},
+					{Index: 165703, Term: 369},
+				},
+				FirstIndex: 165699,
+				LastIndex:  165703,
+			},
+			{
+				Desc: "broken tail log file",
+				FileData: []byte{
+					0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x43, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a,
+					0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a,
+					0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0xf0,
+					0xdf, 0xd9, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x44, 0x7b, 0x22, 0x6f,
+					0x70, 0x22, 0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22,
+					0x76, 0x22, 0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d,
+					0x22, 0x7d, 0x05, 0x02, 0x5b, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x45,
+					0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x38, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c,
+					0x2c, 0x22, 0x76, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x27, 0x93, 0x59, 0x23, 0x01, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x46, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x32, 0x35,
+					0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a, 0x22, 0x41,
+					0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0x85, 0x81, 0x69,
+					0xa3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x47, 0x7b, 0x22, 0x6f, 0x70, 0x22,
+					0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22,
+					0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d,
+					0x45, 0x41, 0xe4, 0x00, // <- The broken byte here
+				},
+				Entries: []*proto.Entry{
+					{Index: 165699, Term: 369},
+					{Index: 165700, Term: 369},
+					{Index: 165701, Term: 369},
+					{Index: 165702, Term: 369},
+				},
+				FirstIndex: 165699,
+				LastIndex:  165702,
+			},
+			{
+				Desc: "broken tail log file (missing last one byte)",
+				FileData: []byte{
+					0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x43, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a,
+					0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a,
+					0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0xf0,
+					0xdf, 0xd9, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x44, 0x7b, 0x22, 0x6f,
+					0x70, 0x22, 0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22,
+					0x76, 0x22, 0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d,
+					0x22, 0x7d, 0x05, 0x02, 0x5b, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x45,
+					0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x38, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c,
+					0x2c, 0x22, 0x76, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x27, 0x93, 0x59, 0x23, 0x01, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x46, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x32, 0x35,
+					0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a, 0x22, 0x41,
+					0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0x85, 0x81, 0x69,
+					0xa3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x47, 0x7b, 0x22, 0x6f, 0x70, 0x22,
+					0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22,
+					0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d,
+					0x45, 0x41, 0xe4, // <- Missing last one byte.
+				},
+				Entries: []*proto.Entry{
+					{Index: 165699, Term: 369},
+					{Index: 165700, Term: 369},
+					{Index: 165701, Term: 369},
+					{Index: 165702, Term: 369},
+				},
+				FirstIndex: 165699,
+				LastIndex:  165702,
+			},
+			{
+				Desc: "broken head log file",
+				FileData: []byte{
+					// The first byte is corrupted
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x43, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a,
+					0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a,
+					0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0xf0,
+					0xdf, 0xd9, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x44, 0x7b, 0x22, 0x6f,
+					0x70, 0x22, 0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22,
+					0x76, 0x22, 0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d,
+					0x22, 0x7d, 0x05, 0x02, 0x5b, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x45,
+					0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x38, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c,
+					0x2c, 0x22, 0x76, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x27, 0x93, 0x59, 0x23, 0x01, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x46, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x32, 0x35,
+					0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a, 0x22, 0x41,
+					0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0x85, 0x81, 0x69,
+					0xa3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x47, 0x7b, 0x22, 0x6f, 0x70, 0x22,
+					0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22,
+					0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d,
+					0x45, 0x41, 0xe4, 0xac,
+				},
+				Entries:    []*proto.Entry{},
+				FirstIndex: 0,
+				LastIndex:  0,
+			},
+			{
+				Desc: "broken middle log file",
+				FileData: []byte{
+					0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x43, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a,
+					0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a,
+					0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0xf0,
+					0xdf, 0xd9, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x44, 0x7b, 0x22, 0x6f,
+					0x70, 0x22, 0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22,
+					0x76, 0x22, 0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d,
+					0x22, 0x7d, 0x05, 0x02, 0x5b, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x45,
+					0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x38, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x00, // <- The broken byte here
+					0x2c, 0x22, 0x76, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x7d, 0x27, 0x93, 0x59, 0x23, 0x01, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x71,
+					0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x46, 0x7b, 0x22, 0x6f, 0x70, 0x22, 0x3a, 0x32, 0x35,
+					0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22, 0x3a, 0x22, 0x41,
+					0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d, 0x85, 0x81, 0x69,
+					0xa3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x00, 0x01, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x87, 0x47, 0x7b, 0x22, 0x6f, 0x70, 0x22,
+					0x3a, 0x32, 0x35, 0x2c, 0x22, 0x6b, 0x22, 0x3a, 0x6e, 0x75, 0x6c, 0x6c, 0x2c, 0x22, 0x76, 0x22,
+					0x3a, 0x22, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x73, 0x41, 0x41, 0x51, 0x30, 0x3d, 0x22, 0x7d,
+					0x45, 0x41, 0xe4, 0xac,
+				},
+				Entries: []*proto.Entry{
+					{Index: 165699, Term: 369},
+					{Index: 165700, Term: 369},
+				},
+				FirstIndex: 165699,
+				LastIndex:  165700,
+			},
+		}
+		testLogFileName = "0000000000000000-0000000000000001.log"
+	)
+
+	for _, sample := range samples {
+		// Prepare test data and write it to test log file.
+		if err = ioutil.WriteFile(path.Join(testPath, testLogFileName), sample.FileData, os.ModePerm); err != nil {
+			t.Fatalf("[%v] prepare test file data fail: %v", sample.Desc, err)
+		}
+
+		// Load test log file
+		logFileName := logFileName{}
+		logFileName.ParseFrom(testLogFileName)
+
+		for testCount := 1; testCount <= 3; testCount++ {
+			var lf *logEntryFile
+			if lf, err = openLogEntryFile(
+				testPath,
+				logFileName,
+				true); err != nil {
+				t.Fatalf("[%v %v] open test log file fail: %v", sample.Desc, testCount, err)
+			}
+
+			// Validate first index
+			fi := lf.FirstIndex()
+			if fi != sample.FirstIndex {
+				t.Fatalf("[%v %v] first index mismatch: expect %v, actual %v", sample.Desc, testCount, sample.FirstIndex, fi)
+			}
+
+			// Validate last index
+			li := lf.LastIndex()
+			if li != sample.LastIndex {
+				t.Fatalf("[%v %v] last index mismatch: expect %v, actual %v", sample.Desc, testCount, sample.LastIndex, li)
+			}
+
+			// Validate each log entry
+			var e *proto.Entry
+			var entryIndex = 0
+			for i := fi; i > 0 && i <= li; i++ {
+				if e, err = lf.Get(i); err != nil {
+					t.Fatalf("[%v %v] get log entry fail: index %v, %v", sample.Desc, testCount, i, err)
+				}
+				var sampleEntry = sample.Entries[entryIndex]
+				if e.Index != sampleEntry.Index || e.Term != sampleEntry.Term || e.Type != sampleEntry.Type {
+					t.Fatalf("[%v %v] entry data mismatch: expact %v, actual %v",
+						sample.Desc, testCount,
+						fmt.Sprintf("%v_%v_%v", sampleEntry.Index, sampleEntry.Term, sampleEntry.Type),
+						fmt.Sprintf("%v_%v_%v", e.Index, e.Term, e.Type))
+				}
+				entryIndex++
+			}
+			_ = lf.Close()
+		}
+
+		_ = os.Remove(path.Join(testPath, testLogFileName))
+	}
+}
+
+func TestAutoFixLastIndexLogEntryFile_BrokenIndex_MissingLastOneByte(t *testing.T) {
+	var err error
+	var testPath = path.Join(os.TempDir(), "test_auto_fix_broken_index_log_entry_file")
+	if err = os.MkdirAll(testPath, os.ModePerm); err != nil {
+		t.Fatalf("prepare test path fail: %v", err)
+	}
+	defer func() {
+		_ = os.RemoveAll(testPath)
+	}()
+
+	var (
+		sampleEntries = []*proto.Entry{
+			{Term: 1, Index: 1},
+			{Term: 1, Index: 2},
+			{Term: 1, Index: 3},
+			{Term: 1, Index: 4},
+		}
+		firstIndex      uint64 = 1
+		lastIndex       uint64 = 4
+		testLogFileName        = "0000000000000000-0000000000000001.log"
+	)
+
+	// Prepare test log file
+	if err = generateLogEntryFileWithIndex(testPath, testLogFileName, sampleEntries); err != nil {
+		return
+	}
+
+	// Truncate last byte
+	var info os.FileInfo
+	if info, err = os.Stat(path.Join(testPath, testLogFileName)); err != nil {
+		t.Fatalf("stat test log file fail: %v", err)
+	}
+	if info.Size() == 0 {
+		t.Fatalf("test log file size is 0")
+	}
+	if err = os.Truncate(path.Join(testPath, testLogFileName), info.Size()-1); err != nil {
+		t.Fatalf("truncate test log file fail: %v", err)
+	}
+
+	// Load and validate log file
+	logFileName := logFileName{}
+	logFileName.ParseFrom(testLogFileName)
+	for testCount := 1; testCount <= 3; testCount++ {
+		var lf *logEntryFile
+		if lf, err = openLogEntryFile(
+			testPath,
+			logFileName,
+			true); err != nil {
+			t.Fatalf("open test log file fail: %v", err)
+		}
+
+		// Validate first index
+		fi := lf.FirstIndex()
+		if fi != firstIndex {
+			t.Fatalf("first index mismatch: expect %v, actual %v", firstIndex, fi)
+		}
+
+		// Validate last index
+		li := lf.LastIndex()
+		if li != lastIndex {
+			t.Fatalf("last index mismatch: expect %v, actual %v", lastIndex, li)
+		}
+
+		// Validate each log entry
+		var e *proto.Entry
+		var entryIndex = 0
+		for i := fi; i > 0 && i <= li; i++ {
+			if e, err = lf.Get(i); err != nil {
+				t.Fatalf("get log entry fail: index %v, %v", i, err)
+			}
+			var sampleEntry = sampleEntries[entryIndex]
+			if e.Index != sampleEntry.Index || e.Term != sampleEntry.Term || e.Type != sampleEntry.Type {
+				t.Fatalf("entry data mismatch: expact %v, actual %v",
+					fmt.Sprintf("%v_%v_%v", sampleEntry.Index, sampleEntry.Term, sampleEntry.Type),
+					fmt.Sprintf("%v_%v_%v", e.Index, e.Term, e.Type))
+			}
+			entryIndex++
+		}
+		_ = lf.Close()
+	}
+}
+
+func TestAutoFixLastIndexLogEntryFile_BrokenIndex_BrokenLastOneByte(t *testing.T) {
+	var err error
+	var testPath = path.Join(os.TempDir(), "test_auto_fix_broken_index_log_entry_file")
+	if err = os.MkdirAll(testPath, os.ModePerm); err != nil {
+		t.Fatalf("prepare test path fail: %v", err)
+	}
+	defer func() {
+		_ = os.RemoveAll(testPath)
+	}()
+
+	var (
+		sampleEntries = []*proto.Entry{
+			{Term: 1, Index: 1},
+			{Term: 1, Index: 2},
+			{Term: 1, Index: 3},
+			{Term: 1, Index: 4},
+		}
+		firstIndex      uint64 = 1
+		lastIndex       uint64 = 4
+		testLogFileName        = "0000000000000000-0000000000000001.log"
+	)
+
+	// Prepare test log file
+	if err = generateLogEntryFileWithIndex(testPath, testLogFileName, sampleEntries); err != nil {
+		return
+	}
+
+	// Overwrite last one byte
+	var f *os.File
+	if f, err = os.OpenFile(path.Join(testPath, testLogFileName), os.O_RDWR, os.ModePerm); err != nil {
+		t.Fatalf("open log file fail: %v", err)
+	}
+	var stat os.FileInfo
+	if stat, err = f.Stat(); err != nil {
+		t.Fatalf("stat log file fail: %v", err)
+	}
+	if _, err = f.WriteAt([]byte{0xff}, stat.Size()-1); err != nil {
+		t.Fatalf("overwrite log file fail: %v", err)
+	}
+	_ = f.Sync()
+	_ = f.Close()
+
+	// Load and validate log file
+	logFileName := logFileName{}
+	logFileName.ParseFrom(testLogFileName)
+	for testCount := 1; testCount <= 3; testCount++ {
+		var lf *logEntryFile
+		if lf, err = openLogEntryFile(
+			testPath,
+			logFileName,
+			true); err != nil {
+			t.Fatalf("open test log file fail: %v", err)
+		}
+
+		// Validate first index
+		fi := lf.FirstIndex()
+		if fi != firstIndex {
+			t.Fatalf("first index mismatch: expect %v, actual %v", firstIndex, fi)
+		}
+
+		// Validate last index
+		li := lf.LastIndex()
+		if li != lastIndex {
+			t.Fatalf("last index mismatch: expect %v, actual %v", lastIndex, li)
+		}
+
+		// Validate each log entry
+		var e *proto.Entry
+		var entryIndex = 0
+		for i := fi; i > 0 && i <= li; i++ {
+			if e, err = lf.Get(i); err != nil {
+				t.Fatalf("get log entry fail: index %v, %v", i, err)
+			}
+			var sampleEntry = sampleEntries[entryIndex]
+			if e.Index != sampleEntry.Index || e.Term != sampleEntry.Term || e.Type != sampleEntry.Type {
+				t.Fatalf("entry data mismatch: expact %v, actual %v",
+					fmt.Sprintf("%v_%v_%v", sampleEntry.Index, sampleEntry.Term, sampleEntry.Type),
+					fmt.Sprintf("%v_%v_%v", e.Index, e.Term, e.Type))
+			}
+			entryIndex++
+		}
+		_ = lf.Close()
+	}
+}
+
+func TestAutoFixLastIndexLogEntryFile_BrokenIndex_OneMoreByteAtTheEnd(t *testing.T) {
+	var err error
+	var testPath = path.Join(os.TempDir(), "test_auto_fix_broken_index_log_entry_file")
+	if err = os.MkdirAll(testPath, os.ModePerm); err != nil {
+		t.Fatalf("prepare test path fail: %v", err)
+	}
+	defer func() {
+		_ = os.RemoveAll(testPath)
+	}()
+
+	var (
+		sampleEntries = []*proto.Entry{
+			{Term: 1, Index: 1},
+			{Term: 1, Index: 2},
+			{Term: 1, Index: 3},
+			{Term: 1, Index: 4},
+		}
+		firstIndex      uint64 = 1
+		lastIndex       uint64 = 4
+		testLogFileName        = "0000000000000000-0000000000000001.log"
+	)
+
+	// Prepare test log file
+	if err = generateLogEntryFileWithIndex(testPath, testLogFileName, sampleEntries); err != nil {
+		return
+	}
+
+	// Append one garbage byte to log file
+	var f *os.File
+	if f, err = os.OpenFile(path.Join(testPath, testLogFileName), os.O_RDWR|os.O_APPEND, os.ModePerm); err != nil {
+		t.Fatalf("append garbage byte to log file fail: %v", err)
+	}
+	if _, err = f.Write([]byte{0x00}); err != nil {
+		t.Fatalf("append garbage byte to log file fail: %v", err)
+	}
+	_ = f.Sync()
+	_ = f.Close()
+
+	// Load and validate log file
+	logFileName := logFileName{}
+	logFileName.ParseFrom(testLogFileName)
+	for testCount := 1; testCount <= 3; testCount++ {
+		var lf *logEntryFile
+		if lf, err = openLogEntryFile(
+			testPath,
+			logFileName,
+			true); err != nil {
+			t.Fatalf("open test log file fail: %v", err)
+		}
+
+		// Validate first index
+		fi := lf.FirstIndex()
+		if fi != firstIndex {
+			t.Fatalf("first index mismatch: expect %v, actual %v", firstIndex, fi)
+		}
+
+		// Validate last index
+		li := lf.LastIndex()
+		if li != lastIndex {
+			t.Fatalf("last index mismatch: expect %v, actual %v", lastIndex, li)
+		}
+
+		// Validate each log entry
+		var e *proto.Entry
+		var entryIndex = 0
+		for i := fi; i > 0 && i <= li; i++ {
+			if e, err = lf.Get(i); err != nil {
+				t.Fatalf("get log entry fail: index %v, %v", i, err)
+			}
+			var sampleEntry = sampleEntries[entryIndex]
+			if e.Index != sampleEntry.Index || e.Term != sampleEntry.Term || e.Type != sampleEntry.Type {
+				t.Fatalf("entry data mismatch: expact %v, actual %v",
+					fmt.Sprintf("%v_%v_%v", sampleEntry.Index, sampleEntry.Term, sampleEntry.Type),
+					fmt.Sprintf("%v_%v_%v", e.Index, e.Term, e.Type))
+			}
+			entryIndex++
+		}
+		_ = lf.Close()
+	}
+}
+
+func generateLogEntryFileWithIndex(path, name string, entries []*proto.Entry) (err error) {
+	logFileName := logFileName{}
+	logFileName.ParseFrom(name)
+
+	var lf *logEntryFile
+	if lf, err = createLogEntryFile(path, logFileName); err != nil {
+		return
+	}
+
+	for _, entry := range entries {
+		if err = lf.Save(entry); err != nil {
+			return
+		}
+	}
+
+	if err = lf.FinishWrite(); err != nil {
+		return
+	}
+	err = lf.Close()
+	return
+}

vendor/github.com/tiglabs/raft/storage/wal/record.go

@@ -47,6 +47,14 @@ func NewCorruptError(filename string, offset int64, reason string) *ErrCorrupt {
 	}
 }
 
+func IsErrCorrupt(err error) (is bool) {
+	if err == nil {
+		return
+	}
+	_, is = err.(*ErrCorrupt)
+	return
+}
+
 type recordType uint8
 
 const (
@@ -55,6 +63,15 @@ const (
 	recTypeFooter   recordType = 3
 )
 
+func (rt recordType) Valid() bool {
+	switch rt {
+	case recTypeLogEntry, recTypeIndex, recTypeFooter:
+		return true
+	default:
+	}
+	return false
+}
+
 func (rt recordType) String() string {
 	switch rt {
 	case recTypeLogEntry:

vendor/github.com/tiglabs/raft/storage/wal/record_reader.go

@@ -18,6 +18,7 @@ import (
 	"bufio"
 	"encoding/binary"
 	"io"
+	"math"
 	"os"
 
 	"github.com/tiglabs/raft/util"
@@ -98,8 +99,20 @@ func (r *recordReader) Read() (recStartOffset int64, rec record, err error) {
 		}
 		return
 	}
+
+	// Decode and validate record type
 	rec.recType = recordType(r.typeLenBuf[0])
+	if !rec.recType.Valid() {
+		err = NewCorruptError(r.filename, recStartOffset, "illegal record type")
+		return
+	}
+
+	// Decode and validate record data length
 	rec.dataLen = binary.BigEndian.Uint64(r.typeLenBuf[1:])
+	if rec.dataLen+4 <= 0 || rec.dataLen > math.MaxUint32 {
+		err = NewCorruptError(r.filename, recStartOffset, "illegal data length")
+		return
+	}
 
 	// read data and crc
 	// WARN：不可以用buffer pool，因为log entry等decode时没有进行拷贝