fix(blobnode): happen EIO when handle shard/chunk io, return broken disk

with: #1000196614

Signed-off-by: mawei029 <mawei2@oppo.com>

blobstore/api/blobnode/shard.go

@@ -243,7 +243,8 @@ func (c *client) MarkDeleteShard(ctx context.Context, host string, args *DeleteS
 
 	urlStr := fmt.Sprintf("%v/shard/markdelete/diskid/%v/vuid/%v/bid/%v", host, args.DiskID, args.Vuid, args.Bid)
 	err = c.PostWith(ctx, urlStr, nil, rpc.NoneBody)
-	return
+
+	return convertEIO(err)
 }
 
 func (c *client) DeleteShard(ctx context.Context, host string, args *DeleteShardArgs) (err error) {
@@ -254,7 +255,8 @@ func (c *client) DeleteShard(ctx context.Context, host string, args *DeleteShard
 
 	urlStr := fmt.Sprintf("%v/shard/delete/diskid/%v/vuid/%v/bid/%v", host, args.DiskID, args.Vuid, args.Bid)
 	err = c.PostWith(ctx, urlStr, nil, rpc.NoneBody)
-	return
+
+	return convertEIO(err)
 }
 
 type StatShardArgs struct {
@@ -273,7 +275,8 @@ func (c *client) StatShard(ctx context.Context, host string, args *StatShardArgs
 		host, args.DiskID, args.Vuid, args.Bid)
 	si = &ShardInfo{}
 	err = c.GetWith(ctx, urlStr, si)
-	return
+
+	return si, convertEIO(err)
 }
 
 type ListShardsArgs struct {
@@ -301,6 +304,7 @@ func (c *client) ListShards(ctx context.Context, host string, args *ListShardsAr
 	listRet := ListShardsRet{}
 	err = c.GetWith(ctx, urlStr, &listRet)
 	if err != nil {
+		err = convertEIO(err)
 		return nil, proto.InValidBlobID, err
 	}
 

blobstore/api/blobnode/shard_test.go

@@ -15,13 +15,26 @@
 package blobnode
 
 import (
+	"syscall"
 	"testing"
 
+	"github.com/cubefs/cubefs/util/errors"
 	"github.com/stretchr/testify/require"
+
+	bloberr "github.com/cubefs/cubefs/blobstore/common/errors"
 )
 
 func TestShardStatus(t *testing.T) {
 	require.Equal(t, ShardStatusDefault, ShardStatus(0))
 	require.Equal(t, ShardStatusNormal, ShardStatus(1))
 	require.Equal(t, ShardStatusMarkDelete, ShardStatus(2))
+
+	var err error
+	require.Nil(t, convertEIO(err))
+
+	err = syscall.EIO
+	require.ErrorIs(t, convertEIO(err), bloberr.ErrDiskBroken)
+
+	err = errors.New("input/output error")
+	require.ErrorIs(t, convertEIO(err), bloberr.ErrDiskBroken)
 }

blobstore/blobnode/chunk.go

@@ -73,6 +73,11 @@ func (s *Service) ChunkCreate(c *rpc.Context) {
 		return
 	}
 
+	if !ds.IsWritable() {
+		c.RespondError(bloberr.ErrDiskBroken)
+		return
+	}
+
 	cs, err := ds.CreateChunk(ctx, args.Vuid, args.ChunkSize)
 	if err != nil {
 		span.Errorf("Failed register vuid:%v, err:%v", args.DiskID, err)
@@ -111,6 +116,11 @@ func (s *Service) ChunkInspect(c *rpc.Context) {
 		return
 	}
 
+	if !ds.IsWritable() {
+		c.RespondError(bloberr.ErrDiskBroken)
+		return
+	}
+
 	err := s.InspectLimiterPerKey.Acquire(args.Vuid)
 	if err != nil {
 		c.RespondError(bloberr.ErrOutOfLimit)

blobstore/blobnode/compact.go

@@ -53,6 +53,11 @@ func (s *Service) ChunkCompact(c *rpc.Context) {
 		return
 	}
 
+	if !ds.IsWritable() {
+		c.RespondError(bloberr.ErrDiskBroken)
+		return
+	}
+
 	cs, exist := ds.GetChunkStorage(args.Vuid)
 	if !exist {
 		span.Errorf("vuid:%v not exist", args.Vuid)

blobstore/blobnode/shard.go

@@ -285,6 +285,11 @@ func (s *Service) ShardList(c *rpc.Context) {
 		return
 	}
 
+	if !ds.IsWritable() {
+		c.RespondError(bloberr.ErrDiskBroken)
+		return
+	}
+
 	cs, exist := ds.GetChunkStorage(args.Vuid)
 	if !exist {
 		span.Errorf("vuid:%d not exist, diskID:%d", args.Vuid, args.DiskID)
@@ -334,6 +339,11 @@ func (s *Service) ShardStat(c *rpc.Context) {
 		return
 	}
 
+	if !ds.IsWritable() {
+		c.RespondError(bloberr.ErrDiskBroken)
+		return
+	}
+
 	cs, exist := ds.GetChunkStorage(args.Vuid)
 	if !exist {
 		span.Errorf("vuid<%d> not exist.args: %v", args.Vuid, args)