cubefs/datanode/server_handler.go

// Copyright 2018 The CubeFS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

package datanode

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path"
	"runtime/debug"
	"strconv"
	"sync"
	"sync/atomic"
	"syscall"

	"github.com/cubefs/cubefs/cmd/common"
	"github.com/cubefs/cubefs/datanode/storage"
	"github.com/cubefs/cubefs/depends/tiglabs/raft"
	"github.com/cubefs/cubefs/proto"
	"github.com/cubefs/cubefs/util"
	"github.com/cubefs/cubefs/util/config"
	"github.com/cubefs/cubefs/util/log"
)

const (
	defaultGOGCLowerLimit = 30
	defaultGOGCUpperLimit = 100
)

var parseArgs = common.ParseArguments

var AutoRepairStatus = true

func (s *DataNode) getDiskAPI(w http.ResponseWriter, r *http.Request) {
	disks := make([]interface{}, 0)
	for _, diskItem := range s.space.GetDisks() {
		disk := &struct {
			Path         string `json:"path"`
			Total        uint64 `json:"total"`
			Used         uint64 `json:"used"`
			Available    uint64 `json:"available"`
			Unallocated  uint64 `json:"unallocated"`
			Allocated    uint64 `json:"allocated"`
			Status       int    `json:"status"`
			RestSize     uint64 `json:"restSize"`
			DiskRdoSize  uint64 `json:"diskRdoSize"`
			Partitions   int    `json:"partitions"`
			Decommission bool   `json:"decommission"`
			IsLost       bool   `json:"isLost"`
		}{
			Path:         diskItem.Path,
			Total:        diskItem.Total,
			Used:         diskItem.Used,
			Available:    diskItem.Available,
			Unallocated:  diskItem.Unallocated,
			Allocated:    diskItem.Allocated,
			Status:       diskItem.Status,
			RestSize:     diskItem.ReservedSpace,
			DiskRdoSize:  diskItem.DiskRdonlySpace,
			Partitions:   diskItem.PartitionCount(),
			Decommission: diskItem.GetDecommissionStatus(),
			IsLost:       diskItem.isLost,
		}
		disks = append(disks, disk)
	}
	diskReport := &struct {
		Disks []interface{} `json:"disks"`
		Zone  string        `json:"zone"`
	}{
		Disks: disks,
		Zone:  s.zoneName,
	}
	s.buildSuccessResp(w, diskReport)
}

func (s *DataNode) getStatAPI(w http.ResponseWriter, r *http.Request) {
	response := &proto.DataNodeHeartbeatResponse{}
	forbiddenVols := make(map[string]struct{})
	volDpRepairBlockSizes := make(map[string]uint64)

	s.buildHeartBeatResponse(response, forbiddenVols, volDpRepairBlockSizes, "")

	s.buildSuccessResp(w, response)
}

func (s *DataNode) setAutoRepairStatus(w http.ResponseWriter, r *http.Request) {
	var autoRepair common.Bool
	if err := parseArgs(r, autoRepair.Key("autoRepair")); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	AutoRepairStatus = autoRepair.V
	s.buildSuccessResp(w, autoRepair.V)
}

func (s *DataNode) getRaftStatus(w http.ResponseWriter, r *http.Request) {
	var raftID common.Uint
	if err := parseArgs(r, raftID.Key("raftID")); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	raftStatus := s.raftStore.RaftStatus(raftID.V)
	s.buildSuccessResp(w, raftStatus)
}

func (s *DataNode) getPartitionsAPI(w http.ResponseWriter, r *http.Request) {
	partitions := make([]interface{}, 0)
	var lock sync.Mutex
	s.space.RangePartitions(func(dp *DataPartition, testID string) bool {
		partition := &struct {
			ID       uint64   `json:"id"`
			Size     int      `json:"size"`
			Used     int      `json:"used"`
			Status   int      `json:"status"`
			Path     string   `json:"path"`
			Replicas []string `json:"replicas"`
			Hosts    []string `json:"hosts"`
		}{
			ID:       dp.partitionID,
			Size:     dp.Size(),
			Used:     dp.Used(),
			Status:   dp.Status(),
			Path:     dp.Path(),
			Replicas: dp.Replicas(),
			Hosts:    dp.getConfigHosts(),
		}
		lock.Lock()
		partitions = append(partitions, partition)
		lock.Unlock()
		return true
	}, "")
	result := &struct {
		Partitions     []interface{} `json:"partitions"`
		PartitionCount int           `json:"partitionCount"`
	}{
		Partitions:     partitions,
		PartitionCount: len(partitions),
	}
	s.buildSuccessResp(w, result)
}

func (s *DataNode) getPartitionAPI(w http.ResponseWriter, r *http.Request) {
	var (
		pid                  common.Uint
		files                []*storage.ExtentInfo
		err                  error
		tinyDeleteRecordSize int64
		raftSt               *raft.Status
	)
	if err := parseArgs(r, pid.ID()); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partition := s.space.Partition(pid.V)
	if partition == nil {
		s.buildFailureResp(w, http.StatusNotFound, "partition not exist")
		return
	}
	if files, tinyDeleteRecordSize, err = partition.ExtentStore().GetAllWatermarks(nil); err != nil {
		err = fmt.Errorf("get watermark fail: %v", err)
		s.buildFailureResp(w, http.StatusInternalServerError, err.Error())
		return
	}

	if !partition.IsDataPartitionLoadFin() {
		raftSt = &raft.Status{Stopped: true}
	} else {
		raftSt = partition.raftPartition.Status()
	}

	result := &struct {
		VolName              string                `json:"volName"`
		ID                   uint64                `json:"id"`
		Size                 int                   `json:"size"`
		Used                 int                   `json:"used"`
		Status               int                   `json:"status"`
		Path                 string                `json:"path"`
		Files                []*storage.ExtentInfo `json:"extents"`
		FileCount            int                   `json:"fileCount"`
		Replicas             []string              `json:"replicas"`
		TinyDeleteRecordSize int64                 `json:"tinyDeleteRecordSize"`
		RaftStatus           *raft.Status          `json:"raftStatus"`
	}{
		VolName:              partition.volumeID,
		ID:                   partition.partitionID,
		Size:                 partition.Size(),
		Used:                 partition.Used(),
		Status:               partition.Status(),
		Path:                 partition.Path(),
		Files:                files,
		FileCount:            len(files),
		Replicas:             partition.Replicas(),
		TinyDeleteRecordSize: tinyDeleteRecordSize,
		RaftStatus:           raftSt,
	}

	if partition.isNormalType() && partition.raftPartition != nil {
		result.RaftStatus = partition.raftPartition.Status()
	}

	s.buildSuccessResp(w, result)
}

func (s *DataNode) getExtentAPI(w http.ResponseWriter, r *http.Request) {
	var (
		pid        common.Uint
		eid        common.Int
		err        error
		extentInfo *storage.ExtentInfo
	)
	if err = parseArgs(r, pid.PartitionID(), eid.ExtentID()); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partition := s.space.Partition(pid.V)
	if partition == nil {
		s.buildFailureResp(w, http.StatusNotFound, "partition not exist")
		return
	}
	if extentInfo, err = partition.ExtentStore().Watermark(uint64(eid.V)); err != nil {
		s.buildFailureResp(w, 500, err.Error())
		return
	}

	s.buildSuccessResp(w, extentInfo)
}

func (s *DataNode) getBlockCrcAPI(w http.ResponseWriter, r *http.Request) {
	var (
		pid    common.Uint
		eid    common.Int
		err    error
		blocks []*storage.BlockCrc
	)
	if err = parseArgs(r, pid.PartitionID(), eid.ExtentID()); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partition := s.space.Partition(pid.V)
	if partition == nil {
		s.buildFailureResp(w, http.StatusNotFound, "partition not exist")
		return
	}
	if blocks, err = partition.ExtentStore().ScanBlocks(uint64(eid.V)); err != nil {
		s.buildFailureResp(w, 500, err.Error())
		return
	}
	s.buildSuccessResp(w, blocks)
}

func (s *DataNode) getTinyDeleted(w http.ResponseWriter, r *http.Request) {
	var (
		pid        common.Uint
		err        error
		extentInfo []storage.ExtentDeleted
	)
	if err = parseArgs(r, pid.ID()); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partition := s.space.Partition(pid.V)
	if partition == nil {
		s.buildFailureResp(w, http.StatusNotFound, "partition not exist")
		return
	}
	if extentInfo, err = partition.ExtentStore().GetHasDeleteTinyRecords(); err != nil {
		s.buildFailureResp(w, 500, err.Error())
		return
	}
	s.buildSuccessResp(w, extentInfo)
}

func (s *DataNode) getNormalDeleted(w http.ResponseWriter, r *http.Request) {
	var (
		pid        common.Uint
		err        error
		extentInfo []storage.ExtentDeleted
	)
	if err = parseArgs(r, pid.ID()); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partition := s.space.Partition(pid.V)
	if partition == nil {
		s.buildFailureResp(w, http.StatusNotFound, "partition not exist")
		return
	}
	if extentInfo, err = partition.ExtentStore().GetHasDeleteExtent(); err != nil {
		s.buildFailureResp(w, 500, err.Error())
		return
	}
	s.buildSuccessResp(w, extentInfo)
}

func (s *DataNode) setQosEnable() func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		var enable common.Bool
		var qosType common.String
		if err := parseArgs(r, enable.Enable(), qosType.QosType()); err != nil {
			s.buildFailureResp(w, http.StatusBadRequest, err.Error())
			return
		}
		switch qosType.V {
		case "normal":
			s.diskQosEnable = enable.V
		case "async":
			s.diskAsyncQosEnable = enable.V
		default:
			err := fmt.Errorf("qos type %v not exist", qosType.V)
			s.buildFailureResp(w, http.StatusBadRequest, err.Error())
			return
		}
		s.buildSuccessResp(w, fmt.Sprintf("Successfully set %v QosEnable to %v", qosType.V, enable.V))
	}
}

func (s *DataNode) setDiskQos(w http.ResponseWriter, r *http.Request) {
	if err := r.ParseForm(); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	parser := func(key string) (val int, err error, has bool) {
		valStr := r.FormValue(key)
		if valStr == "" {
			return 0, nil, false
		}
		has = true
		val, err = strconv.Atoi(valStr)
		return
	}

	updated := false
	for key, pVal := range map[string]*int{
		ConfigDiskReadIocc:       &s.diskReadIocc,
		ConfigDiskReadIops:       &s.diskReadIops,
		ConfigDiskReadFlow:       &s.diskReadFlow,
		ConfigDiskWriteIocc:      &s.diskWriteIocc,
		ConfigDiskWriteIops:      &s.diskWriteIops,
		ConfigDiskWriteFlow:      &s.diskWriteFlow,
		ConfigDiskWQueFactor:     &s.diskWQueFactor,
		ConfigDiskAsyncReadIocc:  &s.diskAsyncReadIocc,
		ConfigDiskAsyncReadIops:  &s.diskAsyncReadIops,
		ConfigDiskAsyncReadFlow:  &s.diskAsyncReadFlow,
		ConfigDiskAsyncWriteIocc: &s.diskAsyncWriteIocc,
		ConfigDiskAsyncWriteIops: &s.diskAsyncWriteIops,
		ConfigDiskAsyncWriteFlow: &s.diskAsyncWriteFlow,
		ConfigDiskDeleteIocc:     &s.diskDeleteIocc,
		ConfigDiskDeleteIops:     &s.diskDeleteIops,
		ConfigDiskDeleteFlow:     &s.diskDeleteFlow,
	} {
		val, err, has := parser(key)
		if err != nil {
			s.buildFailureResp(w, http.StatusBadRequest, err.Error())
			return
		}
		if has {
			updated = true
			*pVal = val
		}
	}

	if updated {
		s.updateQosLimit()
	}
	s.buildSuccessResp(w, "success")
}

func (s *DataNode) getDiskQos(w http.ResponseWriter, r *http.Request) {
	disks := make([]interface{}, 0)
	for _, diskItem := range s.space.GetDisks() {
		disk := &struct {
			Path           string             `json:"path"`
			QosEnable      bool               `json:"qosEnable"`
			AsyncQosEnable bool               `json:"asyncQosEnable"`
			Read           util.LimiterStatus `json:"read"`
			Write          util.LimiterStatus `json:"write"`
			AsyncRead      util.LimiterStatus `json:"asyncRead"`
			AsyncWrite     util.LimiterStatus `json:"asyncWrite"`
			Delete         util.LimiterStatus `json:"delete"`
			IopsStatus     proto.IopsStatus   `json:"IopsStatus"`
		}{
			Path:           diskItem.Path,
			QosEnable:      s.diskQosEnable || s.diskQosEnableFromMaster,
			AsyncQosEnable: s.diskAsyncQosEnable,
			Read:           diskItem.limitRead.Status(false),
			Write:          diskItem.limitWrite.Status(false),
			AsyncRead:      diskItem.limitAsyncRead.Status(false),
			AsyncWrite:     diskItem.limitAsyncWrite.Status(false),
			Delete:         diskItem.limitDelete.Status(false),
			IopsStatus:     s.IopsStatus(),
		}
		disks = append(disks, disk)
	}
	diskStatus := &struct {
		Disks []interface{} `json:"disks"`
		Zone  string        `json:"zone"`
	}{
		Disks: disks,
		Zone:  s.zoneName,
	}
	s.buildSuccessResp(w, diskStatus)
}

func (s *DataNode) getSmuxPoolStat() func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if !s.enableSmuxConnPool {
			s.buildFailureResp(w, 500, "smux pool not supported")
			return
		}
		if s.smuxConnPool == nil {
			s.buildFailureResp(w, 500, "smux pool now is nil")
			return
		}
		stat := s.smuxConnPool.GetStat()
		s.buildSuccessResp(w, stat)
	}
}

func (s *DataNode) setMetricsDegrade(w http.ResponseWriter, r *http.Request) {
	key := "level"
	var level common.Int
	if err := parseArgs(r, level.Key(key).OmitEmpty().
		OnError(func() error {
			w.Write([]byte("Set metrics degrade level parse failed\n"))
			return nil
		}).
		OnValue(func() error {
			atomic.StoreInt64(&s.metricsDegrade, level.V)
			w.Write([]byte(fmt.Sprintf("Set metrics degrade level to %d successfully\n", level.V)))
			return nil
		})); err != nil {
		w.Write([]byte(err.Error()))
		return
	}
}

func (s *DataNode) getMetricsDegrade(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte(fmt.Sprintf("%v\n", atomic.LoadInt64(&s.metricsDegrade))))
}

func (s *DataNode) genClusterVersionFile(w http.ResponseWriter, r *http.Request) {
	paths := make([]string, 0)
	s.space.RangePartitions(func(partition *DataPartition, testID string) bool {
		paths = append(paths, partition.disk.Path)
		return true
	}, "")
	paths = append(paths, s.raftDir)

	for _, p := range paths {
		if _, err := os.Stat(path.Join(p, config.ClusterVersionFile)); err == nil || os.IsExist(err) {
			s.buildFailureResp(w, http.StatusCreated, "cluster version file already exists in "+p)
			return
		}
	}
	for _, p := range paths {
		if err := config.CheckOrStoreClusterUuid(p, s.clusterUuid, true); err != nil {
			s.buildFailureResp(w, http.StatusInternalServerError, "Failed to create cluster version file in "+p)
			return
		}
	}
	s.buildSuccessResp(w, "Generate cluster version file success")
}

func (s *DataNode) buildSuccessResp(w http.ResponseWriter, data interface{}) {
	s.buildJSONResp(w, http.StatusOK, data, "")
}

func (s *DataNode) buildFailureResp(w http.ResponseWriter, code int, msg string) {
	s.buildJSONResp(w, code, nil, msg)
}

// Create response for the API request.
func (s *DataNode) buildJSONResp(w http.ResponseWriter, code int, data interface{}, msg string) {
	var (
		jsonBody []byte
		err      error
	)
	w.WriteHeader(code)
	w.Header().Set("Content-Type", "application/json")
	body := proto.HTTPReply{Code: int32(code), Msg: msg, Data: data}
	if jsonBody, err = json.Marshal(body); err != nil {
		return
	}
	w.Write(jsonBody)
}

type GcExtent struct {
	*storage.ExtentInfo
	GcStatus string `json:"gc_status"`
}

func (s *DataNode) getAllExtent(w http.ResponseWriter, r *http.Request) {
	var (
		partitionID uint64
		err         error
		extents     []*storage.ExtentInfo
	)
	if err = r.ParseForm(); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}

	beforeTime, err := strconv.ParseInt(r.FormValue("beforeTime"), 10, 64)
	if err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}

	if partitionID, err = strconv.ParseUint(r.FormValue("id"), 10, 64); err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partition := s.space.Partition(partitionID)
	if partition == nil {
		s.buildFailureResp(w, http.StatusNotFound, "partition not exist")
		return
	}

	store := partition.ExtentStore()
	extents, err = store.GetAllExtents(beforeTime)
	if err != nil {
		s.buildFailureResp(w, http.StatusInternalServerError, "get all extents failed")
		return
	}

	gcExtents := make([]*GcExtent, len(extents))
	for idx, e := range extents {
		gcExtents[idx] = &GcExtent{
			ExtentInfo: e,
			GcStatus:   store.GetGcFlag(e.FileID).String(),
		}
	}

	s.buildSuccessResp(w, gcExtents)
}

func (s *DataNode) setDiskBadAPI(w http.ResponseWriter, r *http.Request) {
	var (
		err      error
		diskPath common.String
		disk     *Disk
	)
	if err = parseArgs(r, diskPath.DiskPath()); err != nil {
		log.LogErrorf("[setDiskBadAPI] %v", err.Error())
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}

	if disk, err = s.space.GetDisk(diskPath.V); err != nil {
		err = fmt.Errorf("not exit such dissk, path: %v", diskPath.V)
		log.LogErrorf("[setDiskBadAPI] %v", err.Error())
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}

	if disk.Status == proto.Unavailable {
		msg := fmt.Sprintf("disk(%v) status was already unavailable, nothing to do", disk.Path)
		log.LogInfof("[setDiskBadAPI] %v", msg)
		s.buildSuccessResp(w, msg)
		return
	}

	log.LogWarnf("[setDiskBadAPI] set bad disk, path: %v", disk.Path)
	disk.doDiskError()

	s.buildSuccessResp(w, "OK")
}

func (s *DataNode) reloadDataPartition(w http.ResponseWriter, r *http.Request) {
	if !s.checkAllDiskLoaded() {
		s.buildFailureResp(w, http.StatusBadRequest, "please wait for disk loading")
		return
	}
	var pid common.Uint
	if err := parseArgs(r, pid.ID()); err != nil {
		err = fmt.Errorf("parse form fail: %v", err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partitionID := pid.V

	partition := s.space.Partition(partitionID)
	if partition == nil {
		s.buildFailureResp(w, http.StatusNotFound, "partition not exist")
		return
	}
	err := partition.reload(s.space)
	if err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
	} else {
		s.buildSuccessResp(w, "success")
	}
}

// NOTE: use for test
func (s *DataNode) markDataPartitionBroken(w http.ResponseWriter, r *http.Request) {
	const (
		paramID     = "id"
		paramStatus = "status"
	)
	if err := r.ParseForm(); err != nil {
		err = fmt.Errorf("parse form fail: %v", err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partitionID, err := strconv.ParseUint(r.FormValue(paramID), 10, 64)
	if err != nil {
		err = fmt.Errorf("parse param %v fail: %v", paramID, err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	status, err := strconv.ParseBool(r.FormValue(paramStatus))
	if err != nil {
		err = fmt.Errorf("parse param %v fail: %v", paramStatus, err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partition := s.space.Partition(partitionID)
	if partition == nil {
		s.buildFailureResp(w, http.StatusBadRequest, "partition not found")
		return
	}
	if !status {
		partition.checkIsDiskError(syscall.EIO, WriteFlag|ReadFlag)
	} else {
		partition.StartRaft(true)
		partition.resetDiskErrCnt()
		partition.statusUpdate()
	}

	s.buildSuccessResp(w, "success")
}

func (s *DataNode) markDiskBroken(w http.ResponseWriter, r *http.Request) {
	const (
		paramDisk = "disk"
	)
	if err := r.ParseForm(); err != nil {
		err = fmt.Errorf("parse form fail: %v", err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	diskPath := r.FormValue(paramDisk)
	// store disk path and root of dp
	disk, err := s.space.GetDisk(diskPath)
	if err != nil {
		log.LogErrorf("[markDiskBroken] disk(%v) is not found err(%v).", diskPath, err)
		s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("disk %v is not found", diskPath))
		return
	}
	dps := disk.DataPartitionList()
	for _, dpId := range dps {
		partition := s.space.Partition(dpId)
		if partition == nil {
			log.LogErrorf("[markDiskBroken] dp(%v) not found", dpId)
			s.buildFailureResp(w, http.StatusBadRequest, "partition not found")
			return
		}
		partition.checkIsDiskError(syscall.EIO, WriteFlag|ReadFlag)
	}
	s.buildSuccessResp(w, "success")
}

func (s *DataNode) setDiskExtentReadLimitStatus(w http.ResponseWriter, r *http.Request) {
	const (
		paramStatus = "status"
	)
	if err := r.ParseForm(); err != nil {
		err = fmt.Errorf("parse form fail: %v", err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	status, err := strconv.ParseBool(r.FormValue(paramStatus))
	if err != nil {
		err = fmt.Errorf("parse param %v fail: %v", paramStatus, err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	for _, disk := range s.space.disks {
		disk.SetExtentRepairReadLimitStatus(status)
	}
	s.buildSuccessResp(w, "success")
}

type DiskExtentReadLimitInfo struct {
	DiskPath              string `json:"diskPath"`
	ExtentReadLimitStatus bool   `json:"extentReadLimitStatus"`
	Dp                    uint64 `json:"dp"`
}

type DiskExtentReadLimitStatusResponse struct {
	Infos []DiskExtentReadLimitInfo `json:"infos"`
}

func (s *DataNode) queryDiskExtentReadLimitStatus(w http.ResponseWriter, r *http.Request) {
	resp := &DiskExtentReadLimitStatusResponse{}
	for _, disk := range s.space.disks {
		status, dp := disk.QueryExtentRepairReadLimitStatus()
		resp.Infos = append(resp.Infos, DiskExtentReadLimitInfo{DiskPath: disk.Path, ExtentReadLimitStatus: status, Dp: dp})
	}
	s.buildSuccessResp(w, resp)
}

func (s *DataNode) detachDataPartition(w http.ResponseWriter, r *http.Request) {
	const (
		paramID = "id"
	)
	if err := r.ParseForm(); err != nil {
		err = fmt.Errorf("parse form fail: %v", err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partitionID, err := strconv.ParseUint(r.FormValue(paramID), 10, 64)
	if err != nil {
		err = fmt.Errorf("parse param %v fail: %v", paramID, err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partition := s.space.Partition(partitionID)
	if partition == nil {
		s.buildFailureResp(w, http.StatusBadRequest, "partition not exist")
		return
	}
	// store disk path and root of dp
	disk := partition.disk
	rootDir := partition.path
	log.LogDebugf("data partition disk %v rootDir %v", disk, rootDir)

	s.space.partitionMutex.Lock()
	delete(s.space.partitions, partitionID)
	s.space.partitionMutex.Unlock()
	partition.Stop()
	partition.Disk().DetachDataPartition(partition)

	log.LogDebugf("data partition %v is detached", partitionID)
	s.buildSuccessResp(w, "success")
}

func (s *DataNode) releaseDiskExtentReadLimitToken(w http.ResponseWriter, r *http.Request) {
	const (
		paramDisk = "disk"
	)
	if err := r.ParseForm(); err != nil {
		err = fmt.Errorf("parse form fail: %v", err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	diskPath := r.FormValue(paramDisk)
	// store disk path and root of dp
	disk, err := s.space.GetDisk(diskPath)
	if err != nil {
		log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
		s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("disk %v is not found", diskPath))
		return
	}
	disk.ReleaseReadExtentToken()
	s.buildSuccessResp(w, "success")
}

func (s *DataNode) loadDataPartition(w http.ResponseWriter, r *http.Request) {
	const (
		paramID   = "id"
		paramDisk = "disk"
	)
	if err := r.ParseForm(); err != nil {
		err = fmt.Errorf("parse form fail: %v", err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partitionID, err := strconv.ParseUint(r.FormValue(paramID), 10, 64)
	if err != nil {
		err = fmt.Errorf("parse param %v fail: %v", paramID, err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	partition := s.space.Partition(partitionID)
	if partition != nil {
		s.buildFailureResp(w, http.StatusBadRequest, "partition is already loaded")
		return
	}
	diskPath := r.FormValue(paramDisk)
	// store disk path and root of dp
	disk, err := s.space.GetDisk(diskPath)
	if err != nil {
		log.LogErrorf("action[loadDataPartition] disk(%v) is not found err(%v).", diskPath, err)
		s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("disk %v is not found", diskPath))
		return
	}

	fileInfoList, err := os.ReadDir(disk.Path)
	if err != nil {
		log.LogErrorf("action[loadDataPartition] read dir(%v) err(%v).", disk.Path, err)
		s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf(" read dir(%v) err(%v)", disk.Path, err))
		return
	}
	rootDir := ""
	for _, fileInfo := range fileInfoList {
		filename := fileInfo.Name()
		if !disk.isPartitionDir(filename) {
			continue
		}

		if id, _, err := unmarshalPartitionName(filename); err != nil {
			log.LogErrorf("action[RestorePartition] unmarshal partitionName(%v) from disk(%v) err(%v) ",
				filename, disk.Path, err.Error())
			continue
		} else {
			if id == partitionID {
				rootDir = filename
			}
		}
	}
	if rootDir == "" {
		log.LogErrorf("action[loadDataPartition] dp root not found in dir(%v) .", disk.Path)
		s.buildFailureResp(w, http.StatusBadRequest, fmt.Sprintf("dp root not found in dir(%v)", disk.Path))
		return
	}

	log.LogDebugf("data partition disk %v rootDir %v", disk, rootDir)

	_, err = LoadDataPartition(path.Join(diskPath, rootDir), disk)
	if err != nil {
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
	} else {
		s.buildSuccessResp(w, "success")
	}
}

func (s *DataNode) getRaftPeers(w http.ResponseWriter, r *http.Request) {
	const (
		paramRaftID = "id"
	)
	if err := r.ParseForm(); err != nil {
		err = fmt.Errorf("parse form fail: %v", err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	raftID, err := strconv.ParseUint(r.FormValue(paramRaftID), 10, 64)
	if err != nil {
		err = fmt.Errorf("parse param %v fail: %v", paramRaftID, err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	raftPeers := s.raftStore.GetPeers(raftID)
	s.buildSuccessResp(w, raftPeers)
}

func (s *DataNode) setGOGC(w http.ResponseWriter, r *http.Request) {
	const (
		paramGOGC = "gogc"
	)
	var (
		gogcValue int
		err       error
	)
	defer func() {
		if err != nil {
			s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		} else {
			s.buildSuccessResp(w, "set GOGC success")
		}
	}()
	if err = r.ParseForm(); err != nil {
		err = fmt.Errorf("parse form fail: %v", err)
		return
	}
	gogcValue, err = strconv.Atoi(r.FormValue(paramGOGC))
	if err != nil {
		err = fmt.Errorf("parse param %v fail: %v", paramGOGC, err)
		return
	}
	if gogcValue < defaultGOGCLowerLimit || gogcValue > defaultGOGCUpperLimit {
		err = fmt.Errorf("gogc must be greater than or equal to %v and less than or equal to %v", defaultGOGCLowerLimit, defaultGOGCUpperLimit)
		return
	}
	s.useLocalGOGC = true
	if s.gogcValue != gogcValue {
		oldGOGC := s.gogcValue
		debug.SetGCPercent(gogcValue)
		s.gogcValue = gogcValue
		log.LogWarnf("[setGOGC] change GOGC, old(%v) new(%v)", oldGOGC, gogcValue)
	}
}

func (s *DataNode) getGOGC(w http.ResponseWriter, r *http.Request) {
	data := fmt.Sprintf("gogc value is %v", s.gogcValue)
	s.buildSuccessResp(w, data)
}

func (s *DataNode) triggerRaftLogRotate(w http.ResponseWriter, r *http.Request) {
	val, err := MarshalRandWriteRaftLog(proto.OpRandomWrite, 0, 0, 0, nil, 0)
	if err != nil {
		log.LogErrorf("action[triggerRaftLogRotate] marshal error %v", err)
		s.buildFailureResp(w, http.StatusBadRequest, err.Error())
		return
	}
	dataPartitions := s.space.getPartitions()
	trigger := 0
	for _, dp := range dataPartitions {
		if !dp.raftPartition.IsRaftLeader() {
			continue
		}
		_, err = dp.Submit(val)
		if err != nil {
			log.LogErrorf("action[triggerRaftLogRotate] submit error %v", err)
			s.buildFailureResp(w, http.StatusBadRequest, err.Error())
			return
		}
		trigger += 1
	}

	s.buildSuccessResp(w, fmt.Sprintf("trigger dp(%d) raft log rotate successfully.", trigger))
}