From 40f67d570bf068dadcd2f10ec0307afa14f5ecd4 Mon Sep 17 00:00:00 2001 From: yanghonggang Date: Thu, 10 Jul 2025 14:39:46 +0800 Subject: [PATCH] feat(blobstore): add blobstore bench tool The bench tool is used to evaluate the performance of blobstore. It supports performance evaluation for put, get, and del operations. Fixes: #3795 Signed-off-by: yanghonggang Signed-off-by: slasher --- blobstore/tool/bench/backend/blobstore.go | 75 ++ blobstore/tool/bench/backend/store.go | 65 ++ blobstore/tool/bench/bench.go | 696 ++++++++++++++++++ blobstore/tool/bench/db/db.go | 47 ++ blobstore/tool/bench/db/rocksdb.go | 82 +++ build/build.sh | 9 +- .../user-guide/bench/blobstore_bench.md | 167 +++++ .../user-guide/bench/blobstore_bench.md | 169 +++++ 8 files changed, 1309 insertions(+), 1 deletion(-) create mode 100644 blobstore/tool/bench/backend/blobstore.go create mode 100644 blobstore/tool/bench/backend/store.go create mode 100644 blobstore/tool/bench/bench.go create mode 100644 blobstore/tool/bench/db/db.go create mode 100644 blobstore/tool/bench/db/rocksdb.go create mode 100644 docs-zh/source/user-guide/bench/blobstore_bench.md create mode 100644 docs/source/user-guide/bench/blobstore_bench.md diff --git a/blobstore/tool/bench/backend/blobstore.go b/blobstore/tool/bench/backend/blobstore.go new file mode 100644 index 000000000..efcfeec4b --- /dev/null +++ b/blobstore/tool/bench/backend/blobstore.go @@ -0,0 +1,75 @@ +package backend + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/cubefs/cubefs/blobstore/api/access" + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/sdk" +) + +// Blobstore implementation of ObjectStorage interface +type blobStorage struct { + client access.API +} + +func NewBlobStorage(confPath string) (ObjectStorage, error) { + confStr, err := os.ReadFile(confPath) + if err != nil { + return nil, fmt.Errorf("read config file: %v", err) + } + + var conf sdk.Config + if err := json.Unmarshal(confStr, &conf); err != nil { + return nil, fmt.Errorf("parse config: %v", err) + } + + client, err := sdk.New(&conf) + if err != nil { + return nil, fmt.Errorf("create blob client: %v", err) + } + return &blobStorage{client: client}, nil +} + +func (b *blobStorage) PutObject(ctx context.Context, data io.Reader, size int64) (loc LocInfo, err error) { + loc.Value, _, err = b.client.Put(ctx, &access.PutArgs{Size: size, Body: data}) + if err != nil { + return LocInfo{}, fmt.Errorf("put object to blob: %v", err) + } + return loc, nil +} + +func (b *blobStorage) GetObject(ctx context.Context, loc LocInfo, writer io.Writer, size int64) error { + location, err := loc.ExtractBlobLocation() + if err != nil { + return err + } + rc, err := b.client.Get(ctx, &access.GetArgs{Location: location, ReadSize: uint64(size)}) + if err != nil { + return fmt.Errorf("get object from blob: %v", err) + } + defer rc.Close() + + _, err = io.Copy(writer, rc) + if err != nil { + return fmt.Errorf("write data to writer: %v", err) + } + return nil +} + +func (b *blobStorage) DelObject(ctx context.Context, loc LocInfo) error { + location, err := loc.ExtractBlobLocation() + if err != nil { + return err + } + _, err = b.client.Delete(ctx, &access.DeleteArgs{Locations: []proto.Location{location}}) + return err +} + +func (b *blobStorage) Close() error { + return nil +} diff --git a/blobstore/tool/bench/backend/store.go b/blobstore/tool/bench/backend/store.go new file mode 100644 index 000000000..c531b299f --- /dev/null +++ b/blobstore/tool/bench/backend/store.go @@ -0,0 +1,65 @@ +package backend + +import ( + "context" + "fmt" + "io" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +// LocInfo is a wrapper for rados key and blobstore location. +type LocInfo struct { + Value interface{} +} + +// ExtractBlobLocation extracts a blobstore location from the LocInfo instance. +func (loc *LocInfo) ExtractBlobLocation() (proto.Location, error) { + if loc == nil || loc.Value == nil { + return proto.Location{}, fmt.Errorf("loc is nil") + } + + switch v := loc.Value.(type) { + case proto.Location: + return v, nil + default: + return proto.Location{}, fmt.Errorf("loc type error: %T", v) + } +} + +// ExtractRadosKey extracts a rados key from the LocInfo instance. +func (loc *LocInfo) ExtractRadosKey() (string, error) { + if loc == nil || loc.Value == nil { + return "", fmt.Errorf("loc is nil") + } + + switch v := loc.Value.(type) { + case string: + return v, nil + default: + return "", fmt.Errorf("loc type error: %T", v) + } +} + +// Defines bench storage backend operations +// size - is used to optimize buffer management ops +type ObjectStorage interface { + PutObject(ctx context.Context, reader io.Reader, size int64) (location LocInfo, err error) + GetObject(ctx context.Context, location LocInfo, writer io.Writer, size int64) error + DelObject(ctx context.Context, location LocInfo) error + // do cleanup work + Close() error +} + +// dummyStore always do nothing and return success. +// Just used to measure lateny of bench tool itself. +type dummyStore struct{} + +func NewDummyStorage() (ObjectStorage, error) { + return dummyStore{}, nil +} + +func (dummyStore) PutObject(context.Context, io.Reader, int64) (loc LocInfo, err error) { return } +func (dummyStore) GetObject(context.Context, LocInfo, io.Writer, int64) error { return nil } +func (dummyStore) DelObject(context.Context, LocInfo) error { return nil } +func (dummyStore) Close() error { return nil } diff --git a/blobstore/tool/bench/bench.go b/blobstore/tool/bench/bench.go new file mode 100644 index 000000000..becb3c52f --- /dev/null +++ b/blobstore/tool/bench/bench.go @@ -0,0 +1,696 @@ +// ref: https://github.com/markhpc/hsbench/blob/master/hsbench.go +// Copyright (c) 2017 Wasabi Technology, Inc. +// Copyright (c) 2019 Red Hat Inc. +// Copyright (c) 2025 CMCC + +package main + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/rand" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "math" + "os" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/dustin/go-humanize" + + bk "github.com/cubefs/cubefs/blobstore/tool/bench/backend" + "github.com/cubefs/cubefs/blobstore/tool/bench/db" +) + +// Global variables +var ( + backend, database, conf, modes, runName string + objSize, outputFile, poolName string + maxObjCnt, maxErrCnt int64 + durationSecs, threads, loops int + interval float64 + size int64 + objectData []byte + dbDir string +) + +var ( + opCounter int64 + objectCountFlag bool + store bk.ObjectStorage + locdb db.DB +) + +func init() { + flagSet := flag.NewFlagSet("flags", flag.ExitOnError) + flagSet.StringVar(&backend, "b", "blobstore", "Storage backend. 'blobstore' or 'dummy'") + flagSet.StringVar(&conf, "c", os.Getenv("BLOBSTORE_BENCH_CONF"), "Conf file of blobstore") + flagSet.StringVar(&modes, "m", "pgd", "Run modes in order. See NOTES for more info") + flagSet.StringVar(&runName, "pr", "", "Specifies the name of the test run, which also serves as a prefix for generated object names") + flagSet.StringVar(&objSize, "s", "1Mi", "Size of objects in bytes with postfix Ki, Mi, and Gi") + flagSet.StringVar(&database, "db", "", "Database mode. 'rocksdb' or 'memory'") + flagSet.StringVar(&dbDir, "r", "", "RocksDB direcotry. Only for blobstore backend") + + flagSet.Int64Var(&maxObjCnt, "n", -1, "Maximum number of objects <-1 for unlimited>") + flagSet.Int64Var(&maxErrCnt, "e", 3, "Maximum number of errors allowed <-1 for unlimited>") + flagSet.IntVar(&durationSecs, "d", 60, "Maximum test duration in seconds <-1 for unlimited>") + flagSet.IntVar(&threads, "t", 1, "Number of threads to run") + flagSet.IntVar(&loops, "l", 1, "Number of times to repeat bench test") + flag.Float64Var(&interval, "i", 1.0, "Number of seconds between report intervals") + + flag.StringVar(&outputFile, "o", "output.json", "Write JSON output to this file") + + notes := ` +NOTES: + - Valid mode types for the -m mode string are: + p: put objects in blobstore + g: get objects from blobstore + d: delete objects from blobstore +` + + flagSet.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "\nUSAGE: %s [OPTIONS]\n\n", os.Args[0]) + fmt.Fprintf(flag.CommandLine.Output(), "OPTIONS:\n") + flagSet.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s", notes) + } + + var err error + if err = flagSet.Parse(os.Args[1:]); err != nil { + os.Exit(1) + } + + if maxObjCnt < 0 && durationSecs < 0 { + log.Fatal("The number of objects and duration can not both be unlimited") + } + + if conf == "" && backend != "dummy" { + log.Fatal("Missing argument -c for storage cluster.") + } + + for _, r := range modes { + if r != 'p' && r != 'g' && r != 'd' { + log.Fatal("Invalid modes passed to -m, see help for details.") + } + } + + if ((strings.Contains(modes, "g") || strings.Contains(modes, "d")) && + !strings.Contains(modes, "p")) && maxObjCnt < 0 { + log.Fatal("The number of objects(-n XXX) must be specified for g(get)/d(del) test") + } + + if size, err = parseSize(objSize); err != nil { + log.Fatalf("Invalid -s argument for object size: %v", err) + } + + if backend == "blobstore" && database == "rocksdb" && dbDir == "" { + log.Fatal("-r must be specified for blobstore backend") + } + + if maxErrCnt == -1 { + maxErrCnt = math.MaxInt64 + } + + prepareData() +} + +func parseSize(sizeStr string) (int64, error) { + size, err := humanize.ParseBytes(sizeStr) + if err != nil { + return 0, err + } + if size > 1<<30 { + return 0, fmt.Errorf("size too large >1G") + } + return int64(size), nil +} + +func prepareData() { + objectData = make([]byte, size) + rand.Read(objectData) + hasher := md5.New() + hasher.Write(objectData) + objectDataMD5 := hex.EncodeToString(hasher.Sum(nil)) + log.Printf("DataSize=%d MD5=%s", size, objectDataMD5) +} + +// print all paramgs +func printParams() { + log.Println("Parameters:") + log.Printf("backend=%s", backend) + log.Printf("database=%s", database) + log.Printf("conf=%s", conf) + log.Printf("runName=%s", runName) + log.Printf("poolName=%s", poolName) + log.Printf("objSize=%s", objSize) + log.Printf("maxObjCnt=%d", maxObjCnt) + log.Printf("durationSecs=%d", durationSecs) + log.Printf("threads=%d", threads) + log.Printf("loops=%d", loops) + log.Printf("interval=%f", interval) + log.Printf("dbDir=%s", dbDir) +} + +type OutputStats struct { + Loop int + IntervalName string + Seconds float64 + Mode string + Ops int + Mbps float64 + Iops float64 + MinLat float64 + AvgLat float64 + NinetyNineLat float64 + MaxLat float64 + Slowdowns int64 +} + +func (o *OutputStats) log() { + log.Printf("Loop: %d, Int: %s, Dur(s): %.1f, Mode: %s, Ops: %d, MB/s: %.2f, IO/s: %.0f, "+ + "Lat(ms): [min: %.1f, avg: %.1f, 99%%: %.1f, max: %.1f], Slowdowns: %d", + o.Loop, o.IntervalName, o.Seconds, o.Mode, o.Ops, o.Mbps, o.Iops, + o.MinLat, o.AvgLat, o.NinetyNineLat, o.MaxLat, o.Slowdowns) +} + +type IntervalStats struct { + loop int + name string + mode string + bytes int64 + slowdowns int64 + intervalNano int64 + latNano []int64 +} + +func (is *IntervalStats) makeOutputStats() OutputStats { + // Compute and log the stats + ops := len(is.latNano) + totalLat := int64(0) + minLat := float64(0) + maxLat := float64(0) + NinetyNineLat := float64(0) + avgLat := float64(0) + if ops > 0 { + minLat = float64(is.latNano[0]) / 1000000 + maxLat = float64(is.latNano[ops-1]) / 1000000 + for i := range is.latNano { + totalLat += is.latNano[i] + } + avgLat = float64(totalLat) / float64(ops) / 1000000 + NintyNineLatNano := is.latNano[int64(math.Round(0.99*float64(ops)))-1] + NinetyNineLat = float64(NintyNineLatNano) / 1000000 + } + seconds := float64(is.intervalNano) / 1000000000 + mbps := float64(is.bytes) / seconds / (1 << 20) + iops := float64(ops) / seconds + + return OutputStats{ + is.loop, is.name, seconds, is.mode, ops, mbps, iops, + minLat, avgLat, NinetyNineLat, maxLat, is.slowdowns, + } +} + +type ThreadStats struct { + start int64 + curInterval int64 + intervals []IntervalStats +} + +func makeThreadStats(s int64, loop int, mode string, intervalNano int64) ThreadStats { + ts := ThreadStats{s, 0, []IntervalStats{}} + ts.intervals = append(ts.intervals, IntervalStats{loop, "0", mode, 0, 0, intervalNano, []int64{}}) + return ts +} + +func (ts *ThreadStats) updateIntervals(loop int, mode string, intervalNano int64) int64 { + // Interval statistics disabled, so just return the current interval + if intervalNano < 0 { + return ts.curInterval + } + for ts.start+intervalNano*(ts.curInterval+1) < time.Now().UnixNano() { + ts.curInterval++ + ts.intervals = append(ts.intervals, + IntervalStats{loop, strconv.FormatInt(ts.curInterval, 10), mode, 0, 0, intervalNano, []int64{}}) + } + return ts.curInterval +} + +func (ts *ThreadStats) finish() { + ts.curInterval = -1 +} + +type Stats struct { + // threads + threads int + // The loop we are in + loop int + // Test mode being run + mode string + // start time in nanoseconds + startNano int64 + // end time in nanoseconds + endNano int64 + // Duration in nanoseconds for each interval + intervalNano int64 + // Per-thread statistics + threadStats []ThreadStats + // a map of per-interval thread completion counters + intervalCompletions sync.Map + // a counter of how many threads have finished updating stats entirely + completions int32 +} + +func makeStats(loop int, mode string, threads int, intervalNano int64) *Stats { + start := time.Now().UnixNano() + s := &Stats{threads, loop, mode, start, 0, intervalNano, []ThreadStats{}, sync.Map{}, 0} + for i := 0; i < threads; i++ { + s.threadStats = append(s.threadStats, makeThreadStats(start, s.loop, s.mode, s.intervalNano)) + s.updateIntervals(i) + } + return s +} + +func (stats *Stats) makeOutputStats(i int64) (OutputStats, bool) { + // Check bounds first + if stats.intervalNano < 0 || i < 0 { + return OutputStats{}, false + } + // Not safe to log if not all writers have completed. + value, ok := stats.intervalCompletions.Load(i) + if !ok { + return OutputStats{}, false + } + cp, ok := value.(*int32) + if !ok { + return OutputStats{}, false + } + count := atomic.LoadInt32(cp) + if count < int32(stats.threads) { + return OutputStats{}, false + } + + bytes := int64(0) + ops := int64(0) + slowdowns := int64(0) + + for t := 0; t < stats.threads; t++ { + bytes += stats.threadStats[t].intervals[i].bytes + ops += int64(len(stats.threadStats[t].intervals[i].latNano)) + slowdowns += stats.threadStats[t].intervals[i].slowdowns + } + // Aggregate the per-thread Latency slice + tmpLat := make([]int64, ops) + var c int + for t := 0; t < stats.threads; t++ { + c += copy(tmpLat[c:], stats.threadStats[t].intervals[i].latNano) + } + sort.Slice(tmpLat, func(i, j int) bool { return tmpLat[i] < tmpLat[j] }) + is := IntervalStats{stats.loop, strconv.FormatInt(i, 10), stats.mode, bytes, slowdowns, stats.intervalNano, tmpLat} + return is.makeOutputStats(), true +} + +// Only safe to call from the calling thread +func (stats *Stats) updateIntervals(threadNum int) int64 { + curInterval := stats.threadStats[threadNum].curInterval + newInterval := stats.threadStats[threadNum].updateIntervals(stats.loop, stats.mode, stats.intervalNano) + + // Finish has already been called + if curInterval < 0 { + return -1 + } + + for i := curInterval; i < newInterval; i++ { + // load or store the current value + value, _ := stats.intervalCompletions.LoadOrStore(i, new(int32)) + cp, ok := value.(*int32) + if !ok { + log.Printf("updateIntervals: got data of type %T but wanted *int32", value) + continue + } + + count := atomic.AddInt32(cp, 1) + if count == int32(stats.threads) { + if is, ok := stats.makeOutputStats(i); ok { + is.log() + } + } + } + return newInterval +} + +func (stats *Stats) makeTotalStats() (OutputStats, bool) { + // Not safe to log if not all writers have completed. + completions := atomic.LoadInt32(&stats.completions) + if completions < int32(threads) { + log.Printf("log, completions: %d", completions) + return OutputStats{}, false + } + + bytes := int64(0) + ops := int64(0) + slowdowns := int64(0) + + for t := 0; t < stats.threads; t++ { + for i := 0; i < len(stats.threadStats[t].intervals); i++ { + bytes += stats.threadStats[t].intervals[i].bytes + ops += int64(len(stats.threadStats[t].intervals[i].latNano)) + slowdowns += stats.threadStats[t].intervals[i].slowdowns + } + } + // Aggregate the per-thread Latency slice + tmpLat := make([]int64, ops) + var c int + for t := 0; t < stats.threads; t++ { + for i := 0; i < len(stats.threadStats[t].intervals); i++ { + c += copy(tmpLat[c:], stats.threadStats[t].intervals[i].latNano) + } + } + sort.Slice(tmpLat, func(i, j int) bool { return tmpLat[i] < tmpLat[j] }) + is := IntervalStats{stats.loop, "TOTAL", stats.mode, bytes, slowdowns, stats.endNano - stats.startNano, tmpLat} + return is.makeOutputStats(), true +} + +func (stats *Stats) addOp(threadNum int, bytes int64, latNano int64) { + // Interval statistics + cur := stats.threadStats[threadNum].curInterval + if cur < 0 { + return + } + stats.threadStats[threadNum].intervals[cur].bytes += bytes + stats.threadStats[threadNum].intervals[cur].latNano = append(stats.threadStats[threadNum].intervals[cur].latNano, latNano) +} + +func (stats *Stats) addSlowDown(threadNum int) { + cur := stats.threadStats[threadNum].curInterval + stats.threadStats[threadNum].intervals[cur].slowdowns++ +} + +func (stats *Stats) finish(threadNum int) { + stats.updateIntervals(threadNum) + stats.threadStats[threadNum].finish() + count := atomic.AddInt32(&stats.completions, 1) + if count == int32(stats.threads) { + stats.endNano = time.Now().UnixNano() + } +} + +func formatKeyString(objnum int64) string { + return fmt.Sprintf("%s%012d", runName, objnum) +} + +// kickoff put load +func runPutLoad(ctx context.Context, threadNum int, fendtime time.Time, stats *Stats) { + errcnt := int64(0) + for { + if durationSecs > -1 && time.Now().After(fendtime) { + break + } + objnum := atomic.AddInt64(&opCounter, 1) + if maxObjCnt > -1 && objnum >= maxObjCnt { + atomic.AddInt64(&opCounter, -1) + break + } + + fileobj := bytes.NewReader(objectData) + var err error + + start := time.Now().UnixNano() + loc, err := store.PutObject(ctx, fileobj, fileobj.Size()) + if err != nil { + stats.addSlowDown(threadNum) + atomic.AddInt64(&opCounter, -1) + log.Printf("upload err: %v", err) + if errcnt++; errcnt >= maxErrCnt { + break + } + continue + } + + end := time.Now().UnixNano() + stats.updateIntervals(threadNum) + + if backend == "blobstore" { + key := formatKeyString(objnum) + location, _ := loc.ExtractBlobLocation() + if err = locdb.Put(key, location); err != nil { + stats.addSlowDown(threadNum) + atomic.AddInt64(&opCounter, -1) + log.Printf("upload err: %v", err) + if errcnt++; errcnt >= maxErrCnt { + break + } + } + } + if err == nil { + // Update the stats + stats.addOp(threadNum, size, end-start) + } + } + stats.finish(threadNum) +} + +// kickoff get load +func runGetLoad(ctx context.Context, threadNum int, fendtime time.Time, stats *Stats) { + errcnt := int64(0) + for { + if durationSecs > -1 && time.Now().After(fendtime) { + break + } + objnum := atomic.AddInt64(&opCounter, 1) + if maxObjCnt > -1 && objnum >= maxObjCnt { + atomic.AddInt64(&opCounter, -1) + break + } + + // get key/location info from location store + var err error + var loc bk.LocInfo + key := formatKeyString(objnum) + if backend == "blobstore" { + loc.Value, err = locdb.Get(key) + if err != nil { + errcnt++ + stats.addSlowDown(threadNum) + log.Printf("download err %v", err) + continue + } + } + + start := time.Now().UnixNano() + writer := io.Discard // Use io.Discard to discard the data + err = store.GetObject(ctx, loc, writer, size) + end := time.Now().UnixNano() + stats.updateIntervals(threadNum) + + if err != nil { + errcnt++ + stats.addSlowDown(threadNum) + log.Printf("download err %v", err) + } else { + // Update the stats + stats.addOp(threadNum, size, end-start) + } + if errcnt >= maxErrCnt { + break + } + } + stats.finish(threadNum) +} + +// kickoff del load +func runDelLoad(ctx context.Context, threadNum int, fendtime time.Time, stats *Stats) { + errcnt := int64(0) + for { + if durationSecs > -1 && time.Now().After(fendtime) { + break + } + objnum := atomic.AddInt64(&opCounter, 1) + if maxObjCnt > -1 && objnum >= maxObjCnt { + atomic.AddInt64(&opCounter, -1) + break + } + + // get key/location info from location store + var err error + var loc bk.LocInfo + + key := formatKeyString(objnum) + if backend == "blobstore" { + loc.Value, err = locdb.Get(key) + if err != nil { + errcnt++ + stats.addSlowDown(threadNum) + log.Printf("download err %v", err) + continue + } + } + + start := time.Now().UnixNano() + err = store.DelObject(ctx, loc) + end := time.Now().UnixNano() + stats.updateIntervals(threadNum) + + if err != nil { + errcnt++ + stats.addSlowDown(threadNum) + log.Printf("delete err %v", err) + } else { + // delete key/location info. Ignore error. + if backend == "blobstore" { + locdb.Del(key) + } + // Update the stats + stats.addOp(threadNum, size, end-start) + } + if errcnt >= maxErrCnt { + break + } + } + stats.finish(threadNum) +} + +func runWrapper(ctx context.Context, loop int, r rune) []OutputStats { + atomic.StoreInt64(&opCounter, -1) + intervalNano := int64(interval * 1000000000) + endtime := time.Now().Add(time.Second * time.Duration(durationSecs)) + + // If we perviously set the object count after running a put + // test, set the object count back to -1 for the new put test. + if r == 'p' && objectCountFlag { + maxObjCnt = -1 + objectCountFlag = false + } + + var stats *Stats + var wg sync.WaitGroup + switch r { + case 'p': + log.Printf("Running Loop %d Put Test", loop) + stats = makeStats(loop, "PUT", threads, intervalNano) + wg.Add(threads) + for n := 0; n < threads; n++ { + go func(n int) { + defer wg.Done() + runPutLoad(ctx, n, endtime, stats) + }(n) + } + case 'g': + log.Printf("Running Loop %d Get Test", loop) + stats = makeStats(loop, "GET", threads, intervalNano) + wg.Add(threads) + for n := 0; n < threads; n++ { + go func(n int) { + defer wg.Done() + runGetLoad(ctx, n, endtime, stats) + }(n) + } + case 'd': + log.Printf("Running Loop %d Del Test", loop) + stats = makeStats(loop, "DEL", threads, intervalNano) + wg.Add(threads) + for n := 0; n < threads; n++ { + go func(n int) { + defer wg.Done() + runDelLoad(ctx, n, endtime, stats) + }(n) + } + } + wg.Wait() + + // If the user didn't set the object_count, we can set it here + // to limit subsequent get/del tests to valid objects only. + if r == 'p' && maxObjCnt < 0 { + maxObjCnt = atomic.LoadInt64(&opCounter) + 1 + objectCountFlag = true + } + + // Create the Output Stats + os := make([]OutputStats, 0) + for i := int64(0); i >= 0; i++ { + if o, ok := stats.makeOutputStats(i); ok { + os = append(os, o) + } else { + break + } + } + if o, ok := stats.makeTotalStats(); ok { + o.log() + os = append(os, o) + if r == 'p' { + log.Printf("The subsequent '-m g' or '-m d' test should be specified with '-n %d'", o.Ops) + } + } + + return os +} + +func main() { + log.Printf("BlobStore Benchmark") + + printParams() + + // init storage + var err error + switch backend { + case "blobstore": + store, err = bk.NewBlobStorage(conf) + case "dummy": + store, err = bk.NewDummyStorage() + default: + err = fmt.Errorf("backend %s not supported", backend) + } + if err != nil { + log.Fatal(err) + } + defer store.Close() + + switch database { + case "rocksdb": + locdb, err = db.NewRocksDB(dbDir) + default: + locdb, err = db.NewMemoryDB() + } + if err != nil { + log.Fatal(err) + } + + // loop running the tests + ctx := context.Background() + oStats := make([]OutputStats, 0) + for loop := 0; loop < loops; loop++ { + for _, r := range modes { + oStats = append(oStats, runWrapper(ctx, loop+1, r)...) + } + } + + if outputFile == "" { + return + } + + // write test result to specified file + file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY, 0o777) + if err != nil { + log.Fatal("Could not open JSON file for writing.") + } + defer file.Close() + + data, err := json.Marshal(oStats) + if err != nil { + log.Fatal("Error marshaling JSON: ", err) + } + _, err = file.Write(data) + if err != nil { + log.Fatal("Error writing to JSON file: ", err) + } + file.Sync() +} diff --git a/blobstore/tool/bench/db/db.go b/blobstore/tool/bench/db/db.go new file mode 100644 index 000000000..3ec52a7a4 --- /dev/null +++ b/blobstore/tool/bench/db/db.go @@ -0,0 +1,47 @@ +package db + +import ( + "fmt" + "sync" + + "github.com/cubefs/cubefs/blobstore/common/proto" +) + +// DB defines metadata operations +type DB interface { + Put(key string, location proto.Location) error + Get(key string) (location proto.Location, err error) + Del(key string) error + Close() error +} + +// memorydb implements DB with map +type memorydb struct { + db sync.Map +} + +func NewMemoryDB() (DB, error) { + return &memorydb{}, nil +} + +func (m *memorydb) Put(key string, loc proto.Location) error { + m.db.Store(key, loc) + return nil +} + +func (m *memorydb) Get(key string) (proto.Location, error) { + val, _ := m.db.Load(key) + if val == nil { + return proto.Location{}, fmt.Errorf("key(%s) not found", key) + } + return val.(proto.Location), nil +} + +func (m *memorydb) Del(key string) error { + m.db.Delete(key) + return nil +} + +func (r *memorydb) Close() error { + return nil +} diff --git a/blobstore/tool/bench/db/rocksdb.go b/blobstore/tool/bench/db/rocksdb.go new file mode 100644 index 000000000..046f54015 --- /dev/null +++ b/blobstore/tool/bench/db/rocksdb.go @@ -0,0 +1,82 @@ +package db + +import ( + "fmt" + + "github.com/tecbot/gorocksdb" + + "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/util/bytespool" +) + +// RocksDB implements DB with RocksDB +type RocksDB struct { + db *gorocksdb.DB // RocksDB instance for storing key-location mappings +} + +func NewRocksDB(dbDir string) (DB, error) { + opts := gorocksdb.NewDefaultOptions() + opts.SetCreateIfMissing(true) + db, err := gorocksdb.OpenDb(opts, dbDir) + if err != nil { + return nil, err + } + return &RocksDB{db: db}, nil +} + +func (r *RocksDB) Put(key string, loc proto.Location) error { + wo := gorocksdb.NewDefaultWriteOptions() + defer wo.Destroy() + + n := 40 + len(loc.Slices)*20 + buf := bytespool.AllocPointer(n) + n = loc.Encode2(*buf) + + err := r.db.Put(wo, []byte(key), (*buf)[:n]) + bytespool.FreePointer(buf) + if err != nil { + return fmt.Errorf("key(%s) put to rocksdb: %v", key, err) + } + return nil +} + +func (r *RocksDB) Get(key string) (proto.Location, error) { + ro := gorocksdb.NewDefaultReadOptions() + defer ro.Destroy() + + locationBytes, err := r.db.Get(ro, []byte(key)) + if err != nil { + return proto.Location{}, fmt.Errorf("key(%s) retrieve from rocksdb: %v", key, err) + } + defer locationBytes.Free() + + if locationBytes.Size() == 0 { + return proto.Location{}, fmt.Errorf("key(%s) not exist in rocksdb", key) + } + + var location proto.Location + if _, err := location.Decode(locationBytes.Data()); err != nil { + return proto.Location{}, fmt.Errorf("key(%s) parse location: %v", key, err) + } + return location, nil +} + +func (r *RocksDB) Del(key string) error { + wo := gorocksdb.NewDefaultWriteOptions() + defer wo.Destroy() + if err := r.db.Delete(wo, []byte(key)); err != nil { + return fmt.Errorf("key(%s) delete from rocksdb: %v", key, err) + } + return nil +} + +func (r *RocksDB) Close() error { + flushOpts := gorocksdb.NewDefaultFlushOptions() + defer flushOpts.Destroy() + err := r.db.Flush(flushOpts) + if err != nil { + return fmt.Errorf("flush database: %v", err) + } + r.db.Close() + return nil +} diff --git a/build/build.sh b/build/build.sh index 964a61a7c..34be18ddb 100755 --- a/build/build.sh +++ b/build/build.sh @@ -427,10 +427,17 @@ build_blobstore_dialtest_bin() { CGO_ENABLED=0 go build ${MODFLAGS} -gcflags=all=-trimpath=${BlobPath} -asmflags=all=-trimpath=${BlobPath} -ldflags="${LDFlags}" -o ${BuildBinPath}/blobstore/blobstore-dialtest ${SrcPath}/blobstore/testing/dial/main } +build_blobstore_bench() { + pushd $SrcPath > /dev/null + echo -n "build blobstore bench" + go build ${MODFLAGS} -gcflags=all=-trimpath="${SrcPath}" -asmflags=all=-trimpath="${SrcPath}" -ldflags="${LDFlags}" -o "${BuildBinPath}/blobstore/blobstore-bench" "${SrcPath}/blobstore/tool/bench" + popd > /dev/null +} + build_blobstore() { pushd $SrcPath >/dev/null echo -n "build blobstore " - build_clustermgr && build_blobnode && build_access && build_scheduler && build_proxy && build_blobstore_cli && build_shardnode && echo "success" || echo "failed" + build_clustermgr && build_blobnode && build_access && build_scheduler && build_proxy && build_blobstore_cli && build_shardnode && build_blobstore_bench && echo "success" || echo "failed" popd >/dev/null } diff --git a/docs-zh/source/user-guide/bench/blobstore_bench.md b/docs-zh/source/user-guide/bench/blobstore_bench.md new file mode 100644 index 000000000..c6cc25791 --- /dev/null +++ b/docs-zh/source/user-guide/bench/blobstore_bench.md @@ -0,0 +1,167 @@ +# blobstore bench 工具使用 + +bench 工具用于评估 blobstore 的性能。支持 put、get、del 性能评估。 + +## 帮助 + +```text +# build/bin/blobstore/blobstore-bench -h + +USAGE: build/bin/blobstore/blobstore-bench [OPTIONS] + +OPTIONS: + -b string + Storage backend. 'blobstore' or 'dummy' (default "blobstore") + -c string + Conf file of blobstore + -d int + Maximum test duration in seconds <-1 for unlimited> (default 60) + -db string + Database mode. 'rocksdb' or 'memory' + -e int + Maximum number of errors allowed <-1 for unlimited> (default 3) + -l int + Number of times to repeat bench test (default 1) + -m string + Run modes in order. See NOTES for more info (default "pgd") + -n int + Maximum number of objects <-1 for unlimited> (default -1) + -pr string + Specifies the name of the test run, which also serves as a prefix for generated object names + -r string + RocksDB direcotry. Only for blobstore backend + -s string + Size of objects in bytes with postfix Ki, Mi, and Gi (default "1Mi") + -t int + Number of threads to run (default 1) + +NOTES: + - Valid mode types for the -m mode string are: + p: put objects in blobstore + g: get objects from blobstore + d: delete objects from blobstore +``` + +## 示例 + +### 测试 blobstore 的性能 + +准备配置文件 blob.conf： + +```json +{ + "idc": "z0", + "code_mode_put_quorums": { + "11": 4 + }, + "code_mode_get_ordered": { + "11": true + }, + "cluster_config": { + "clusters": [{"cluster_id": 1, "hosts": ["http://127.0.0.1:9998"]}] + } +} +``` + +执行测试: + +```text +# build/bin/blobstore/blobstore-bench -b blobstore -c blob.conf -db rocksdb -r build/db/ -d 5 -m pgd -pr mybench -s 10Ki -t 2 -l 2 +2025/07/10 10:45:23 DataSize=10240 MD5=122506ab6d703619d577a93dd6966e80 +2025/07/10 10:45:23 BlobStore Benchmark +2025/07/10 10:45:23 Parameters: +2025/07/10 10:45:23 backend=blobstore +2025/07/10 10:45:23 database=rocksdb +2025/07/10 10:45:23 conf=blob.conf +2025/07/10 10:45:23 runName=mybench +2025/07/10 10:45:23 poolName= +2025/07/10 10:45:23 objSize=10Ki +2025/07/10 10:45:23 maxObjCnt=-1 +2025/07/10 10:45:23 durationSecs=5 +2025/07/10 10:45:23 threads=2 +2025/07/10 10:45:23 loops=2 +2025/07/10 10:45:23 interval=1.000000 +2025/07/10 10:45:23 dbDir=build/db/ +2025/07/10 10:45:23 Running Loop 0 Put Test +2025/07/10 10:45:24 Loop: 0, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 677, MB/s: 6.61, IO/s: 677, Lat(ms): [ min: 1.9, avg: 2.7, 99%: 5.3, max: 27.7 ], Slowdowns: 0 +2025/07/10 10:45:25 Loop: 0, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 482, MB/s: 4.71, IO/s: 482, Lat(ms): [ min: 1.9, avg: 3.8, 99%: 12.2, max: 16.3 ], Slowdowns: 0 +2025/07/10 10:45:26 Loop: 0, Int: 2, Dur(s): 1.0, Mode: PUT, Ops: 629, MB/s: 6.14, IO/s: 629, Lat(ms): [ min: 1.7, avg: 2.9, 99%: 11.2, max: 15.5 ], Slowdowns: 0 +2025/07/10 10:45:27 Loop: 0, Int: 3, Dur(s): 1.0, Mode: PUT, Ops: 504, MB/s: 4.92, IO/s: 504, Lat(ms): [ min: 1.9, avg: 3.7, 99%: 6.8, max: 10.7 ], Slowdowns: 0 +2025/07/10 10:45:28 Loop: 0, Int: 4, Dur(s): 1.0, Mode: PUT, Ops: 531, MB/s: 5.19, IO/s: 531, Lat(ms): [ min: 1.7, avg: 3.5, 99%: 13.3, max: 36.8 ], Slowdowns: 0 +2025/07/10 10:45:28 Loop: 0, Int: TOTAL, Dur(s): 5.0, Mode: PUT, Ops: 2825, MB/s: 5.49, IO/s: 562, Lat(ms): [ min: 1.7, avg: 3.3, 99%: 11.1, max: 36.8 ], Slowdowns: 0 +2025/07/10 10:45:28 The subsequent '-m g' or '-m d' test should be specified with '-n 2825' +2025/07/10 10:45:28 Running Loop 0 Get Test +2025/07/10 10:45:29 Loop: 0, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 1286, MB/s: 12.56, IO/s: 1286, Lat(ms): [ min: 0.9, avg: 1.5, 99%: 4.1, max: 7.8 ], Slowdowns: 0 +2025/07/10 10:45:30 Loop: 0, Int: 1, Dur(s): 1.0, Mode: GET, Ops: 1339, MB/s: 13.08, IO/s: 1339, Lat(ms): [ min: 0.7, avg: 1.5, 99%: 3.9, max: 11.2 ], Slowdowns: 0 +2025/07/10 10:45:30 Loop: 0, Int: TOTAL, Dur(s): 2.1, Mode: GET, Ops: 2825, MB/s: 12.91, IO/s: 1322, Lat(ms): [ min: 0.7, avg: 1.5, 99%: 4.0, max: 11.2 ], Slowdowns: 0 +2025/07/10 10:45:30 Running Loop 0 Del Test +2025/07/10 10:45:31 Loop: 0, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 2215, MB/s: 21.63, IO/s: 2215, Lat(ms): [ min: 0.4, avg: 0.8, 99%: 1.9, max: 26.3 ], Slowdowns: 0 +2025/07/10 10:45:31 Loop: 0, Int: TOTAL, Dur(s): 1.2, Mode: DEL, Ops: 2825, MB/s: 22.20, IO/s: 2273, Lat(ms): [ min: 0.4, avg: 0.7, 99%: 1.9, max: 26.3 ], Slowdowns: 0 +2025/07/10 10:45:31 Running Loop 1 Put Test +2025/07/10 10:45:32 Loop: 1, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 720, MB/s: 7.03, IO/s: 720, Lat(ms): [ min: 1.8, avg: 2.5, 99%: 3.9, max: 5.0 ], Slowdowns: 0 +2025/07/10 10:45:33 Loop: 1, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 603, MB/s: 5.89, IO/s: 603, Lat(ms): [ min: 1.8, avg: 3.0, 99%: 9.5, max: 27.8 ], Slowdowns: 0 +2025/07/10 10:45:34 Loop: 1, Int: 2, Dur(s): 1.0, Mode: PUT, Ops: 728, MB/s: 7.11, IO/s: 728, Lat(ms): [ min: -3.2, avg: 2.5, 99%: 4.3, max: 5.5 ], Slowdowns: 0 +2025/07/10 10:45:35 Loop: 1, Int: 3, Dur(s): 1.0, Mode: PUT, Ops: 703, MB/s: 6.87, IO/s: 703, Lat(ms): [ min: 1.7, avg: 2.6, 99%: 5.1, max: 9.8 ], Slowdowns: 0 +2025/07/10 10:45:36 Loop: 1, Int: TOTAL, Dur(s): 5.0, Mode: PUT, Ops: 3485, MB/s: 6.81, IO/s: 698, Lat(ms): [ min: -3.2, avg: 2.6, 99%: 5.8, max: 27.8 ], Slowdowns: 0 +2025/07/10 10:45:36 The subsequent '-m g' or '-m d' test should be specified with '-n 3485' +2025/07/10 10:45:36 Running Loop 1 Get Test +2025/07/10 10:45:37 Loop: 1, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 1332, MB/s: 13.01, IO/s: 1332, Lat(ms): [ min: 0.8, avg: 1.5, 99%: 4.0, max: 13.7 ], Slowdowns: 0 +2025/07/10 10:45:38 Loop: 1, Int: 1, Dur(s): 1.0, Mode: GET, Ops: 1220, MB/s: 11.91, IO/s: 1220, Lat(ms): [ min: 0.7, avg: 1.6, 99%: 7.4, max: 15.9 ], Slowdowns: 0 +2025/07/10 10:45:39 Loop: 1, Int: TOTAL, Dur(s): 2.7, Mode: GET, Ops: 3485, MB/s: 12.77, IO/s: 1307, Lat(ms): [ min: 0.7, avg: 1.5, 99%: 5.1, max: 15.9 ], Slowdowns: 0 +2025/07/10 10:45:39 Running Loop 1 Del Test +2025/07/10 10:45:40 Loop: 1, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 2805, MB/s: 27.39, IO/s: 2805, Lat(ms): [ min: 0.4, avg: 0.6, 99%: 1.4, max: 8.8 ], Slowdowns: 0 +2025/07/10 10:45:40 Loop: 1, Int: TOTAL, Dur(s): 1.2, Mode: DEL, Ops: 3485, MB/s: 27.49, IO/s: 2815, Lat(ms): [ min: 0.4, avg: 0.6, 99%: 1.3, max: 8.8 ], Slowdowns: 0 +``` + +### 测试自身开销 + +* 只是为了评估工具自身的开销 + +```text +# build/bin/blobstore/blobstore-bench -b dummy -d 3 -m pgd -pr dummytest -s 10Ki -t 2 -l 2 +2025/07/10 10:48:04 DataSize=10240 MD5=7d3112f86cdbcb45694735dd53928d94 +2025/07/10 10:48:04 BlobStore Benchmark +2025/07/10 10:48:04 Parameters: +2025/07/10 10:48:04 backend=dummy +2025/07/10 10:48:04 database= +2025/07/10 10:48:04 conf= +2025/07/10 10:48:04 runName=dummytest +2025/07/10 10:48:04 poolName= +2025/07/10 10:48:04 objSize=10Ki +2025/07/10 10:48:04 maxObjCnt=-1 +2025/07/10 10:48:04 durationSecs=3 +2025/07/10 10:48:04 threads=2 +2025/07/10 10:48:04 loops=2 +2025/07/10 10:48:04 interval=1.000000 +2025/07/10 10:48:04 dbDir= +2025/07/10 10:48:04 Running Loop 0 Put Test +2025/07/10 10:48:06 Loop: 0, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 4626891, MB/s: 45184.48, IO/s: 4626891, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/10 10:48:06 Loop: 0, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 1309567, MB/s: 12788.74, IO/s: 1309567, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.0 ], Slowdowns: 0 +2025/07/10 10:48:08 Loop: 0, Int: TOTAL, Dur(s): 3.0, Mode: PUT, Ops: 10025255, MB/s: 32141.80, IO/s: 3291320, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.8 ], Slowdowns: 0 +2025/07/10 10:48:08 The subsequent '-m g' or '-m d' test should be specified with '-n 10025255' +2025/07/10 10:48:08 Running Loop 0 Get Test +2025/07/10 10:48:09 Loop: 0, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 5868435, MB/s: 57308.94, IO/s: 5868435, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.1 ], Slowdowns: 0 +2025/07/10 10:48:10 Loop: 0, Int: TOTAL, Dur(s): 1.8, Mode: GET, Ops: 10025255, MB/s: 55878.86, IO/s: 5721996, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.1 ], Slowdowns: 0 +2025/07/10 10:48:10 Running Loop 0 Del Test +2025/07/10 10:48:12 Loop: 0, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 2955369, MB/s: 28861.03, IO/s: 2955369, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.1 ], Slowdowns: 0 +2025/07/10 10:48:12 Loop: 0, Int: 1, Dur(s): 1.0, Mode: DEL, Ops: 2766585, MB/s: 27017.43, IO/s: 2766585, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 1.4 ], Slowdowns: 0 +2025/07/10 10:48:14 Loop: 0, Int: TOTAL, Dur(s): 2.8, Mode: DEL, Ops: 10025255, MB/s: 34423.86, IO/s: 3525003, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 1.4 ], Slowdowns: 0 +2025/07/10 10:48:14 Running Loop 1 Put Test +2025/07/10 10:48:15 Loop: 1, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 5314085, MB/s: 51895.36, IO/s: 5314085, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/10 10:48:16 Loop: 1, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 5246882, MB/s: 51239.08, IO/s: 5246882, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.1 ], Slowdowns: 0 +2025/07/10 10:48:17 Loop: 1, Int: TOTAL, Dur(s): 3.0, Mode: PUT, Ops: 15934614, MB/s: 51870.53, IO/s: 5311542, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/10 10:48:17 The subsequent '-m g' or '-m d' test should be specified with '-n 15934614' +2025/07/10 10:48:17 Running Loop 1 Get Test +2025/07/10 10:48:18 Loop: 1, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 5406327, MB/s: 52796.16, IO/s: 5406327, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.4 ], Slowdowns: 0 +2025/07/10 10:48:19 Loop: 1, Int: 1, Dur(s): 1.0, Mode: GET, Ops: 4264340, MB/s: 41643.95, IO/s: 4264340, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.1 ], Slowdowns: 0 +2025/07/10 10:48:21 Loop: 1, Int: TOTAL, Dur(s): 3.0, Mode: GET, Ops: 15303558, MB/s: 49816.30, IO/s: 5101189, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.4 ], Slowdowns: 0 +2025/07/10 10:48:21 Running Loop 1 Del Test +2025/07/10 10:48:22 Loop: 1, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 5842697, MB/s: 57057.59, IO/s: 5842697, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.4 ], Slowdowns: 0 +2025/07/10 10:48:23 Loop: 1, Int: 1, Dur(s): 1.0, Mode: DEL, Ops: 4858864, MB/s: 47449.84, IO/s: 4858864, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 1.0 ], Slowdowns: 0 +2025/07/10 10:48:24 Loop: 1, Int: TOTAL, Dur(s): 3.0, Mode: DEL, Ops: 15723475, MB/s: 51183.42, IO/s: 5241183, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 1.0 ], Slowdowns: 0 +``` + +## 其他 + +* `-m`：可以指定单个操作，也可以指定多个操作的组合，比如 `p`、`g`、`d`、`pg`、`pd`、`pggg`，但是一旦执行了 `d` 数据就会被删除 +* `-n`：对于 `d/del` 操作是必须要要指定的，为了确定对象的最大编号，用于确定操作停止的边界 diff --git a/docs/source/user-guide/bench/blobstore_bench.md b/docs/source/user-guide/bench/blobstore_bench.md new file mode 100644 index 000000000..054a21009 --- /dev/null +++ b/docs/source/user-guide/bench/blobstore_bench.md @@ -0,0 +1,169 @@ +# Using blobstore bench Tool + +The blobstore-bench is used to evaluate the performance of blobstore. It supports performance assessments for put, get, and del operations. + +## Usage + +```text +# build/bin/blobstore/blobstore-bench -h + +USAGE: build/bin/blobstore/blobstore-bench [OPTIONS] + +OPTIONS: + -b string + Storage backend. 'blobstore' or 'dummy' (default "blobstore") + -c string + Conf file of blobstore + -d int + Maximum test duration in seconds <-1 for unlimited> (default 60) + -db string + Database mode. 'rocksdb' or 'memory' + -e int + Maximum number of errors allowed <-1 for unlimited> (default 3) + -l int + Number of times to repeat bench test (default 1) + -m string + Run modes in order. See NOTES for more info (default "pgd") + -n int + Maximum number of objects <-1 for unlimited> (default -1) + -pr string + Specifies the name of the test run, which also serves as a prefix for generated object names + -r string + RocksDB direcotry. Only for blobstore backend + -s string + Size of objects in bytes with postfix Ki, Mi, and Gi (default "1Mi") + -t int + Number of threads to run (default 1) + +NOTES: + - Valid mode types for the -m mode string are: + p: put objects in blobstore + g: get objects from blobstore + d: delete objects from blobstore +``` + +## Examples + +### Testing Blobstore Performance + +Prepare the configuration file blob.conf: + +```json +{ + "idc": "z0", + "code_mode_put_quorums": { + "11": 4 + }, + "code_mode_get_ordered": { + "11": true + }, + "cluster_config": { + "clusters": [{"cluster_id": 1, "hosts": ["http://127.0.0.1:9998"]}] + } +} +``` + +Run the test: + +```text +# build/bin/blobstore/blobstore-bench -b blobstore -c blob.conf -db rocksdb -r build/db/ -d 5 -m pgd -pr mybench -s 10Ki -t 2 -l 2 +2025/07/10 10:45:23 DataSize=10240 MD5=122506ab6d703619d577a93dd6966e80 +2025/07/10 10:45:23 BlobStore Benchmark +2025/07/10 10:45:23 Parameters: +2025/07/10 10:45:23 backend=blobstore +2025/07/10 10:45:23 database=rocksdb +2025/07/10 10:45:23 conf=blob.conf +2025/07/10 10:45:23 runName=mybench +2025/07/10 10:45:23 poolName= +2025/07/10 10:45:23 objSize=10Ki +2025/07/10 10:45:23 maxObjCnt=-1 +2025/07/10 10:45:23 durationSecs=5 +2025/07/10 10:45:23 threads=2 +2025/07/10 10:45:23 loops=2 +2025/07/10 10:45:23 interval=1.000000 +2025/07/10 10:45:23 dbDir=build/db/ +2025/07/12 21:28:47 Running Loop 0 Put Test +2025/07/12 21:28:48 Loop: 0, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 547, MB/s: 5.22, IO/s: 547, Lat(ms): [ min: 1.6, avg: 3.4, 99%: 5.6, max: 317.2 ], Slowdowns: 0 +2025/07/12 21:28:49 Loop: 0, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 704, MB/s: 6.71, IO/s: 704, Lat(ms): [ min: 1.8, avg: 2.6, 99%: 6.3, max: 10.2 ], Slowdowns: 0 +2025/07/12 21:28:50 Loop: 0, Int: 2, Dur(s): 1.0, Mode: PUT, Ops: 626, MB/s: 5.97, IO/s: 626, Lat(ms): [ min: 1.8, avg: 2.9, 99%: 11.9, max: 19.7 ], Slowdowns: 0 +2025/07/12 21:28:51 Loop: 0, Int: 3, Dur(s): 1.0, Mode: PUT, Ops: 681, MB/s: 6.49, IO/s: 681, Lat(ms): [ min: 1.8, avg: 2.7, 99%: 5.4, max: 12.0 ], Slowdowns: 0 +2025/07/12 21:28:52 Loop: 0, Int: 4, Dur(s): 1.0, Mode: PUT, Ops: 524, MB/s: 5.00, IO/s: 524, Lat(ms): [ min: 1.9, avg: 3.5, 99%: 8.1, max: 40.8 ], Slowdowns: 0 +2025/07/12 21:28:52 Loop: 0, Int: TOTAL, Dur(s): 5.0, Mode: PUT, Ops: 3084, MB/s: 5.88, IO/s: 617, Lat(ms): [ min: 1.6, avg: 3.0, 99%: 7.8, max: 317.2 ], Slowdowns: 0 +2025/07/12 21:28:52 The subsequent '-m g' or '-m d' test should be specified with '-n 3084' +2025/07/12 21:28:52 Running Loop 0 Get Test +2025/07/12 21:28:53 Loop: 0, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 1355, MB/s: 12.92, IO/s: 1355, Lat(ms): [ min: 1.0, avg: 1.5, 99%: 2.9, max: 23.5 ], Slowdowns: 0 +2025/07/12 21:28:54 Loop: 0, Int: 1, Dur(s): 1.0, Mode: GET, Ops: 1344, MB/s: 12.82, IO/s: 1344, Lat(ms): [ min: 0.6, avg: 1.5, 99%: 3.3, max: 22.6 ], Slowdowns: 0 +2025/07/12 21:28:54 Loop: 0, Int: TOTAL, Dur(s): 2.3, Mode: GET, Ops: 3084, MB/s: 12.94, IO/s: 1357, Lat(ms): [ min: 0.6, avg: 1.5, 99%: 3.1, max: 23.5 ], Slowdowns: 0 +2025/07/12 21:28:54 Running Loop 0 Del Test +2025/07/12 21:28:55 Loop: 0, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 2428, MB/s: 23.16, IO/s: 2428, Lat(ms): [ min: 0.4, avg: 0.7, 99%: 1.2, max: 118.8 ], Slowdowns: 0 +2025/07/12 21:28:55 Loop: 0, Int: TOTAL, Dur(s): 1.3, Mode: DEL, Ops: 3084, MB/s: 22.98, IO/s: 2409, Lat(ms): [ min: 0.4, avg: 0.7, 99%: 1.2, max: 118.8 ], Slowdowns: 0 +2025/07/12 21:28:55 Running Loop 1 Put Test +2025/07/12 21:28:56 Loop: 1, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 755, MB/s: 7.20, IO/s: 755, Lat(ms): [ min: 1.7, avg: 2.4, 99%: 4.3, max: 9.8 ], Slowdowns: 0 +2025/07/12 21:28:57 Loop: 1, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 763, MB/s: 7.28, IO/s: 763, Lat(ms): [ min: 1.6, avg: 2.4, 99%: 4.1, max: 9.9 ], Slowdowns: 0 +2025/07/12 21:28:58 Loop: 1, Int: 2, Dur(s): 1.0, Mode: PUT, Ops: 707, MB/s: 6.74, IO/s: 707, Lat(ms): [ min: 1.8, avg: 2.6, 99%: 4.4, max: 7.1 ], Slowdowns: 0 +2025/07/12 21:28:59 Loop: 1, Int: 3, Dur(s): 1.0, Mode: PUT, Ops: 739, MB/s: 7.05, IO/s: 739, Lat(ms): [ min: 1.6, avg: 2.5, 99%: 4.2, max: 7.8 ], Slowdowns: 0 +2025/07/12 21:29:00 Loop: 1, Int: 4, Dur(s): 1.0, Mode: PUT, Ops: 740, MB/s: 7.06, IO/s: 740, Lat(ms): [ min: 1.7, avg: 2.5, 99%: 4.7, max: 7.7 ], Slowdowns: 0 +2025/07/12 21:29:00 Loop: 1, Int: TOTAL, Dur(s): 5.0, Mode: PUT, Ops: 3706, MB/s: 7.07, IO/s: 741, Lat(ms): [ min: 1.6, avg: 2.5, 99%: 4.4, max: 9.9 ], Slowdowns: 0 +2025/07/12 21:29:00 The subsequent '-m g' or '-m d' test should be specified with '-n 3706' +2025/07/12 21:29:00 Running Loop 1 Get Test +2025/07/12 21:29:01 Loop: 1, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 1272, MB/s: 12.13, IO/s: 1272, Lat(ms): [ min: 0.9, avg: 1.6, 99%: 4.2, max: 12.9 ], Slowdowns: 0 +2025/07/12 21:29:02 Loop: 1, Int: 1, Dur(s): 1.0, Mode: GET, Ops: 1296, MB/s: 12.36, IO/s: 1296, Lat(ms): [ min: 0.8, avg: 1.5, 99%: 3.8, max: 15.0 ], Slowdowns: 0 +2025/07/12 21:29:03 Loop: 1, Int: TOTAL, Dur(s): 2.9, Mode: GET, Ops: 3706, MB/s: 12.20, IO/s: 1279, Lat(ms): [ min: 0.8, avg: 1.6, 99%: 3.7, max: 15.0 ], Slowdowns: 0 +2025/07/12 21:29:03 Running Loop 1 Del Test +2025/07/12 21:29:04 Loop: 1, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 2880, MB/s: 27.47, IO/s: 2880, Lat(ms): [ min: 0.4, avg: 0.6, 99%: 1.1, max: 1.7 ], Slowdowns: 0 +2025/07/12 21:29:04 Loop: 1, Int: TOTAL, Dur(s): 1.3, Mode: DEL, Ops: 3706, MB/s: 27.38, IO/s: 2871, Lat(ms): [ min: 0.4, avg: 0.6, 99%: 1.2, max: 2.4 ], Slowdowns: 0 +``` + +### Testing Overhead + +* Just to evaluate the overhead of the tool itself + +```text +# build/bin/blobstore/blobstore-bench -b dummy -d 3 -m pgd -pr dummytest -s 10Ki -t 2 -l 2 +2025/07/10 10:48:04 DataSize=10240 MD5=80a8c5663ee7ca537add8fd887c6ce93 +2025/07/10 10:48:04 BlobStore Benchmark +2025/07/10 10:48:04 Parameters: +2025/07/10 10:48:04 backend=dummy +2025/07/10 10:48:04 database= +2025/07/12 21:29:35 conf= +2025/07/12 21:29:35 runName=dummytest +2025/07/12 21:29:35 poolName= +2025/07/12 21:29:35 objSize=10Ki +2025/07/12 21:29:35 maxObjCnt=-1 +2025/07/12 21:29:35 durationSecs=3 +2025/07/12 21:29:35 threads=2 +2025/07/12 21:29:35 loops=2 +2025/07/12 21:29:35 interval=1.000000 +2025/07/12 21:29:35 dbDir= +2025/07/12 21:29:35 Running Loop 0 Put Test +2025/07/12 21:29:36 Loop: 0, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 4868550, MB/s: 46430.11, IO/s: 4868550, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:37 Loop: 0, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 4275910, MB/s: 40778.26, IO/s: 4275910, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:39 Loop: 0, Int: TOTAL, Dur(s): 3.0, Mode: PUT, Ops: 13223270, MB/s: 42060.02, IO/s: 4410313, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:39 The subsequent '-m g' or '-m d' test should be specified with '-n 13223270' +2025/07/12 21:29:39 Running Loop 0 Get Test +2025/07/12 21:29:40 Loop: 0, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 5984797, MB/s: 57075.47, IO/s: 5984797, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.3 ], Slowdowns: 0 +2025/07/12 21:29:41 Loop: 0, Int: 1, Dur(s): 1.0, Mode: GET, Ops: 5263634, MB/s: 50197.93, IO/s: 5263634, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.1 ], Slowdowns: 0 +2025/07/12 21:29:42 Loop: 0, Int: TOTAL, Dur(s): 2.4, Mode: GET, Ops: 13223270, MB/s: 53229.78, IO/s: 5581547, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.3 ], Slowdowns: 0 +2025/07/12 21:29:42 Running Loop 0 Del Test +2025/07/12 21:29:43 Loop: 0, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 5578031, MB/s: 53196.25, IO/s: 5578031, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.1 ], Slowdowns: 0 +2025/07/12 21:29:44 Loop: 0, Int: 1, Dur(s): 1.0, Mode: DEL, Ops: 5294927, MB/s: 50496.36, IO/s: 5294927, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.1 ], Slowdowns: 0 +2025/07/12 21:29:44 Loop: 0, Int: TOTAL, Dur(s): 2.5, Mode: DEL, Ops: 13223270, MB/s: 51446.07, IO/s: 5394511, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.1 ], Slowdowns: 0 +2025/07/12 21:29:44 Running Loop 1 Put Test +2025/07/12 21:29:45 Loop: 1, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 5397124, MB/s: 51470.99, IO/s: 5397124, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:46 Loop: 1, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 4989291, MB/s: 47581.59, IO/s: 4989291, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:48 Loop: 1, Int: TOTAL, Dur(s): 3.0, Mode: PUT, Ops: 15547268, MB/s: 49423.48, IO/s: 5182427, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:48 The subsequent '-m g' or '-m d' test should be specified with '-n 15547268' +2025/07/12 21:29:48 Running Loop 1 Get Test +2025/07/12 21:29:49 Loop: 1, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 5637404, MB/s: 53762.47, IO/s: 5637404, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:50 Loop: 1, Int: 1, Dur(s): 1.0, Mode: GET, Ops: 5601770, MB/s: 53422.64, IO/s: 5601770, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:51 Loop: 1, Int: TOTAL, Dur(s): 2.8, Mode: GET, Ops: 15547268, MB/s: 53826.12, IO/s: 5644078, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:51 Running Loop 1 Del Test +2025/07/12 21:29:52 Loop: 1, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 5658764, MB/s: 53966.18, IO/s: 5658764, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:53 Loop: 1, Int: 1, Dur(s): 1.0, Mode: DEL, Ops: 5424048, MB/s: 51727.75, IO/s: 5424048, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +2025/07/12 21:29:54 Loop: 1, Int: TOTAL, Dur(s): 2.8, Mode: DEL, Ops: 15547268, MB/s: 53546.62, IO/s: 5614770, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.2 ], Slowdowns: 0 +``` + +## Others + +* `-m`: You can specify a single operation or a combination of multiple operations, such as `p`, `g`, `d`, `pg`, `pd`, `pggg`. However, once `d` is executed, the data will be deleted. +* `-n`: This must be specified for the `d/del` operation to determine the maximum object number, which is used to define the stopping boundary for the operation.