diff --git a/blobstore/access/stream/metric.go b/blobstore/access/stream/metric.go index bb140f8df..acc9718ab 100644 --- a/blobstore/access/stream/metric.go +++ b/blobstore/access/stream/metric.go @@ -15,9 +15,12 @@ package stream import ( + "os" + "github.com/prometheus/client_golang/prometheus" "github.com/cubefs/cubefs/blobstore/common/proto" + "github.com/cubefs/cubefs/blobstore/common/rpc/auditlog" ) var unhealthMetric = prometheus.NewCounterVec( @@ -40,11 +43,27 @@ var downloadMetric = prometheus.NewCounterVec( []string{"cluster", "way", "reason"}, ) +var readwriteMetric *prometheus.HistogramVec + var SteamReportDownload = reportDownload func init() { prometheus.MustRegister(unhealthMetric) prometheus.MustRegister(downloadMetric) + + hostname, _ := os.Hostname() + readwriteMetric = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "blobstore", + Subsystem: "access", + Name: "read_write_duration_ms", + Help: "read write duration ms", + Buckets: auditlog.Buckets, + ConstLabels: map[string]string{"host": hostname}, + }, + []string{"cluster", "idc", "api"}, + ) + prometheus.MustRegister(readwriteMetric) } func reportUnhealth(cid proto.ClusterID, action, module, host, reason string) { @@ -54,3 +73,8 @@ func reportUnhealth(cid proto.ClusterID, action, module, host, reason string) { func reportDownload(cid proto.ClusterID, way, reason string) { downloadMetric.WithLabelValues(cid.ToString(), way, reason).Inc() } + +// upload_read, upload_write, download_read, download_write +func reportReadwrite(cid, idc, api string, ms int64) { + readwriteMetric.WithLabelValues(cid, idc, api).Observe(float64(ms)) +} diff --git a/blobstore/access/stream/stream_get.go b/blobstore/access/stream/stream_get.go index 4a4dbc32c..6ff87d41b 100644 --- a/blobstore/access/stream/stream_get.go +++ b/blobstore/access/stream/stream_get.go @@ -142,6 +142,7 @@ func (h *Handler) Get(ctx context.Context, w io.Writer, location proto.Location, getTime := new(timeReadWrite) defer func() { span.AppendRPCTrackLog([]string{getTime.String()}) + getTime.Report(clusterID.ToString(), h.IDC, false) }() // try to read data shard only, diff --git a/blobstore/access/stream/stream_put.go b/blobstore/access/stream/stream_put.go index 6c4fe571a..16b0e3952 100644 --- a/blobstore/access/stream/stream_put.go +++ b/blobstore/access/stream/stream_put.go @@ -102,6 +102,7 @@ func (h *Handler) Put(ctx context.Context, // release ec buffer which have not takeover buffer.Release() span.AppendRPCTrackLog([]string{putTime.String()}) + putTime.Report(clusterID.ToString(), h.IDC, true) }() // concurrent buffer in per request diff --git a/blobstore/access/stream/stream_putat.go b/blobstore/access/stream/stream_putat.go index f96bd62d0..cf28d0d5f 100644 --- a/blobstore/access/stream/stream_putat.go +++ b/blobstore/access/stream/stream_putat.go @@ -62,6 +62,7 @@ func (h *Handler) PutAt(ctx context.Context, rc io.Reader, defer func() { buffer.Release() span.AppendRPCTrackLog([]string{putTime.String()}) + putTime.Report(clusterID.ToString(), h.IDC, true) }() shards, err := encoder.Split(buffer.ECDataBuf) diff --git a/blobstore/access/stream/stream_time.go b/blobstore/access/stream/stream_time.go index 78073fbcf..86ad52ad8 100644 --- a/blobstore/access/stream/stream_time.go +++ b/blobstore/access/stream/stream_time.go @@ -38,6 +38,16 @@ func (t *timeReadWrite) IncW(dur time.Duration) { atomic.AddInt64(&t.w, int64(dur)) } +func (t *timeReadWrite) Report(cid, idc string, upload bool) { + if upload { + reportReadwrite(cid, idc, "upload_read", atomic.LoadInt64(&t.r)/1e6) + reportReadwrite(cid, idc, "upload_write", atomic.LoadInt64(&t.w)/1e6) + } else { + reportReadwrite(cid, idc, "download_read", atomic.LoadInt64(&t.r)/1e6) + reportReadwrite(cid, idc, "download_write", atomic.LoadInt64(&t.w)/1e6) + } +} + // String within milliseconds func (t *timeReadWrite) String() string { a := atomic.LoadInt64(&t.a) / 1e6