mirror of
https://github.com/cubefs/cubefs.git
synced 2026-08-02 02:00:56 +00:00
feat(rpc): apply filter in auditlog
Signed-off-by: slasher <shenjie1@oppo.com>
This commit is contained in:
slasher
梁曟風
parent
91421b7dc7
commit
7d3a90f8e9
@ -21,7 +21,6 @@ import (
|
||||
"net/http"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@ -54,6 +53,7 @@ type jsonAuditlog struct {
|
||||
decoder Decoder
|
||||
metricSender MetricSender
|
||||
logFile LogCloser
|
||||
logFilter LogFilter
|
||||
|
||||
logPool sync.Pool
|
||||
bodyPool sync.Pool
|
||||
@ -138,11 +138,17 @@ func Open(module string, cfg *Config) (ph rpc.ProgressHandler, logFile LogCloser
|
||||
}
|
||||
}
|
||||
|
||||
logFilter, err := newLogFilter(cfg.Filters)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Info(err, "new log filter").Detail(err)
|
||||
}
|
||||
|
||||
return &jsonAuditlog{
|
||||
module: module,
|
||||
decoder: &defaultDecoder{},
|
||||
metricSender: NewPrometheusSender(cfg.MetricConfig),
|
||||
logFile: logFile,
|
||||
logFilter: logFilter,
|
||||
|
||||
logPool: sync.Pool{
|
||||
New: func() interface{} {
|
||||
@ -251,7 +257,7 @@ func (j *jsonAuditlog) Handler(w http.ResponseWriter, req *http.Request, f func(
|
||||
|
||||
j.metricSender.Send(auditLog.ToBytesWithTab(b))
|
||||
|
||||
if j.logFile == nil || (len(j.cfg.KeywordsFilter) > 0 && defaultLogFilter(req, j.cfg.KeywordsFilter)) {
|
||||
if j.logFile == nil || j.logFilter.Filter(auditLog) {
|
||||
return
|
||||
}
|
||||
|
||||
@ -268,18 +274,6 @@ func (j *jsonAuditlog) Handler(w http.ResponseWriter, req *http.Request, f func(
|
||||
}
|
||||
}
|
||||
|
||||
// defaultLogFilter support uri and method filter based on keywords
|
||||
func defaultLogFilter(r *http.Request, words []string) bool {
|
||||
method := strings.ToLower(r.Method)
|
||||
for _, word := range words {
|
||||
str := strings.ToLower(word)
|
||||
if method == str || strings.Contains(r.RequestURI, str) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ExtraHeader provides extra response header writes to the ResponseWriter.
|
||||
func ExtraHeader(w http.ResponseWriter) http.Header {
|
||||
h := make(http.Header)
|
||||
|
||||
@ -56,6 +56,7 @@ func initServer(t *testing.T, name string, cfg Config) (server *httptest.Server,
|
||||
|
||||
cfg.LogDir = tmpDir
|
||||
cfg.MetricConfig = PrometheusConfig{Idc: name}
|
||||
cfg.Filters = []FilterConfig{{Must: Conditions{"term": {"method": "GET"}}}}
|
||||
|
||||
var ah rpc.ProgressHandler
|
||||
ah, lc, err = Open(name, &cfg)
|
||||
@ -177,7 +178,7 @@ func initNoContentLengthServer(t *testing.T) (server *httptest.Server, tmpDir st
|
||||
|
||||
func TestOpen(t *testing.T) {
|
||||
cfg := Config{
|
||||
KeywordsFilter: []string{"Get"},
|
||||
Filters: []FilterConfig{{Must: Conditions{"term": {"method": "GET"}}}},
|
||||
}
|
||||
server, tmpDir, lc := initServer(t, "testOpen", cfg)
|
||||
defer func() {
|
||||
|
||||
@ -38,8 +38,8 @@ type Config struct {
|
||||
Backup int `json:"backup"`
|
||||
MetricConfig PrometheusConfig `json:"metric_config"`
|
||||
|
||||
// KeywordsFilter log filter based on uri and request method
|
||||
KeywordsFilter []string `json:"keywords_filter"`
|
||||
// Filters are or relations
|
||||
Filters []FilterConfig `json:"filters"`
|
||||
|
||||
// LogFormat valid value is "text" or "json", default is "text"
|
||||
LogFormat string `json:"log_format"`
|
||||
|
||||
@ -27,6 +27,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/cubefs/cubefs/blobstore/common/rpc"
|
||||
"github.com/cubefs/cubefs/blobstore/common/rpc/auditlog"
|
||||
"github.com/cubefs/cubefs/blobstore/common/trace"
|
||||
)
|
||||
|
||||
@ -460,6 +461,10 @@ func (a *app) List(c *rpc.Context) {
|
||||
files = append(files, resp)
|
||||
}
|
||||
|
||||
extraHeader := auditlog.ExtraHeader(c.Writer)
|
||||
extraHeader.Add("http-header-key", "http-header-value")
|
||||
extraHeader["oWN-defined-KEY"] = []string{"ODvalue", "0x10"}
|
||||
|
||||
c.RespondStatusData(http.StatusOK, files)
|
||||
}
|
||||
|
||||
|
||||
@ -2,7 +2,14 @@
|
||||
"log_level": "debug",
|
||||
"auditlog": {
|
||||
"logdir": "/tmp/examples/",
|
||||
"rotate_new": false
|
||||
"rotate_new": false,
|
||||
"filters": [
|
||||
{
|
||||
"must": {"range": {"status_code": "405-405"}},
|
||||
"must_not": {"match": {"method": "POST"}}
|
||||
},
|
||||
{"should": {"term": {"path": ["/x", "/y", "/z"]}}}
|
||||
]
|
||||
},
|
||||
"app": {
|
||||
"bind_addr": ":9999",
|
||||
|
||||
@ -17,13 +17,12 @@
|
||||
"service": "服务名",
|
||||
"tag": "标签",
|
||||
"team": "团队",
|
||||
|
||||
"enable_req_length_cnt": "是否启用请求长度统计，true或者false,默认false",
|
||||
"enable_resp_length_cnt": "是否启用响应长度统计，true或者false,默认false",
|
||||
"enable_resp_duration": "是否启用响应时延，true或者false,默认false",
|
||||
"max_api_level": "api最大层级数，如/get/name为2"
|
||||
},
|
||||
"keywords_filter": "关键字过滤，目前支持url和请求方法过滤，如/download、get等"
|
||||
"filters": "按照日志字段多条件组合匹配过滤日志"
|
||||
},
|
||||
"auth": {
|
||||
"enable_auth": "是否开启鉴权，true或者false，默认false",
|
||||
@ -38,4 +37,4 @@
|
||||
"maxbackups": "保留日志文件个数"
|
||||
}
|
||||
}
|
||||
```
|
||||
```
|
||||
|
||||
@ -263,11 +263,9 @@ BlobNode的配置是基于[公有配置](./base.md)，以下配置说明主要
|
||||
"chunkbits": 29,
|
||||
"log_file_suffix": ".log",
|
||||
"backup": 10,
|
||||
"keywords_filter": [
|
||||
"list",
|
||||
"metrics",
|
||||
"/shard/get/"
|
||||
],
|
||||
"filters": [
|
||||
{"should": {"match": {"path": ["list", "metrics", "/shard/get/"]}}}
|
||||
],
|
||||
"metric_config": {
|
||||
"idc": "bjht",
|
||||
"service": "BLOBNODE",
|
||||
@ -292,4 +290,4 @@ BlobNode的配置是基于[公有配置](./base.md)，以下配置说明主要
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
```
|
||||
|
||||
@ -17,13 +17,12 @@ The basic configuration is shared by each module and mainly includes server port
|
||||
"service": "service name",
|
||||
"tag": "tag",
|
||||
"team": "team",
|
||||
|
||||
"enable_req_length_cnt": "whether to enable request length statistics, true or false, default is false",
|
||||
"enable_resp_length_cnt": "whether to enable response length statistics, true or false, default is false",
|
||||
"enable_resp_duration": "whether to enable response latency, true or false, default is false",
|
||||
"max_api_level": "maximum API level, such as 2 for /get/name"
|
||||
},
|
||||
"keywords_filter": "keyword filter, currently supports URL and request method filtering, such as /download, GET, etc."
|
||||
"filters": "Filter log by multi-criteria matching of log's fields"
|
||||
},
|
||||
"auth": {
|
||||
"enable_auth": "whether to enable authentication, true or false, default is false",
|
||||
@ -38,4 +37,4 @@ The basic configuration is shared by each module and mainly includes server port
|
||||
"maxbackups": "number of log files to keep"
|
||||
}
|
||||
}
|
||||
```
|
||||
```
|
||||
|
||||
@ -265,11 +265,9 @@ BlobNode configuration is based on the [public configuration](./base.md), and th
|
||||
"chunkbits": 29,
|
||||
"log_file_suffix": ".log",
|
||||
"backup": 10,
|
||||
"keywords_filter": [
|
||||
"list",
|
||||
"metrics",
|
||||
"/shard/get/"
|
||||
],
|
||||
"filters": [
|
||||
{"should": {"match": {"path": ["list", "metrics", "/shard/get/"]}}}
|
||||
],
|
||||
"metric_config": {
|
||||
"idc": "bjht",
|
||||
"service": "BLOBNODE",
|
||||
@ -294,4 +292,4 @@ BlobNode configuration is based on the [public configuration](./base.md), and th
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
```
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
