// Copyright 2019 The CubeFS Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or // implied. See the License for the specific language governing // permissions and limitations under the License. package objectnode import ( "crypto/hmac" "crypto/md5" "crypto/sha1" "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/base64" "fmt" "net" "net/http" "net/url" "os" "regexp" "strings" "time" "github.com/cubefs/cubefs/util" "github.com/cubefs/cubefs/util/log" ) const ( pathSep = "/" tempFileNameSep = "_" ) var ( emptyPathItem = PathItem{} // Regular expression used to match one or more path separators. regexpSepPrefix = regexp.MustCompile("^/+") // Regular expression to match more than two consecutive path separators. regexpDupSep = regexp.MustCompile("/{2,}") ) var keyEscapedSkipBytes = []byte{'/', '*', '.', '-', '_'} // PathItem defines path node attribute information, // including node name and whether it is a directory. type PathItem struct { Name string IsDirectory bool } // PathIterator is a path iterator. Allocated to sequentially iterate each path node from a complete path. type PathIterator struct { cursor int path string inited bool } func (p *PathIterator) init() { if !p.inited { // p.path = strings.TrimSpace(p.path) loc := regexpSepPrefix.FindStringIndex(p.path) if len(loc) == 2 { p.path = p.path[loc[1]:] } p.path = regexpDupSep.ReplaceAllString(p.path, pathSep) p.inited = true } } func (p *PathIterator) HasNext() bool { p.init() return p.cursor < len(p.path) } func (p *PathIterator) Reset() { p.cursor = 0 } func (p PathIterator) ToSlice() []PathItem { newIterator := NewPathIterator(p.path) result := make([]PathItem, 0) for newIterator.HasNext() { result = append(result, newIterator.Next()) } return result } func (p *PathIterator) Next() PathItem { p.init() if p.cursor >= len(p.path) { return emptyPathItem } var item PathItem index := strings.Index(p.path[p.cursor:], pathSep) if index >= 0 { item = PathItem{ Name: p.path[p.cursor : p.cursor+index], IsDirectory: true, } p.cursor = p.cursor + index + 1 } else { item = PathItem{ Name: p.path[p.cursor:], IsDirectory: false, } p.cursor = len(p.path) } return item } func NewPathIterator(path string) PathIterator { return PathIterator{ path: path, } } func splitPath(path string) (dirs []string, filename string) { pathParts := strings.Split(path, pathSep) if len(pathParts) > 1 { dirs = pathParts[:len(pathParts)-1] } filename = pathParts[len(pathParts)-1] return } func tempFileName(origin string) string { return "." + origin + tempFileNameSep + util.RandomString(16, util.LowerLetter|util.UpperLetter) } func formatSimpleTime(time time.Time) string { return time.UTC().Format("2006-01-02T15:04:05") } func formatTimeISO(time time.Time) string { return time.UTC().Format("2006-01-02T15:04:05.000Z") } func formatTimeISOLocal(time time.Time) string { return time.Local().Format("2006-01-02T15:04:05.000Z") } func formatTimeRFC1123(time time.Time) string { return time.UTC().Format(http.TimeFormat) } func parseTimeRFC1123(timeStr string) (time.Time, error) { t, err := time.Parse("Mon, 2 Jan 2006 15:04:05 GMT", timeStr) if err != nil { return t, err } return t, err } func transferError(key string, err error) Error { // TODO: complete sys error transfer ossError := Error{ Key: key, Message: err.Error(), } return ossError } // get request remote IP func getRequestIP(r *http.Request) string { IPAddress := r.Header.Get("X-Real-Ip") if IPAddress == "" { IPAddress = r.Header.Get("X-Forwarded-For") } if IPAddress == "" { IPAddress = r.RemoteAddr } if ok := strings.Contains(IPAddress, ":"); ok { IPAddress = strings.Split(IPAddress, ":")[0] } return IPAddress } // check ipnet contains ip func isIPNetContainsIP(ipStr, ipnetStr string) (bool, error) { if !strings.Contains(ipnetStr, "/") { if ipStr == ipnetStr { return true, nil } else { return false, nil } } _, ipnet, err := net.ParseCIDR(ipnetStr) if err != nil { log.LogInfof("parse ipnet error ipnet %v", ipnetStr) return false, err } ip := net.ParseIP(ipStr) if ipnet.Contains(ip) { return true, nil } return false, nil } func patternMatch(pattern, key string) bool { if pattern == "" { return key == pattern } if pattern == "*" { return true } matched, err := regexp.MatchString(pattern, key) if err != nil { log.LogErrorf("patternMatch error %v", err) return false } return matched } func wrapUnescapedQuot(src string) string { return "\"" + src + "\"" } func encodeKey(key, encodingType string) string { isKeyEscapedSkipByte := func(b byte) bool { for _, skipByte := range keyEscapedSkipBytes { if b == skipByte { return true } } return false } if strings.ToLower(encodingType) == "url" { encodedKeyBuilder := strings.Builder{} for i := 0; i < len(key); i++ { b := byte(key[i]) if isKeyEscapedSkipByte(b) { encodedKeyBuilder.Write([]byte{b}) } else { encodedKeyBuilder.Write([]byte(url.QueryEscape(string(b)))) } } return encodedKeyBuilder.String() } return key } func SplitFileRange(size, blockSize int64) (ranges [][2]int64) { blocks := size / blockSize if size%blockSize != 0 { blocks += 1 } ranges = make([][2]int64, 0, blocks) remain := size aboveRage := [2]int64{0, 0} for remain > 0 { curRange := [2]int64{aboveRage[1], 0} if remain < blockSize { curRange[1] = size remain = 0 } else { curRange[1] = blockSize remain -= blockSize } ranges = append(ranges, curRange) aboveRage[0], aboveRage[1] = curRange[0], curRange[1] } return ranges } // Checking and parsing user-defined metadata from request header. // The optional user-defined metadata names must begin with "x-amz-meta-" to // distinguish them from other HTTP headers. // Notes: // The PUT request header is limited to 8 KB in size. Within the PUT request header, // the user-defined metadata is limited to 2 KB in size. The size of user-defined // metadata is measured by taking the sum of the number of bytes in the UTF-8 encoding // of each key and value. // Reference: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html func ParseUserDefinedMetadata(header http.Header) map[string]string { metadata := make(map[string]string) for name, values := range header { if strings.HasPrefix(name, http.CanonicalHeaderKey(XAmzMetaPrefix)) && name != http.CanonicalHeaderKey(XAmzMetadataDirective) { metaName := strings.ToLower(name[len(XAmzMetaPrefix):]) metaValue := strings.Join(values, ",") if !strings.HasPrefix(metaName, "oss:") { metadata[metaName] = metaValue } } } return metadata } // validate Cache-Control var ( cacheControlDir = []string{"public", "private", "no-cache", "no-store", "no-transform", "must-revalidate", "proxy-revalidate"} maxAgeRegexp = regexp.MustCompile("^((max-age)|(s-maxage))=[1-9][0-9]*$") ) func ValidateCacheControl(cacheControl string) bool { cacheDirs := strings.Split(cacheControl, ",") for _, dir := range cacheDirs { if !contains(cacheControlDir, dir) && !maxAgeRegexp.MatchString(dir) { log.LogErrorf("invalid cache-control directive: %v", dir) return false } } return true } func ValidateCacheExpires(expires string) bool { var err error var stamp time.Time if stamp, err = time.Parse(RFC1123Format, expires); err != nil { log.LogErrorf("invalid expires: %v", expires) return false } expiresInt := stamp.Unix() now := time.Now().UTC().Unix() if now < expiresInt { return true } log.LogErrorf("Expires less than now: %v, now: %v", expires, now) return false } func GetMD5(b []byte) string { hash := md5.New() hash.Write(b) return base64.StdEncoding.EncodeToString(hash.Sum(nil)) } func contains(items []string, key string) bool { for _, s := range items { if s == key { return true } } return false } func MakeMD5(data []byte) []byte { hash := md5.New() hash.Write(data) return hash.Sum(nil) } func MakeSha256(data []byte) []byte { hash := sha256.New() hash.Write(data) return hash.Sum(nil) } func MakeHmacSha1(key, data []byte) []byte { hash := hmac.New(sha1.New, key) hash.Write(data) return hash.Sum(nil) } func MakeHmacSha256(key, data []byte) []byte { hash := hmac.New(sha256.New, key) hash.Write(data) return hash.Sum(nil) } func ParseCompatibleTime(timeStr string) (t time.Time, err error) { layouts := []string{ time.RFC1123, time.RFC1123Z, time.RFC3339, ISO8601Format, ISO8601Layout, ISO8601LayoutCompatible, } for _, layout := range layouts { if t, err = time.Parse(layout, timeStr); err == nil { return } } return } // GetRootCAs loads all X.509 certificates from the specified files. func GetRootCAs(file ...string) (*x509.CertPool, error) { rootCAs := x509.NewCertPool() for _, f := range file { rootPEM, err := os.ReadFile(f) if err != nil || rootPEM == nil { return nil, fmt.Errorf("loading or parsing rootCA file failed: %w", err) } if !rootCAs.AppendCertsFromPEM(rootPEM) { return nil, fmt.Errorf("failed to parse root certificate from %q", f) } } return rootCAs, nil } // NewTLSConfig creates a new tls.Config object for configuring TLS settings. // clientCert: file path to the client's certificate. // clientKey: file path to the client's private key. func NewTLSConfig(clientCert, clientKey string) (*tls.Config, error) { tlsConfig := tls.Config{ MinVersion: tls.VersionTLS12, } if clientCert != "" && clientKey != "" { cert, err := tls.LoadX509KeyPair(clientCert, clientKey) if err != nil { return &tlsConfig, err } tlsConfig.Certificates = []tls.Certificate{cert} } return &tlsConfig, nil }