Add: CLI tool to improve cluster management

Signed-off-by: Mofei Zhang <mofei2816@gmail.com>

cli/cli-sample.json

@@ -0,0 +1,5 @@
+{
+  "masterAddr": [
+    "master.chubao.io"
+  ]
+}

cli/cli.go

@@ -0,0 +1,92 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/chubaofs/chubaofs/cli/cmd"
+	"github.com/chubaofs/chubaofs/sdk/master"
+	"github.com/spf13/cobra"
+)
+
+var (
+	CommitID   string
+	BranchName string
+	BuildTime  string
+)
+
+var (
+	defaultHomeDir, _ = os.UserHomeDir()
+	defaultConfigName = ".cfs-cli.json"
+	defaultConfigPath = path.Join(defaultHomeDir, defaultConfigName)
+	defaultConfigData = []byte(`
+{
+  "masterAddr": [
+    "master.chubao.io"
+  ]
+}
+`)
+)
+
+type config struct {
+	MasterAddr []string `json:"masterAddr"`
+}
+
+func runCLI() (err error) {
+	var cfg *config
+	if cfg, err = loadConfig(); err != nil {
+		return
+	}
+	err = setupCommands(cfg).Execute()
+	return
+}
+
+func loadConfig() (*config, error) {
+	var err error
+	var configData []byte
+	if configData, err = ioutil.ReadFile(defaultConfigPath); err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	if os.IsNotExist(err) {
+		if err = ioutil.WriteFile(defaultConfigPath, defaultConfigData, 0600); err != nil {
+			return nil, err
+		}
+		configData = defaultConfigData
+	}
+	var config = &config{}
+	if err = json.Unmarshal(configData, config); err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+func setupCommands(cfg *config) *cobra.Command {
+	fmt.Printf("Using master address: %v\n", cfg.MasterAddr)
+	var mc = master.NewMasterClient(cfg.MasterAddr, false)
+	return cmd.NewRootCmd(mc)
+}
+
+func main() {
+	var err error
+	if err = runCLI(); err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+}

cli/cmd/cluster.go

@@ -0,0 +1,75 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/chubaofs/chubaofs/proto"
+	"github.com/chubaofs/chubaofs/sdk/master"
+	"github.com/spf13/cobra"
+)
+
+const (
+	cmdClusterUse   = "cluster [COMMAND]"
+	cmdClusterShort = "Manage cluster components"
+)
+
+func newClusterCmd(client *master.MasterClient) *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   cmdClusterUse,
+		Short: cmdClusterShort,
+	}
+	cmd.AddCommand(
+		newClusterInfoCmd(client),
+	)
+	return cmd
+}
+
+const (
+	cmdClusterInfoUse   = "info"
+	cmdClusterInfoShort = "Show cluster summary information"
+)
+
+func newClusterInfoCmd(client *master.MasterClient) *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   cmdClusterInfoUse,
+		Short: cmdClusterInfoShort,
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			var cv *proto.ClusterView
+			if cv, err = client.AdminAPI().GetCluster(); err != nil {
+				errout("Get cluster info fail:\n%v\n", err)
+				os.Exit(1)
+			}
+			stdout("\n[Summary]\n")
+			stdout(formatClusterView(cv))
+			stdout("\n")
+		},
+	}
+	return cmd
+}
+
+func formatClusterView(cv *proto.ClusterView) string {
+	var sb = strings.Builder{}
+	sb.WriteString(fmt.Sprintf("  Name          : %v\n", cv.Name))
+	sb.WriteString(fmt.Sprintf("  Auto allocate : %v\n", !cv.DisableAutoAlloc))
+	sb.WriteString(fmt.Sprintf("  MetaNode count: %v\n", len(cv.MetaNodes)))
+	sb.WriteString(fmt.Sprintf("  DataNode count: %v\n", len(cv.DataNodes)))
+	sb.WriteString(fmt.Sprintf("  Volume count  : %v\n", len(cv.VolStatInfo)))
+	return sb.String()
+}

cli/cmd/root.go

@@ -0,0 +1,50 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/chubaofs/chubaofs/sdk/master"
+	"github.com/spf13/cobra"
+)
+
+const (
+	cmdRootShort = "ChubaoFS Command Line Interface (CLI)"
+)
+
+func NewRootCmd(client *master.MasterClient) *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   path.Base(os.Args[0]),
+		Short: cmdRootShort,
+		Args:  cobra.MinimumNArgs(0),
+	}
+	cmd.AddCommand(
+		newClusterCmd(client),
+		newVolCmd(client),
+		newUserCmd(client),
+	)
+	return cmd
+}
+
+func stdout(format string, a ...interface{}) {
+	_, _ = fmt.Fprintf(os.Stdout, format, a...)
+}
+
+func errout(format string, a ...interface{}) {
+	_, _ = fmt.Fprintf(os.Stderr, format, a...)
+}

cli/cmd/user.go

@@ -0,0 +1,175 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/chubaofs/chubaofs/proto"
+	"github.com/chubaofs/chubaofs/sdk/master"
+	"github.com/spf13/cobra"
+)
+
+const (
+	cmdUserUse   = "user [COMMAND]"
+	cmdUserShort = "Manage cluster users"
+)
+
+func newUserCmd(client *master.MasterClient) *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   cmdUserUse,
+		Short: cmdUserShort,
+		Args:  cobra.MinimumNArgs(0),
+	}
+	cmd.AddCommand(
+		newUserCreateCmd(client),
+		newUserInfoCmd(client),
+	)
+	return cmd
+}
+
+const (
+	cmdUserCreateUse   = "create [USER ID]"
+	cmdUserCreateShort = "Create a new user"
+)
+
+func newUserCreateCmd(client *master.MasterClient) *cobra.Command {
+	var optPassword string
+	var optAccessKey string
+	var optSecretKey string
+	var optUserType string
+	var cmd = &cobra.Command{
+		Use:   cmdUserCreateUse,
+		Short: cmdUserCreateShort,
+		Args:  cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			var userID = args[0]
+			var password = optPassword
+			var accessKey = optAccessKey
+			var secretKey = optSecretKey
+			var userType = proto.UserTypeFromString(optUserType)
+
+			if !userType.Valid() {
+				errout("Invalid user type.")
+				os.Exit(1)
+			}
+
+			// display information before create
+			var displayPassword = "[default]"
+			if optPassword != "" {
+				displayPassword = optPassword
+			}
+			var displayAccessKey = "[auto generate]"
+			var displaySecretKey = "[auto generate]"
+			if optAccessKey != "" && optSecretKey != "" {
+				displayAccessKey = optAccessKey
+				displaySecretKey = optSecretKey
+			}
+			var displayUserType = userType.String()
+			fmt.Printf("Create a new ChubaoFS cluster user\n")
+			stdout("  User ID   : %v\n", userID)
+			stdout("  Password  : %v\n", displayPassword)
+			stdout("  Access Key: %v\n", displayAccessKey)
+			stdout("  Secret Key: %v\n", displaySecretKey)
+			stdout("  Type      : %v\n", displayUserType)
+
+			// ask user for confirm
+			stdout("\nConfirm (yes/no)[yes]: ")
+			var userConfirm string
+			_, _ = fmt.Scanln(&userConfirm)
+			if userConfirm != "yes" && len(userConfirm) != 0 {
+				stdout("Abort by user.\n")
+				return
+			}
+			var param = proto.UserCreateParam{
+				ID:        userID,
+				Password:  password,
+				AccessKey: accessKey,
+				SecretKey: secretKey,
+				Type:      userType,
+			}
+			var akPolicy *proto.AKPolicy
+			if akPolicy, err = client.UserAPI().Create(&param); err != nil {
+				errout("Create user failed: %v\n", err)
+				os.Exit(1)
+			}
+
+			// display operation result
+			stdout("Create user success:\n")
+			printUserInfo(akPolicy)
+			return
+		},
+	}
+	cmd.Flags().StringVar(&optPassword, "password", "", "Specify user password")
+	cmd.Flags().StringVar(&optAccessKey, "access-key", "", "Specify user access key for object storage interface authentication")
+	cmd.Flags().StringVar(&optSecretKey, "secret-key", "", "Specify user secret key for object storage interface authentication")
+	cmd.Flags().StringVar(&optUserType, "user-type", "normal", "Specify user type [normal | admin]")
+	return cmd
+}
+
+const (
+	cmdUserInfoUse   = "info [USER ID]"
+	cmdUserInfoShort = "Show detail information about specified user"
+)
+
+func newUserInfoCmd(client *master.MasterClient) *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   cmdUserInfoUse,
+		Short: cmdUserInfoShort,
+		Args:  cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			var userID = args[0]
+			var akp *proto.AKPolicy
+			if akp, err = client.UserAPI().GetUserInfo(userID); err != nil {
+				errout("Get user info failed: %v\n", err)
+				os.Exit(1)
+			}
+			printUserInfo(akp)
+		},
+	}
+
+	return cmd
+}
+
+func printUserInfo(akp *proto.AKPolicy) {
+	stdout("\n[Summary]\n")
+	stdout("  User ID    : %v\n", akp.UserID)
+	stdout("  Access Key : %v\n", akp.AccessKey)
+	stdout("  Secret Key : %v\n", akp.SecretKey)
+	stdout("  Type       : %v\n", akp.UserType)
+	stdout("  Create Time: %v\n", akp.CreateTime)
+	if akp.Policy == nil {
+		return
+	}
+	stdout("\n[Own volumes]\n")
+	if len(akp.Policy.OwnVols) != 0 {
+		for _, vol := range akp.Policy.OwnVols {
+			stdout("  %s\n", vol)
+		}
+	} else {
+		stdout("  None\n")
+	}
+	stdout("\n[Authorized volumes]\n")
+	if len(akp.Policy.AuthorizedVols) != 0 {
+		for vol := range akp.Policy.AuthorizedVols {
+			stdout("  %s\n", vol)
+		}
+	} else {
+		stdout("  None\n")
+	}
+}

cli/cmd/vol.go

@@ -0,0 +1,310 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package cmd
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"math"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/chubaofs/chubaofs/proto"
+	"github.com/chubaofs/chubaofs/sdk/master"
+	"github.com/spf13/cobra"
+)
+
+const (
+	cmdVolUse   = "volume [COMMAND]"
+	cmdVolShort = "Manage cluster volumes"
+)
+
+func newVolCmd(client *master.MasterClient) *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   cmdVolUse,
+		Short: cmdVolShort,
+		Args:  cobra.MinimumNArgs(0),
+	}
+	cmd.AddCommand(
+		newVolListCmd(client),
+		newVolCreateCmd(client),
+		newVolInfoCmd(client),
+		newVolDeleteCmd(client),
+	)
+	return cmd
+}
+
+const (
+	cmdVolListUse   = "list"
+	cmdVolListShort = "List cluster volumes"
+)
+
+func newVolListCmd(client *master.MasterClient) *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   cmdVolListUse,
+		Short: cmdVolListShort,
+		Run: func(cmd *cobra.Command, args []string) {
+
+		},
+	}
+	return cmd
+}
+
+const (
+	cmdVolCreateUse             = "create [VOLUME NAME] [USER ID]"
+	cmdVolCreateShort           = "Create a new volume"
+	cmdVolDefaultMPCount        = 3
+	cmdVolDefaultDPSize         = 10
+	cmdVolDefaultCapacity       = 10 // 100GB
+	cmdVolDefaultReplicas       = 3
+	cmdVolDefaultFollowerReader = true
+)
+
+func newVolCreateCmd(client *master.MasterClient) *cobra.Command {
+	var optMPCount int
+	var optDPCount uint64
+	var optCapacity uint64
+	var optReplicas int
+	var optFollowerRead bool
+	var cmd = &cobra.Command{
+		Use:   cmdVolCreateUse,
+		Short: cmdVolCreateShort,
+		Args:  cobra.MinimumNArgs(2),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			var volumeName = args[0]
+			var userID = args[1]
+			stdout("Create a new volume:\n")
+			stdout("  Name                : %v\n", volumeName)
+			stdout("  Owner               : %v\n", userID)
+			stdout("  Meta partition count: %v\n", optMPCount)
+			stdout("  Dara partition count: %v\n", optDPCount)
+			stdout("  Capacity            : %v\n", optCapacity)
+			stdout("  Replicas            : %v\n", optReplicas)
+			stdout("  Allow follower read : %v\n", optFollowerRead)
+
+			// confirm
+			// ask user for confirm
+			stdout("\nConfirm (yes/no)[yes]: ")
+			var userConfirm string
+			_, _ = fmt.Scanln(&userConfirm)
+			if userConfirm != "yes" && len(userConfirm) != 0 {
+				stdout("Abort by user.\n")
+				return
+			}
+			err = client.AdminAPI().CreateVolume(
+				volumeName, userID, optMPCount, optDPCount,
+				optCapacity, optReplicas, optFollowerRead)
+			if err != nil {
+				errout("Create volume failed case:\n%v\n", err)
+				os.Exit(1)
+			}
+			stdout("Create volume success.\n")
+			return
+		},
+	}
+	cmd.Flags().IntVar(&optMPCount, "mp-count", cmdVolDefaultMPCount, "Specify init meta partition count")
+	cmd.Flags().Uint64Var(&optDPCount, "dp-count", cmdVolDefaultDPSize, "Specify init data partition count")
+	cmd.Flags().Uint64Var(&optCapacity, "capacity", cmdVolDefaultCapacity, "Specify volume capacity [Unit: GB]")
+	cmd.Flags().IntVar(&optReplicas, "replicas", cmdVolDefaultReplicas, "Specify volume replicas number")
+	cmd.Flags().BoolVar(&optFollowerRead, "follower-read", cmdVolDefaultFollowerReader, "Allow read form replica follower")
+	return cmd
+}
+
+const (
+	cmdVolInfoUse               = "info [VOLUME NAME]"
+	cmdVolInfoShort             = "Show volume information"
+	cmdVolInfoDefaultMetaDetail = false
+	cmdVolInfoDefaultDataDetail = false
+)
+
+func newVolInfoCmd(client *master.MasterClient) *cobra.Command {
+	var optMetaDetail bool
+	var optDataDetail bool
+	var cmd = &cobra.Command{
+		Use:   cmdVolInfoUse,
+		Short: cmdVolInfoShort,
+		Args:  cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			var volumeName = args[0]
+			var svv *proto.SimpleVolView
+			if svv, err = client.AdminAPI().GetVolumeSimpleInfo(volumeName); err != nil {
+				errout("Get volume info failed:\n%v\n", err)
+				os.Exit(1)
+			}
+			// print summary info
+			stdout("\n[Summary]\n%s\n", formatSimpleVolView(svv))
+
+			// print metadata detail
+			if optMetaDetail {
+				var views []*proto.MetaPartitionView
+				if views, err = client.ClientAPI().GetMetaPartitions(volumeName); err != nil {
+					errout("Get volume metadata detail information failed:\n%v\n", err)
+					os.Exit(1)
+				}
+				stdout("\n[Metadata detail]\n")
+				stdout("  %10v\t%12v\t%12v\t%12v\t%10v\t%18v\t%18v\n",
+					"ID", "MAX INODE", "START", "END", "STATUS", "LEADER", "MEMBERS")
+				sort.SliceStable(views, func(i, j int) bool {
+					return views[i].PartitionID < views[j].PartitionID
+				})
+				for _, view := range views {
+					stdout(formatMetaPartitionView(view))
+				}
+				stdout("\n")
+			}
+
+			// print data detail
+			if optDataDetail {
+				var view *proto.DataPartitionsView
+				if view, err = client.ClientAPI().GetDataPartitions(volumeName); err != nil {
+					errout("Get volume data detail information failed:\n%v\n", err)
+					os.Exit(1)
+				}
+				stdout("\n[Data detail]\n")
+				stdout("  %10v\t%8v\t%10v\t%18v\t%18v\n",
+					"ID", "REPLICAS", "STATUS", "LEADER", "MEMBERS")
+				sort.SliceStable(view.DataPartitions, func(i, j int) bool {
+					return view.DataPartitions[i].PartitionID < view.DataPartitions[j].PartitionID
+				})
+				for _, dp := range view.DataPartitions {
+					stdout(formatDataPartitionView(dp))
+				}
+				stdout("\n")
+			}
+			return
+		},
+	}
+	cmd.Flags().BoolVarP(&optMetaDetail, "meta-detail", "m", cmdVolInfoDefaultMetaDetail, "Display metadata detail information")
+	cmd.Flags().BoolVarP(&optDataDetail, "data-detail", "d", cmdVolInfoDefaultDataDetail, "Display data detail information")
+	return cmd
+}
+
+const (
+	cmdVolDeleteUse   = "delete [VOLUME NAME]"
+	cmdVolDeleteShort = "Delete a volume from cluster"
+)
+
+func newVolDeleteCmd(client *master.MasterClient) *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   cmdVolDeleteUse,
+		Short: cmdVolDeleteShort,
+		Args:  cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			var volumeName = args[0]
+			// ask user for confirm
+			stdout("Delete volume [%v] (yes/no)[no]:", volumeName)
+			var userConfirm string
+			_, _ = fmt.Scanln(&userConfirm)
+			if userConfirm != "yes" {
+				stdout("Abort by user.\n")
+				return
+			}
+			var svv *proto.SimpleVolView
+			if svv, err = client.AdminAPI().GetVolumeSimpleInfo(volumeName); err != nil {
+				errout("Delete volume failed:\n%v\n", err)
+				os.Exit(1)
+			}
+
+			if err = client.AdminAPI().DeleteVolume(volumeName, calcAuthKey(svv.Owner)); err != nil {
+				errout("Delete volume failed:\n%v\n", err)
+				os.Exit(1)
+			}
+			stdout("Delete volume success.\n")
+		},
+	}
+	return cmd
+}
+
+func formatSimpleVolView(svv *proto.SimpleVolView) string {
+	var statusToString = func(status uint8) string {
+		switch status {
+		case 0:
+			return "normal"
+		case 1:
+			return "marked delete"
+		default:
+			return "Unknown"
+		}
+	}
+	var sb = strings.Builder{}
+	sb.WriteString(fmt.Sprintf("  ID                  : %v\n", svv.ID))
+	sb.WriteString(fmt.Sprintf("  Name                : %v\n", svv.Name))
+	sb.WriteString(fmt.Sprintf("  Owner               : %v\n", svv.Owner))
+	sb.WriteString(fmt.Sprintf("  Zone                : %v\n", svv.ZoneName))
+	sb.WriteString(fmt.Sprintf("  Status              : %v\n", statusToString(svv.Status)))
+	sb.WriteString(fmt.Sprintf("  Capacity            : %v GB\n", svv.Capacity))
+	sb.WriteString(fmt.Sprintf("  Create time         : %v\n", svv.CreateTime))
+	sb.WriteString(fmt.Sprintf("  Authenticate        : %v\n", svv.Authenticate))
+	sb.WriteString(fmt.Sprintf("  Follower read       : %v\n", svv.FollowerRead))
+	sb.WriteString(fmt.Sprintf("  Cross zone          : %v\n", svv.CrossZone))
+	sb.WriteString(fmt.Sprintf("  Meta partition count: %v\n", svv.MpCnt))
+	sb.WriteString(fmt.Sprintf("  Meta replicas       : %v\n", svv.MpReplicaNum))
+	sb.WriteString(fmt.Sprintf("  Data partition count: %v\n", svv.DpCnt))
+	sb.WriteString(fmt.Sprintf("  Data replicas       : %v\n", svv.DpReplicaNum))
+	return sb.String()
+}
+
+func formatMetaPartitionView(view *proto.MetaPartitionView) string {
+	var statusToString = func(status int8) string {
+		switch status {
+		case 1:
+			return "read only"
+		case 2:
+			return "writable"
+		case -1:
+			return "unavailable"
+		default:
+			return "unknown"
+		}
+	}
+	var rangeToString = func(num uint64) string {
+		if num >= math.MaxInt64 {
+			return "unlimited"
+		}
+		return strconv.FormatUint(num, 10)
+	}
+	return fmt.Sprintf("  %10v\t%12v\t%12v\t%12v\t%10v\t%18v\t%18v\n",
+		view.PartitionID, view.MaxInodeID, view.Start, rangeToString(view.End), statusToString(view.Status), view.LeaderAddr, view.Members)
+}
+
+func formatDataPartitionView(view *proto.DataPartitionResponse) string {
+	var statusToString = func(status int8) string {
+		switch status {
+		case 1:
+			return "read only"
+		case 2:
+			return "writable"
+		case -1:
+			return "unavailable"
+		default:
+			return "unknown"
+		}
+	}
+	return fmt.Sprintf("  %10v\t%8v\t%10v\t%18v\t%18v\n",
+		view.PartitionID, view.ReplicaNum, statusToString(view.Status), view.LeaderAddr, view.Hosts)
+}
+
+func calcAuthKey(key string) (authKey string) {
+	h := md5.New()
+	_, _ = h.Write([]byte(key))
+	cipherStr := h.Sum(nil)
+	return strings.ToLower(hex.EncodeToString(cipherStr))
+}

cli/main.go

@@ -1,19 +0,0 @@
-// Copyright 2018 The Chubao Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-// implied. See the License for the specific language governing
-// permissions and limitations under the License.
-
-package main
-
-func main()  {
-
-}

master/api_service.go

@@ -1790,7 +1790,12 @@ func (m *Server) associateVolWithUser(userID, volName string) error {
 		return err
 	}
 	if err == proto.ErrOSSUserNotExists {
-		if akPolicy, err = m.user.createKey(userID, DefaultPassword, proto.User); err != nil {
+		var param = proto.UserCreateParam{
+			ID:       userID,
+			Password: DefaultUserPassword,
+			Type:     proto.UserTypeNormal,
+		}
+		if akPolicy, err = m.user.createKey(&param); err != nil {
 			return err
 		}
 	}

master/api_service_user.go

@@ -14,39 +14,28 @@ import (
 func (m *Server) createUser(w http.ResponseWriter, r *http.Request) {
 	var (
 		akPolicy *proto.AKPolicy
-		owner    string
-		password string
 		err      error
 	)
-	if owner, password, err = parseOwnerAndPassword(r); err != nil {
+	var bytes []byte
+	if bytes, err = ioutil.ReadAll(r.Body); err != nil {
 		sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
 		return
 	}
-	if akPolicy, err = m.user.createKey(owner, password, proto.User); err != nil {
-		sendErrReply(w, r, newErrHTTPReply(err))
+	var param = proto.UserCreateParam{}
+	if err = json.Unmarshal(bytes, &param); err != nil {
+		sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
+		return
+	}
+	if param.Type == proto.UserTypeRoot {
+		sendErrReply(w, r, newErrHTTPReply(proto.ErrInvalidUserType))
 		return
 	}
-	sendOkReply(w, r, newSuccessHTTPReply(akPolicy))
-}
 
-func (m *Server) createUserWithKey(w http.ResponseWriter, r *http.Request) {
-	var (
-		akPolicy *proto.AKPolicy
-		owner    string
-		password string
-		ak       string
-		sk       string
-		err      error
-	)
-	if owner, password, ak, sk, err = parseOwnerAndKey(r); err != nil {
-		sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()})
-		return
-	}
-	if akPolicy, err = m.user.createUserWithKey(owner, password, ak, sk, proto.User); err != nil {
+	if akPolicy, err = m.user.createKey(&param); err != nil {
 		sendErrReply(w, r, newErrHTTPReply(err))
 		return
 	}
-	sendOkReply(w, r, newSuccessHTTPReply(akPolicy))
+	_ = sendOkReply(w, r, newSuccessHTTPReply(akPolicy))
 }
 
 func (m *Server) deleteUser(w http.ResponseWriter, r *http.Request) {

master/http_server.go

@@ -198,12 +198,9 @@ func (m *Server) registerAPIRoutes(router *mux.Router) {
 		HandlerFunc(m.decommissionDisk)
 
 	// user management APIs
-	router.NewRoute().Methods(http.MethodGet, http.MethodPost).
+	router.NewRoute().Methods(http.MethodPost).
 		Path(proto.UserCreate).
 		HandlerFunc(m.createUser)
-	router.NewRoute().Methods(http.MethodGet, http.MethodPost).
-		Path(proto.UserCreateWithKey).
-		HandlerFunc(m.createUserWithKey)
 	router.NewRoute().Methods(http.MethodGet, http.MethodPost).
 		Path(proto.UserDelete).
 		HandlerFunc(m.deleteUser)

master/master_manager.go

@@ -156,12 +156,24 @@ func (m *Server) refreshUser() (err error) {
 	for volName, vol := range m.cluster.allVols() {
 		if _, err = m.user.getUserInfo(vol.Owner); err == cfsProto.ErrOSSUserNotExists {
 			if len(vol.OSSAccessKey) > 0 && len(vol.OSSSecretKey) > 0 {
-				akPolicy, err = m.user.createUserWithKey(vol.Owner, DefaultPassword, vol.OSSAccessKey, vol.OSSSecretKey, cfsProto.User)
+				var param = cfsProto.UserCreateParam{
+					ID:        vol.Owner,
+					Password:  DefaultUserPassword,
+					AccessKey: vol.OSSAccessKey,
+					SecretKey: vol.OSSSecretKey,
+					Type:      cfsProto.UserTypeNormal,
+				}
+				akPolicy, err = m.user.createKey(&param)
 				if err != nil && err != cfsProto.ErrDuplicateUserID && err != cfsProto.ErrDuplicateAccessKey {
 					return err
 				}
 			} else {
-				akPolicy, err = m.user.createKey(vol.Owner, DefaultPassword, cfsProto.User)
+				var param = cfsProto.UserCreateParam{
+					ID:       vol.Owner,
+					Password: DefaultUserPassword,
+					Type:     cfsProto.UserTypeNormal,
+				}
+				akPolicy, err = m.user.createKey(&param)
 				if err != nil && err != cfsProto.ErrDuplicateUserID {
 					return err
 				}
@@ -174,11 +186,16 @@ func (m *Server) refreshUser() (err error) {
 			}
 		}
 	}
-	if !m.user.SuperAdminExist {
-		if _, err = m.user.createKey(cfsProto.Root, cfsProto.RootPassword, cfsProto.SuperAdmin); err != nil {
+	if !m.user.rootExist {
+		var param = cfsProto.UserCreateParam{
+			ID:       cfsProto.RootUserID,
+			Password: DefaultRootPasswd,
+			Type:     cfsProto.UserTypeRoot,
+		}
+		if _, err = m.user.createKey(&param); err != nil {
 			return err
 		}
-		m.user.SuperAdminExist = true
+		m.user.rootExist = true
 	}
 	return nil
 }

master/user.go

@@ -13,22 +13,23 @@ import (
 )
 
 const (
-	accessKeyLength = 16
-	secretKeyLength = 32
-	ALL             = "all"
-	DefaultPassword = "123456"
+	accessKeyLength     = 16
+	secretKeyLength     = 32
+	ALL                 = "all"
+	DefaultRootPasswd   = "ChubaoFSRoot"
+	DefaultUserPassword = "ChubaoFSUser"
 )
 
 type User struct {
-	fsm             *MetadataFsm
-	partition       raftstore.Partition
-	akStore         sync.Map //K: ak, V: AKPolicy
-	userAk          sync.Map //K: user, V: ak
-	volAKs          sync.Map //K: vol, V: aks
-	akStoreMutex    sync.RWMutex
-	userAKMutex     sync.RWMutex
-	volAKsMutex     sync.RWMutex
-	SuperAdminExist bool
+	fsm          *MetadataFsm
+	partition    raftstore.Partition
+	akStore      sync.Map //K: ak, V: AKPolicy
+	userAk       sync.Map //K: user, V: ak
+	volAKs       sync.Map //K: vol, V: aks
+	akStoreMutex sync.RWMutex
+	userAKMutex  sync.RWMutex
+	volAKsMutex  sync.RWMutex
+	rootExist    bool
 }
 
 func newUser(fsm *MetadataFsm, partition raftstore.Partition) (u *User) {
@@ -38,27 +39,35 @@ func newUser(fsm *MetadataFsm, partition raftstore.Partition) (u *User) {
 	return
 }
 
-func (u *User) createKey(userID, password string, userType proto.UserType) (akPolicy *proto.AKPolicy, err error) {
+func (u *User) createKey(param *proto.UserCreateParam) (akPolicy *proto.AKPolicy, err error) {
 	var (
 		userAK     *proto.UserAK
 		userPolicy *proto.UserPolicy
 		exist      bool
 	)
-	if !proto.IsUserType(userType) {
-		err = proto.ErrUserType
+	if param.ID == "" {
+		err = proto.ErrInvalidUserID
 		return
 	}
-	if userType == proto.SuperAdmin {
-		if u.SuperAdminExist {
-			err = proto.ErrSuperAdminExists
-			return
-		} else if userID != proto.Root {
-			err = proto.ErrDuplicateUserID
-			return
-		}
+	if !param.Type.Valid() {
+		err = proto.ErrInvalidUserType
+		return
 	}
-	accessKey := util.RandomString(accessKeyLength, util.Numeric|util.LowerLetter|util.UpperLetter)
-	secretKey := util.RandomString(secretKeyLength, util.Numeric|util.LowerLetter|util.UpperLetter)
+
+	var userID = param.ID
+	var password = param.Password
+	if password == "" {
+		password = DefaultUserPassword
+	}
+	var accessKey = param.AccessKey
+	if accessKey == "" {
+		accessKey = util.RandomString(accessKeyLength, util.Numeric|util.LowerLetter|util.UpperLetter)
+	}
+	var secretKey = param.SecretKey
+	if secretKey == "" {
+		secretKey = util.RandomString(secretKeyLength, util.Numeric|util.LowerLetter|util.UpperLetter)
+	}
+	var userType = param.Type
 	u.akStoreMutex.Lock()
 	defer u.akStoreMutex.Unlock()
 	u.userAKMutex.Lock()
@@ -89,54 +98,6 @@ func (u *User) createKey(userID, password string, userType proto.UserType) (akPo
 	return
 }
 
-func (u *User) createUserWithKey(userID, password, accessKey, secretKey string, userType proto.UserType) (akPolicy *proto.AKPolicy, err error) {
-	var (
-		userAK     *proto.UserAK
-		userPolicy *proto.UserPolicy
-		exist      bool
-	)
-	if !proto.IsUserType(userType) {
-		err = proto.ErrUserType
-		return
-	}
-	if userType == proto.SuperAdmin {
-		if u.SuperAdminExist {
-			err = proto.ErrSuperAdminExists
-			return
-		} else if userID != proto.Root {
-			err = proto.ErrDuplicateUserID
-			return
-		}
-	}
-	u.akStoreMutex.Lock()
-	defer u.akStoreMutex.Unlock()
-	u.userAKMutex.Lock()
-	defer u.userAKMutex.Unlock()
-	//check duplicate
-	if _, exist = u.userAk.Load(userID); exist {
-		err = proto.ErrDuplicateUserID
-		return
-	}
-	if _, exist = u.akStore.Load(accessKey); exist {
-		err = proto.ErrDuplicateAccessKey
-		return
-	}
-	userPolicy = proto.NewUserPolicy()
-	akPolicy = &proto.AKPolicy{AccessKey: accessKey, SecretKey: secretKey, Policy: userPolicy,
-		UserID: userID, UserType: userType, CreateTime: time.Unix(time.Now().Unix(), 0).Format(proto.TimeFormat)}
-	userAK = &proto.UserAK{UserID: userID, AccessKey: accessKey, Password: sha1String(password)}
-	if err = u.syncAddAKPolicy(akPolicy); err != nil {
-		return
-	}
-	if err = u.syncAddUserAK(userAK); err != nil {
-		return
-	}
-	u.akStore.Store(accessKey, akPolicy)
-	u.userAk.Store(userID, userAK)
-	log.LogInfof("action[createUserWithKey], userID: %v, accesskey[%v], secretkey[%v]", userID, accessKey, secretKey)
-	return
-}
-
 func (u *User) deleteKey(userID string) (err error) {
 	var (
 		userAK   *proto.UserAK
@@ -155,7 +116,7 @@ func (u *User) deleteKey(userID string) (err error) {
 		err = proto.ErrOwnVolExists
 		return
 	}
-	if akPolicy.UserType == proto.SuperAdmin {
+	if akPolicy.UserType == proto.UserTypeRoot {
 		err = proto.ErrNoPermission
 		return
 	}
@@ -253,7 +214,7 @@ func (u *User) deleteVolPolicy(volName string) (err error) {
 			if action == ALL {
 				userPolicy = &proto.UserPolicy{OwnVols: []string{volName}}
 			} else {
-				userPolicy = &proto.UserPolicy{NoneOwnVol: map[string][]string{volName: {action}}}
+				userPolicy = &proto.UserPolicy{AuthorizedVols: map[string][]string{volName: {action}}}
 			}
 			akPolicy.Policy.Delete(userPolicy)
 			if err = u.syncUpdateAKPolicy(akPolicy); err != nil {
@@ -308,7 +269,7 @@ func (u *User) addVolAKs(ak string, policy *proto.UserPolicy) (err error) {
 			return
 		}
 	}
-	for vol, actions := range policy.NoneOwnVol {
+	for vol, actions := range policy.AuthorizedVols {
 		for _, action := range actions {
 			if err = u.addAKToVol(ak, action, vol); err != nil {
 				return
@@ -353,7 +314,7 @@ func (u *User) deleteVolAKs(ak string, policy *proto.UserPolicy) (err error) {
 			return
 		}
 	}
-	for vol, actions := range policy.NoneOwnVol {
+	for vol, actions := range policy.AuthorizedVols {
 		for _, action := range actions {
 			if err = u.deleteAKFromVol(ak, action, vol); err != nil {
 				return

objectnode/policy.go

@@ -261,7 +261,7 @@ func (o *ObjectNode) policyCheck(f http.HandlerFunc) http.HandlerFunc {
 			allowed = true
 			return
 		}
-		if apis, exit := akPolicy.Policy.NoneOwnVol[param.bucket]; exit {
+		if apis, exit := akPolicy.Policy.AuthorizedVols[param.bucket]; exit {
 			if !contains(apis, param.Action().String()) {
 				allowed = false
 				log.LogWarnf("policyCheck: user policy not allowed: requestID(%v) accessKey(%v) action(%v)",

proto/admin_proto.go

@@ -70,7 +70,6 @@ const (
 
 	// APIs for user management
 	UserCreate          = "/user/create"
-	UserCreateWithKey   = "/user/createWithKey"
 	UserDelete          = "/user/delete"
 	UserAddPolicy       = "/user/addPolicy"
 	UserDeletePolicy    = "/user/deletePolicy"

proto/errors.go

@@ -74,7 +74,8 @@ var (
 	ErrZoneNotExists                   = errors.New("zone not exists")
 	ErrOwnVolExists                    = errors.New("own vol not empty")
 	ErrSuperAdminExists                = errors.New("super administrator exists ")
-	ErrUserType                        = errors.New("user type error")
+	ErrInvalidUserID                   = errors.New("invalid user ID")
+	ErrInvalidUserType                 = errors.New("invalid user type")
 	ErrNoPermission                    = errors.New("no permission to delete super admin")
 )
 
@@ -132,7 +133,7 @@ const (
 	ErrCodeNotExists
 	ErrCodeOwnVolExists
 	ErrCodeSuperAdminExists
-	ErrCodeUserType
+	ErrCodeInvalidUserType
 	ErrCodeNoPermission
 )
 
@@ -188,6 +189,6 @@ var Err2CodeMap = map[error]int32{
 	ErrZoneNotExists:                   ErrCodeNotExists,
 	ErrOwnVolExists:                    ErrCodeOwnVolExists,
 	ErrSuperAdminExists:                ErrCodeSuperAdminExists,
-	ErrUserType:                        ErrCodeUserType,
+	ErrInvalidUserType:                 ErrCodeInvalidUserType,
 	ErrNoPermission:                    ErrCodeNoPermission,
 }

proto/user_proto.go

@@ -1,17 +1,69 @@
+// Copyright 2018 The Chubao Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
 package proto
 
 import "sync"
 
-type UserType string
+type UserType uint8
 
 const (
-	SuperAdmin UserType = "super_admin"
-	Admin      UserType = "admin"
-	User       UserType = "user"
+	UserTypeInvalid UserType = 0x0
+	UserTypeRoot    UserType = 0x1
+	UserTypeAdmin   UserType = 0x2
+	UserTypeNormal  UserType = 0x3
 )
 
+func (u UserType) Valid() bool {
+	switch u {
+	case UserTypeRoot,
+		UserTypeAdmin,
+		UserTypeNormal:
+		return true
+	default:
+	}
+	return false
+}
+
+func (u UserType) String() string {
+	switch u {
+	case UserTypeRoot:
+		return "root"
+	case UserTypeAdmin:
+		return "admin"
+	case UserTypeNormal:
+		return "normal"
+	default:
+	}
+	return "invalid"
+}
+
+func UserTypeFromString(name string) UserType {
+	switch name {
+	case "root":
+		return UserTypeRoot
+	case "admin":
+		return UserTypeAdmin
+	case "normal":
+		return UserTypeNormal
+	default:
+	}
+	return UserTypeInvalid
+}
+
 const (
-	Root         = "root"
+	RootUserID   = "root"
 	RootPassword = "SuperAdminOfChubaoFS"
 )
 
@@ -31,15 +83,15 @@ type AKPolicy struct {
 }
 
 type UserPolicy struct {
-	OwnVols    []string
-	NoneOwnVol map[string][]string // mapping: volume -> actions
-	mu         sync.RWMutex
+	OwnVols        []string            `json:"own_vols"`
+	AuthorizedVols map[string][]string `json:"authorized_vols"` // mapping: volume -> actions
+	mu             sync.RWMutex
 }
 
 func NewUserPolicy() *UserPolicy {
 	return &UserPolicy{
-		OwnVols:    make([]string, 0),
-		NoneOwnVol: make(map[string][]string),
+		OwnVols:        make([]string, 0),
+		AuthorizedVols: make(map[string][]string),
 	}
 }
 
@@ -79,11 +131,11 @@ func (policy *UserPolicy) Add(addPolicy *UserPolicy) {
 	policy.mu.Lock()
 	defer policy.mu.Unlock()
 	policy.OwnVols = append(policy.OwnVols, addPolicy.OwnVols...)
-	for k, v := range addPolicy.NoneOwnVol {
-		if apis, ok := policy.NoneOwnVol[k]; ok {
-			policy.NoneOwnVol[k] = append(apis, addPolicy.NoneOwnVol[k]...)
+	for k, v := range addPolicy.AuthorizedVols {
+		if apis, ok := policy.AuthorizedVols[k]; ok {
+			policy.AuthorizedVols[k] = append(apis, addPolicy.AuthorizedVols[k]...)
 		} else {
-			policy.NoneOwnVol[k] = v
+			policy.AuthorizedVols[k] = v
 		}
 	}
 }
@@ -92,9 +144,9 @@ func (policy *UserPolicy) Delete(deletePolicy *UserPolicy) {
 	policy.mu.Lock()
 	defer policy.mu.Unlock()
 	policy.OwnVols = removeSlice(policy.OwnVols, deletePolicy.OwnVols)
-	for k, v := range deletePolicy.NoneOwnVol {
-		if apis, ok := policy.NoneOwnVol[k]; ok {
-			policy.NoneOwnVol[k] = removeSlice(apis, v)
+	for k, v := range deletePolicy.AuthorizedVols {
+		if apis, ok := policy.AuthorizedVols[k]; ok {
+			policy.AuthorizedVols[k] = removeSlice(apis, v)
 		}
 	}
 }
@@ -125,7 +177,7 @@ func CleanPolicy(policy *UserPolicy) (newUserPolicy *UserPolicy) {
 			newUserPolicy.OwnVols = append(newUserPolicy.OwnVols, vol)
 		}
 	}
-	for vol, apis := range policy.NoneOwnVol {
+	for vol, apis := range policy.AuthorizedVols {
 		checkMap := make(map[string]bool)
 		newAPI := make([]string, 0)
 		for _, api := range apis {
@@ -134,15 +186,17 @@ func CleanPolicy(policy *UserPolicy) (newUserPolicy *UserPolicy) {
 				newAPI = append(newAPI, api)
 			}
 		}
-		newUserPolicy.NoneOwnVol[vol] = newAPI
+		newUserPolicy.AuthorizedVols[vol] = newAPI
 	}
 	return
 }
 
-func IsUserType(userType UserType) bool {
-	if userType == SuperAdmin || userType == Admin || userType == User {
-		return true
-	} else {
-		return false
-	}
+type UserCreateParam struct {
+	ID        string `json:"id"`
+	Password  string `json:"pwd"`
+	AccessKey string `json:"ak"`
+	SecretKey string `json:"sk"`
+	Type      UserType
 }
+
+type UserUpdateParam = UserCreateParam

sdk/master/api_user.go

@@ -11,25 +11,13 @@ type UserAPI struct {
 	mc *MasterClient
 }
 
-func (api *UserAPI) CreateUser(userID string) (akPolicy *proto.AKPolicy, err error) {
-	var request = newAPIRequest(http.MethodPut, proto.UserCreate)
-	request.addParam("owner", userID)
-	var data []byte
-	if data, err = api.mc.serveRequest(request); err != nil {
+func (api *UserAPI) Create(param *proto.UserCreateParam) (akPolicy *proto.AKPolicy, err error) {
+	var request = newAPIRequest(http.MethodPost, proto.UserCreate)
+	var reqBody []byte
+	if reqBody, err = json.Marshal(param); err != nil {
 		return
 	}
-	akPolicy = &proto.AKPolicy{}
-	if err = json.Unmarshal(data, akPolicy); err != nil {
-		return
-	}
-	return
-}
-
-func (api *UserAPI) CreateUserWithKey(userID, ak, sk string) (akPolicy *proto.AKPolicy, err error) {
-	var request = newAPIRequest(http.MethodPut, proto.UserCreateWithKey)
-	request.addParam("owner", userID)
-	request.addParam("ak", ak)
-	request.addParam("sk", sk)
+	request.addBody(reqBody)
 	var data []byte
 	if data, err = api.mc.serveRequest(request); err != nil {
 		return