From d609fedb5c031e27f79dce9c004fdbb101070ac1 Mon Sep 17 00:00:00 2001 From: Mofei Zhang Date: Thu, 12 Dec 2019 17:56:08 +0800 Subject: [PATCH] feature: object storage interface Add ObjectNode provides S3-compatibile APIs. Fusion Storage interface expose two interface (POSIX and S3-compatible) for file operation. Signed-off-by: Mofei Zhang --- .gitignore | 1 - .gitlab-ci.yml | 50 + MAINTAINERS.md | 9 +- authnode/api_service.go | 56 +- authnode/authnode_manager.go | 3 + authnode/cluster.go | 28 +- authnode/const.go | 3 + authnode/http_server.go | 10 +- authnode/keystore_cache_op.go | 27 +- authnode/keystore_fsm.go | 12 +- authnode/keystore_fsm_op.go | 49 + authnode/server.go | 5 +- authtool/authtool.go | 12 +- build/build.sh | 81 +- client/fs/super.go | 2 +- clientv2/fs/super.go | 2 +- cmd/build.sh | 0 cmd/cfg/meta.json | 2 +- cmd/cfg/objectnode.json | 15 + cmd/cmd.go | 7 + datanode/data_partition_repair.go | 2 +- datanode/partition.go | 22 +- datanode/server.go | 58 +- datanode/wrap_operator.go | 25 +- docker/Dockerfile-cfs | 11 - docker/authnode/authnode1.json | 2 +- docker/authnode/authnode2.json | 2 +- docker/authnode/run_docker4auth.sh | 4 +- docker/build_docker_cfs.sh | 15 - docker/conf/metanode.json | 6 + docker/conf/objectnode.json | 12 + docker/docker-compose-auth.yml | 23 + docker/docker-compose.yml | 16 + docker/run_docker.sh | 15 +- docker/run_docker_auth.sh | 30 +- docker/script/start_client.sh | 2 +- docker/script/start_client_auth.sh | 5 +- docker/script/start_objectnode.sh | 7 + docs/source/index.rst | 1 + docs/source/user-guide/objectnode.rst | 57 + master/api_service.go | 91 +- master/api_service_test.go | 1 - master/cluster.go | 2 + master/cluster_test.go | 2 +- master/data_partition_check.go | 8 +- master/http_server.go | 4 +- master/master_manager_test.go | 8 +- master/meta_partition_test.go | 2 +- master/metadata_fsm_op.go | 23 +- master/server.go | 5 +- master/vol.go | 27 + master/vol_test.go | 6 +- metanode/api_handler.go | 2 +- metanode/const.go | 7 +- metanode/extend.go | 169 + metanode/extend_test.go | 57 + metanode/manager.go | 26 + metanode/manager_op.go | 258 +- metanode/manager_resp.go | 18 +- metanode/metanode.go | 77 +- metanode/multipart.go | 337 ++ metanode/multipart_test.go | 151 + metanode/partition.go | 99 +- metanode/partition_free_list.go | 30 +- metanode/partition_fsm.go | 129 +- metanode/partition_fsmop_extend.go | 46 + metanode/partition_fsmop_inode.go | 7 +- metanode/partition_fsmop_multipart.go | 47 + metanode/partition_item.go | 253 +- metanode/partition_op_extend.go | 131 + metanode/partition_op_extent.go | 30 +- metanode/partition_op_inode.go | 16 + metanode/partition_op_multipart.go | 216 ++ metanode/partition_store.go | 271 +- metanode/partition_store_ticket.go | 10 +- objectnode/acl.go | 266 ++ objectnode/acl_handler.go | 154 + objectnode/acl_test.go | 2 + objectnode/api_handler.go | 129 + objectnode/api_handler_bucket.go | 40 + objectnode/api_handler_multipart.go | 373 +++ objectnode/api_handler_object.go | 790 +++++ objectnode/api_middleware.go | 125 + objectnode/auth.go | 51 + objectnode/auth_signature_v2.go | 394 +++ objectnode/auth_signature_v4.go | 577 ++++ objectnode/auth_signature_v4_test.go | 1 + objectnode/chunk.go | 188 ++ objectnode/chunk_test.go | 65 + objectnode/const.go | 98 + objectnode/fs.go | 95 + objectnode/fs_manager.go | 89 + objectnode/fs_store.go | 13 + objectnode/fs_store_authnode.go | 33 + objectnode/fs_store_object.go | 36 + objectnode/fs_store_xattr.go | 89 + objectnode/fs_stream.go | 81 + objectnode/fs_stream_test.go | 88 + objectnode/fs_volume.go | 1395 ++++++++ objectnode/fs_volume_test.go | 1 + objectnode/policy.go | 236 ++ objectnode/policy_action.go | 94 + objectnode/policy_condition.go | 432 +++ objectnode/policy_handler.go | 114 + objectnode/policy_handler_test.go | 2 + objectnode/policy_statement.go | 114 + objectnode/policy_test.go | 55 + objectnode/result.go | 266 ++ objectnode/result_error.go | 93 + objectnode/result_test.go | 199 ++ objectnode/router.go | 261 ++ objectnode/server.go | 188 ++ objectnode/server_test.go | 54 + objectnode/signer.go | 72 + objectnode/stringset.go | 135 + objectnode/util.go | 117 + proto/admin_proto.go | 28 +- proto/auth_proto.go | 61 +- proto/errors.go | 3 + proto/fs_proto.go | 152 +- proto/packet.go | 44 +- raftstore/partition.go | 8 - raftstore/store_rocksdb.go | 15 +- sdk/data/stream/extent_client.go | 10 +- sdk/data/stream/extent_handler.go | 2 +- sdk/data/stream/stream_reader.go | 2 +- sdk/data/stream/stream_writer.go | 2 +- sdk/data/wrapper/wrapper.go | 80 +- sdk/master/api_admin.go | 181 ++ sdk/master/api_client.go | 138 + sdk/master/api_node.go | 105 + sdk/master/client.go | 238 ++ sdk/master/request.go | 44 + sdk/meta/api.go | 292 ++ sdk/meta/meta.go | 51 +- sdk/meta/operation.go | 472 +++ sdk/meta/partition.go | 1 - sdk/meta/view.go | 229 +- storage/extent.go | 5 +- storage/fallocate_darwin.go | 6 + storage/fallocate_linux.go | 7 + storage/fallocate_windows.go | 6 + storage/persistence_crc.go | 5 +- util/caps/caps.go | 161 +- util/keystore/keystore.go | 42 +- util/master_helper.go | 33 +- util/string.go | 67 + vendor/bazil.org/fuse/.gitattributes | 2 + vendor/bazil.org/fuse/.gitignore | 11 + vendor/github.com/davecgh/go-spew/LICENSE | 15 - .../github.com/davecgh/go-spew/spew/bypass.go | 145 - .../davecgh/go-spew/spew/bypasssafe.go | 38 - .../github.com/davecgh/go-spew/spew/common.go | 341 -- .../github.com/davecgh/go-spew/spew/config.go | 306 -- vendor/github.com/davecgh/go-spew/spew/doc.go | 211 -- .../github.com/davecgh/go-spew/spew/dump.go | 509 --- .../github.com/davecgh/go-spew/spew/format.go | 419 --- .../github.com/davecgh/go-spew/spew/spew.go | 148 - vendor/github.com/edsrzf/mmap-go/.gitignore | 8 + vendor/github.com/edsrzf/mmap-go/LICENSE | 25 + vendor/github.com/edsrzf/mmap-go/README.md | 12 + vendor/github.com/edsrzf/mmap-go/mmap.go | 117 + vendor/github.com/edsrzf/mmap-go/mmap_unix.go | 51 + .../github.com/edsrzf/mmap-go/mmap_windows.go | 143 + vendor/github.com/google/uuid/.travis.yml | 9 + vendor/github.com/google/uuid/CONTRIBUTING.md | 10 + vendor/github.com/google/uuid/CONTRIBUTORS | 9 + vendor/github.com/google/uuid/LICENSE | 27 + vendor/github.com/google/uuid/README.md | 19 + vendor/github.com/google/uuid/dce.go | 80 + vendor/github.com/google/uuid/doc.go | 12 + vendor/github.com/google/uuid/go.mod | 1 + vendor/github.com/google/uuid/hash.go | 53 + vendor/github.com/google/uuid/marshal.go | 37 + vendor/github.com/google/uuid/node.go | 90 + vendor/github.com/google/uuid/node_js.go | 12 + vendor/github.com/google/uuid/node_net.go | 33 + vendor/github.com/google/uuid/sql.go | 59 + vendor/github.com/google/uuid/time.go | 123 + vendor/github.com/google/uuid/util.go | 43 + vendor/github.com/google/uuid/uuid.go | 245 ++ vendor/github.com/google/uuid/version1.go | 44 + vendor/github.com/google/uuid/version4.go | 38 + vendor/github.com/gorilla/mux/AUTHORS | 8 + .../github.com/gorilla/mux/ISSUE_TEMPLATE.md | 11 + vendor/github.com/gorilla/mux/LICENSE | 27 + vendor/github.com/gorilla/mux/README.md | 649 ++++ vendor/github.com/gorilla/mux/bench_test.go | 49 + vendor/github.com/gorilla/mux/context.go | 18 + vendor/github.com/gorilla/mux/context_test.go | 30 + vendor/github.com/gorilla/mux/doc.go | 306 ++ .../example_authentication_middleware_test.go | 46 + .../gorilla/mux/example_route_test.go | 51 + vendor/github.com/gorilla/mux/go.mod | 1 + vendor/github.com/gorilla/mux/middleware.go | 72 + .../github.com/gorilla/mux/middleware_test.go | 437 +++ vendor/github.com/gorilla/mux/mux.go | 607 ++++ vendor/github.com/gorilla/mux/mux_test.go | 2846 +++++++++++++++++ vendor/github.com/gorilla/mux/old_test.go | 704 ++++ vendor/github.com/gorilla/mux/regexp.go | 328 ++ vendor/github.com/gorilla/mux/route.go | 710 ++++ vendor/github.com/gorilla/mux/test_helpers.go | 19 + .../github.com/gorilla/websocket/.gitignore | 25 + .../github.com/gorilla/websocket/.travis.yml | 19 + vendor/github.com/gorilla/websocket/AUTHORS | 9 + vendor/github.com/gorilla/websocket/LICENSE | 22 + vendor/github.com/gorilla/websocket/README.md | 64 + vendor/github.com/gorilla/websocket/client.go | 395 +++ .../gorilla/websocket/client_clone.go | 16 + .../gorilla/websocket/client_clone_legacy.go | 38 + .../gorilla/websocket/client_server_test.go | 905 ++++++ .../gorilla/websocket/client_test.go | 32 + .../gorilla/websocket/compression.go | 148 + .../gorilla/websocket/compression_test.go | 80 + vendor/github.com/gorilla/websocket/conn.go | 1166 +++++++ .../gorilla/websocket/conn_broadcast_test.go | 132 + .../github.com/gorilla/websocket/conn_test.go | 635 ++++ .../gorilla/websocket/conn_write.go | 15 + .../gorilla/websocket/conn_write_legacy.go | 18 + vendor/github.com/gorilla/websocket/doc.go | 227 ++ .../gorilla/websocket/example_test.go | 46 + .../websocket/examples/autobahn/README.md | 13 + .../examples/autobahn/fuzzingclient.json | 15 + .../websocket/examples/autobahn/server.go | 265 ++ .../gorilla/websocket/examples/chat/README.md | 102 + .../gorilla/websocket/examples/chat/client.go | 137 + .../gorilla/websocket/examples/chat/home.html | 98 + .../gorilla/websocket/examples/chat/hub.go | 53 + .../gorilla/websocket/examples/chat/main.go | 40 + .../websocket/examples/command/README.md | 19 + .../websocket/examples/command/home.html | 102 + .../websocket/examples/command/main.go | 193 ++ .../gorilla/websocket/examples/echo/README.md | 17 + .../gorilla/websocket/examples/echo/client.go | 82 + .../gorilla/websocket/examples/echo/server.go | 133 + .../websocket/examples/filewatch/README.md | 9 + .../websocket/examples/filewatch/main.go | 193 ++ vendor/github.com/gorilla/websocket/go.mod | 1 + vendor/github.com/gorilla/websocket/go.sum | 2 + vendor/github.com/gorilla/websocket/join.go | 42 + .../github.com/gorilla/websocket/join_test.go | 36 + vendor/github.com/gorilla/websocket/json.go | 60 + .../github.com/gorilla/websocket/json_test.go | 116 + vendor/github.com/gorilla/websocket/mask.go | 54 + .../github.com/gorilla/websocket/mask_safe.go | 15 + .../github.com/gorilla/websocket/mask_test.go | 72 + .../github.com/gorilla/websocket/prepared.go | 102 + .../gorilla/websocket/prepared_test.go | 74 + vendor/github.com/gorilla/websocket/proxy.go | 77 + vendor/github.com/gorilla/websocket/server.go | 363 +++ .../gorilla/websocket/server_test.go | 119 + vendor/github.com/gorilla/websocket/trace.go | 19 + .../github.com/gorilla/websocket/trace_17.go | 12 + vendor/github.com/gorilla/websocket/util.go | 283 ++ .../github.com/gorilla/websocket/util_test.go | 96 + .../gorilla/websocket/x_net_proxy.go | 473 +++ .../prometheus/procfs/.golangci.yml | 6 + vendor/github.com/prometheus/procfs/go.sum | 2 + .../prometheus/procfs/internal/fs/fs.go | 52 + vendor/github.com/tiglabs/raft/.gitignore | 24 + vendor/modules.txt | 56 + .../build_tools/build_detect_platform | 74 +- .../snappy-1.1.7/.appveyor.yml | 0 .../third-party => }/snappy-1.1.7/.travis.yml | 0 .../third-party => }/snappy-1.1.7/AUTHORS | 0 .../snappy-1.1.7/CMakeLists.txt | 0 .../snappy-1.1.7/CONTRIBUTING.md | 0 .../third-party => }/snappy-1.1.7/COPYING | 0 .../third-party => }/snappy-1.1.7/NEWS | 0 .../third-party => }/snappy-1.1.7/README.md | 0 vendor/snappy-1.1.7/cmake/SnappyConfig.cmake | 1 + .../snappy-1.1.7/cmake/config.h.in | 0 .../snappy-1.1.7/format_description.txt | 0 .../snappy-1.1.7/framing_format.txt | 0 .../third-party => }/snappy-1.1.7/snappy-c.cc | 0 .../third-party => }/snappy-1.1.7/snappy-c.h | 0 .../snappy-1.1.7/snappy-internal.h | 0 .../snappy-1.1.7/snappy-sinksource.cc | 0 .../snappy-1.1.7/snappy-sinksource.h | 0 .../snappy-1.1.7/snappy-stubs-internal.cc | 0 .../snappy-1.1.7/snappy-stubs-internal.h | 0 .../snappy-1.1.7/snappy-stubs-public.h.in | 0 .../snappy-1.1.7/snappy-test.cc | 0 .../snappy-1.1.7/snappy-test.h | 0 .../third-party => }/snappy-1.1.7/snappy.cc | 0 .../third-party => }/snappy-1.1.7/snappy.h | 0 .../snappy-1.1.7/snappy_unittest.cc | 0 .../snappy-1.1.7/testdata/alice29.txt | 0 .../snappy-1.1.7/testdata/asyoulik.txt | 0 .../snappy-1.1.7/testdata/baddata1.snappy | Bin .../snappy-1.1.7/testdata/baddata2.snappy | Bin .../snappy-1.1.7/testdata/baddata3.snappy | Bin .../snappy-1.1.7/testdata/fireworks.jpeg | Bin .../snappy-1.1.7/testdata/geo.protodata | Bin .../snappy-1.1.7/testdata/html | 0 .../snappy-1.1.7/testdata/html_x_4 | 0 .../snappy-1.1.7/testdata/kppkn.gtb | Bin .../snappy-1.1.7/testdata/lcet10.txt | 0 .../snappy-1.1.7/testdata/paper-100k.pdf | 8 +- .../snappy-1.1.7/testdata/plrabn12.txt | 0 .../snappy-1.1.7/testdata/urls.10K | 0 301 files changed, 28412 insertions(+), 3004 deletions(-) create mode 100644 .gitlab-ci.yml mode change 100644 => 100755 cmd/build.sh create mode 100644 cmd/cfg/objectnode.json delete mode 100644 docker/Dockerfile-cfs delete mode 100755 docker/build_docker_cfs.sh create mode 100644 docker/conf/objectnode.json create mode 100755 docker/script/start_objectnode.sh create mode 100644 docs/source/user-guide/objectnode.rst create mode 100644 metanode/extend.go create mode 100644 metanode/extend_test.go create mode 100644 metanode/multipart.go create mode 100644 metanode/multipart_test.go create mode 100644 metanode/partition_fsmop_extend.go create mode 100644 metanode/partition_fsmop_multipart.go create mode 100644 metanode/partition_op_extend.go create mode 100644 metanode/partition_op_multipart.go create mode 100644 objectnode/acl.go create mode 100644 objectnode/acl_handler.go create mode 100644 objectnode/acl_test.go create mode 100644 objectnode/api_handler.go create mode 100644 objectnode/api_handler_bucket.go create mode 100644 objectnode/api_handler_multipart.go create mode 100644 objectnode/api_handler_object.go create mode 100644 objectnode/api_middleware.go create mode 100644 objectnode/auth.go create mode 100644 objectnode/auth_signature_v2.go create mode 100644 objectnode/auth_signature_v4.go create mode 100644 objectnode/auth_signature_v4_test.go create mode 100644 objectnode/chunk.go create mode 100644 objectnode/chunk_test.go create mode 100644 objectnode/const.go create mode 100644 objectnode/fs.go create mode 100644 objectnode/fs_manager.go create mode 100644 objectnode/fs_store.go create mode 100644 objectnode/fs_store_authnode.go create mode 100644 objectnode/fs_store_object.go create mode 100644 objectnode/fs_store_xattr.go create mode 100644 objectnode/fs_stream.go create mode 100644 objectnode/fs_stream_test.go create mode 100644 objectnode/fs_volume.go create mode 100644 objectnode/fs_volume_test.go create mode 100644 objectnode/policy.go create mode 100644 objectnode/policy_action.go create mode 100644 objectnode/policy_condition.go create mode 100644 objectnode/policy_handler.go create mode 100644 objectnode/policy_handler_test.go create mode 100644 objectnode/policy_statement.go create mode 100644 objectnode/policy_test.go create mode 100644 objectnode/result.go create mode 100644 objectnode/result_error.go create mode 100644 objectnode/result_test.go create mode 100644 objectnode/router.go create mode 100644 objectnode/server.go create mode 100644 objectnode/server_test.go create mode 100644 objectnode/signer.go create mode 100644 objectnode/stringset.go create mode 100644 objectnode/util.go create mode 100644 sdk/master/api_admin.go create mode 100644 sdk/master/api_client.go create mode 100644 sdk/master/api_node.go create mode 100644 sdk/master/client.go create mode 100644 sdk/master/request.go create mode 100644 storage/fallocate_darwin.go create mode 100644 storage/fallocate_linux.go create mode 100644 storage/fallocate_windows.go create mode 100644 util/string.go create mode 100644 vendor/bazil.org/fuse/.gitattributes create mode 100644 vendor/bazil.org/fuse/.gitignore delete mode 100644 vendor/github.com/davecgh/go-spew/LICENSE delete mode 100644 vendor/github.com/davecgh/go-spew/spew/bypass.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/bypasssafe.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/common.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/config.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/doc.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/dump.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/format.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/spew.go create mode 100644 vendor/github.com/edsrzf/mmap-go/.gitignore create mode 100644 vendor/github.com/edsrzf/mmap-go/LICENSE create mode 100644 vendor/github.com/edsrzf/mmap-go/README.md create mode 100644 vendor/github.com/edsrzf/mmap-go/mmap.go create mode 100644 vendor/github.com/edsrzf/mmap-go/mmap_unix.go create mode 100644 vendor/github.com/edsrzf/mmap-go/mmap_windows.go create mode 100644 vendor/github.com/google/uuid/.travis.yml create mode 100644 vendor/github.com/google/uuid/CONTRIBUTING.md create mode 100644 vendor/github.com/google/uuid/CONTRIBUTORS create mode 100644 vendor/github.com/google/uuid/LICENSE create mode 100644 vendor/github.com/google/uuid/README.md create mode 100644 vendor/github.com/google/uuid/dce.go create mode 100644 vendor/github.com/google/uuid/doc.go create mode 100644 vendor/github.com/google/uuid/go.mod create mode 100644 vendor/github.com/google/uuid/hash.go create mode 100644 vendor/github.com/google/uuid/marshal.go create mode 100644 vendor/github.com/google/uuid/node.go create mode 100644 vendor/github.com/google/uuid/node_js.go create mode 100644 vendor/github.com/google/uuid/node_net.go create mode 100644 vendor/github.com/google/uuid/sql.go create mode 100644 vendor/github.com/google/uuid/time.go create mode 100644 vendor/github.com/google/uuid/util.go create mode 100644 vendor/github.com/google/uuid/uuid.go create mode 100644 vendor/github.com/google/uuid/version1.go create mode 100644 vendor/github.com/google/uuid/version4.go create mode 100644 vendor/github.com/gorilla/mux/AUTHORS create mode 100644 vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md create mode 100644 vendor/github.com/gorilla/mux/LICENSE create mode 100644 vendor/github.com/gorilla/mux/README.md create mode 100644 vendor/github.com/gorilla/mux/bench_test.go create mode 100644 vendor/github.com/gorilla/mux/context.go create mode 100644 vendor/github.com/gorilla/mux/context_test.go create mode 100644 vendor/github.com/gorilla/mux/doc.go create mode 100644 vendor/github.com/gorilla/mux/example_authentication_middleware_test.go create mode 100644 vendor/github.com/gorilla/mux/example_route_test.go create mode 100644 vendor/github.com/gorilla/mux/go.mod create mode 100644 vendor/github.com/gorilla/mux/middleware.go create mode 100644 vendor/github.com/gorilla/mux/middleware_test.go create mode 100644 vendor/github.com/gorilla/mux/mux.go create mode 100644 vendor/github.com/gorilla/mux/mux_test.go create mode 100644 vendor/github.com/gorilla/mux/old_test.go create mode 100644 vendor/github.com/gorilla/mux/regexp.go create mode 100644 vendor/github.com/gorilla/mux/route.go create mode 100644 vendor/github.com/gorilla/mux/test_helpers.go create mode 100644 vendor/github.com/gorilla/websocket/.gitignore create mode 100644 vendor/github.com/gorilla/websocket/.travis.yml create mode 100644 vendor/github.com/gorilla/websocket/AUTHORS create mode 100644 vendor/github.com/gorilla/websocket/LICENSE create mode 100644 vendor/github.com/gorilla/websocket/README.md create mode 100644 vendor/github.com/gorilla/websocket/client.go create mode 100644 vendor/github.com/gorilla/websocket/client_clone.go create mode 100644 vendor/github.com/gorilla/websocket/client_clone_legacy.go create mode 100644 vendor/github.com/gorilla/websocket/client_server_test.go create mode 100644 vendor/github.com/gorilla/websocket/client_test.go create mode 100644 vendor/github.com/gorilla/websocket/compression.go create mode 100644 vendor/github.com/gorilla/websocket/compression_test.go create mode 100644 vendor/github.com/gorilla/websocket/conn.go create mode 100644 vendor/github.com/gorilla/websocket/conn_broadcast_test.go create mode 100644 vendor/github.com/gorilla/websocket/conn_test.go create mode 100644 vendor/github.com/gorilla/websocket/conn_write.go create mode 100644 vendor/github.com/gorilla/websocket/conn_write_legacy.go create mode 100644 vendor/github.com/gorilla/websocket/doc.go create mode 100644 vendor/github.com/gorilla/websocket/example_test.go create mode 100644 vendor/github.com/gorilla/websocket/examples/autobahn/README.md create mode 100644 vendor/github.com/gorilla/websocket/examples/autobahn/fuzzingclient.json create mode 100644 vendor/github.com/gorilla/websocket/examples/autobahn/server.go create mode 100644 vendor/github.com/gorilla/websocket/examples/chat/README.md create mode 100644 vendor/github.com/gorilla/websocket/examples/chat/client.go create mode 100644 vendor/github.com/gorilla/websocket/examples/chat/home.html create mode 100644 vendor/github.com/gorilla/websocket/examples/chat/hub.go create mode 100644 vendor/github.com/gorilla/websocket/examples/chat/main.go create mode 100644 vendor/github.com/gorilla/websocket/examples/command/README.md create mode 100644 vendor/github.com/gorilla/websocket/examples/command/home.html create mode 100644 vendor/github.com/gorilla/websocket/examples/command/main.go create mode 100644 vendor/github.com/gorilla/websocket/examples/echo/README.md create mode 100644 vendor/github.com/gorilla/websocket/examples/echo/client.go create mode 100644 vendor/github.com/gorilla/websocket/examples/echo/server.go create mode 100644 vendor/github.com/gorilla/websocket/examples/filewatch/README.md create mode 100644 vendor/github.com/gorilla/websocket/examples/filewatch/main.go create mode 100644 vendor/github.com/gorilla/websocket/go.mod create mode 100644 vendor/github.com/gorilla/websocket/go.sum create mode 100644 vendor/github.com/gorilla/websocket/join.go create mode 100644 vendor/github.com/gorilla/websocket/join_test.go create mode 100644 vendor/github.com/gorilla/websocket/json.go create mode 100644 vendor/github.com/gorilla/websocket/json_test.go create mode 100644 vendor/github.com/gorilla/websocket/mask.go create mode 100644 vendor/github.com/gorilla/websocket/mask_safe.go create mode 100644 vendor/github.com/gorilla/websocket/mask_test.go create mode 100644 vendor/github.com/gorilla/websocket/prepared.go create mode 100644 vendor/github.com/gorilla/websocket/prepared_test.go create mode 100644 vendor/github.com/gorilla/websocket/proxy.go create mode 100644 vendor/github.com/gorilla/websocket/server.go create mode 100644 vendor/github.com/gorilla/websocket/server_test.go create mode 100644 vendor/github.com/gorilla/websocket/trace.go create mode 100644 vendor/github.com/gorilla/websocket/trace_17.go create mode 100644 vendor/github.com/gorilla/websocket/util.go create mode 100644 vendor/github.com/gorilla/websocket/util_test.go create mode 100644 vendor/github.com/gorilla/websocket/x_net_proxy.go create mode 100644 vendor/github.com/prometheus/procfs/.golangci.yml create mode 100644 vendor/github.com/prometheus/procfs/go.sum create mode 100644 vendor/github.com/prometheus/procfs/internal/fs/fs.go create mode 100644 vendor/github.com/tiglabs/raft/.gitignore create mode 100644 vendor/modules.txt rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/.appveyor.yml (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/.travis.yml (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/AUTHORS (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/CMakeLists.txt (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/CONTRIBUTING.md (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/COPYING (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/NEWS (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/README.md (100%) create mode 100644 vendor/snappy-1.1.7/cmake/SnappyConfig.cmake rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/cmake/config.h.in (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/format_description.txt (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/framing_format.txt (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-c.cc (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-c.h (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-internal.h (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-sinksource.cc (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-sinksource.h (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-stubs-internal.cc (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-stubs-internal.h (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-stubs-public.h.in (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-test.cc (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy-test.h (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy.cc (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy.h (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/snappy_unittest.cc (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/alice29.txt (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/asyoulik.txt (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/baddata1.snappy (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/baddata2.snappy (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/baddata3.snappy (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/fireworks.jpeg (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/geo.protodata (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/html (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/html_x_4 (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/kppkn.gtb (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/lcet10.txt (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/paper-100k.pdf (98%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/plrabn12.txt (100%) rename vendor/{rocksdb-5.9.2/third-party => }/snappy-1.1.7/testdata/urls.10K (100%) diff --git a/.gitignore b/.gitignore index 0f4639eca..84a0704ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.gitignore build/bin build/rocksdb build/snappy diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..f260ac204 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,50 @@ +before_script: + - go version + - echo $CI_PROJECT_DIR + - export GOPATH=/export/workspace/go + - echo $GOPATH + - echo ${CI_COMMIT_REF_NAME} + - rm -fr $GOPATH/src/github.com/chubaofs/chubaofs + - ln -s $CI_PROJECT_DIR $GOPATH/src/github.com/chubaofs/ + - echo $CI_PROJECT_DIR + - echo $GOPATH/src/github.com/chubaofs/ + - cd $GOPATH/src/github.com/chubaofs/chubaofs + +stages: + - build + - deploy + - restart + - ltptest + +build_main: + stage: build + tags: + - cfs + script: + - sh /export/App/cfsci/scripts/build.sh + +deploy: + stage: deploy + tags: + - cfs + script: + - sh /export/App/cfsci/scripts/deploy.sh + +restart: + stage: restart + tags: + - cfs + script: + - sh /export/App/cfsci/scripts/stop-ltp.sh + - sh /export/App/cfsci/scripts/stop-client.sh + - sh /export/App/cfsci/scripts/stop-server.sh + - sh /export/App/cfsci/scripts/start-server.sh + - sh /export/App/cfsci/scripts/start-client.sh + +ltptest: + stage: ltptest + tags: + - cfs + script: + - sh /export/App/cfsci/scripts/start-ltp.sh + diff --git a/MAINTAINERS.md b/MAINTAINERS.md index b974308c8..3f98bde82 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -5,13 +5,12 @@ The ChubaoFS maintainers are: * Shuoran Liu @sjzlsr: Client and SDK * Hongyin Zhu @zhuhyc: Master * Jianxing Zhao @znlstar: MetaNode -* Mofei Zhang @mervinkid: MetaNode and S3 Gateway +* Mofei Zhang @mervinkid: MetaNode and ObjectNode * Tianpeng Li @Skypigltp: DataNode and Raft -* Yubo Li @yuboLee: S3 Gateway and Console +* Yubo Li @yuboLee: ObjectNode and Console * Wei Ding @wding109: Research and Open Source Strategy -* Zhengyi Zhu @zhuzhengyi: Monitoring and S3 Gateway +* Zhengyi Zhu @zhuzhengyi: Monitoring and ObjectNode * Liying Zhang @Vivian7755: Product Management & Advocate * Junyuan Zeng @jzeng4: Authorization Node * Xihao Xu @xxscott: CSI Driver -* Wenjia Wu @wenjia322: Authorization Node -* Chengyu Liu @@chengyu-l: Helm and CSI Driver \ No newline at end of file +* Wenjia Wu @wenjia322: Authorization Node \ No newline at end of file diff --git a/authnode/api_service.go b/authnode/api_service.go index 614f3a1e5..fb13f9316 100644 --- a/authnode/api_service.go +++ b/authnode/api_service.go @@ -178,7 +178,7 @@ func genAuthRaftNodeOpResp(req *proto.APIAccessReq, ts int64, key []byte, msg st } if message, err = cryptoutil.EncodeMessage(jresp, key); err != nil { - err = fmt.Errorf("encdoe message for response failed %s", err.Error()) + err = fmt.Errorf("encode message for response failed %s", err.Error()) return } @@ -302,17 +302,27 @@ func (m *Server) handleGetKey(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, } func (m *Server) handleAddCaps(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) { - if res, err = m.cluster.AddCaps(keyInfo.ID, keyInfo); err != nil { - return + if keyInfo.ID == "" { + var akInfo *keystore.AccessKeyInfo + if akInfo, err = m.cluster.GetAKInfo(keyInfo.AccessKey); err != nil { + return + } + return m.cluster.AddCaps(akInfo.ID, keyInfo) + } else { + return m.cluster.AddCaps(keyInfo.ID, keyInfo) } - return } func (m *Server) handleDeleteCaps(keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) { - if res, err = m.cluster.DeleteCaps(keyInfo.ID, keyInfo); err != nil { - return + if keyInfo.ID == "" { + var akInfo *keystore.AccessKeyInfo + if akInfo, err = m.cluster.GetAKInfo(keyInfo.AccessKey); err != nil { + return + } + return m.cluster.DeleteCaps(akInfo.ID, keyInfo) + } else { + return m.cluster.DeleteCaps(keyInfo.ID, keyInfo) } - return } func (m *Server) extractClientReqInfo(r *http.Request) (plaintext []byte, err error) { @@ -335,6 +345,28 @@ func (m *Server) extractClientReqInfo(r *http.Request) (plaintext []byte, err er return } +func (m *Server) osCapsOp(writer http.ResponseWriter, request *http.Request) { + //TODO + /* + case proto.MsgAuthOSAddCapsReq: + fallthrough + case proto.MsgAuthOSDeleteCapsReq: + if err = keyInfo.IsValidAK(); err != nil { + sendErrReply(w, r, &proto.HTTPAuthReply{Code: proto.ErrCodeParamError, Msg: err.Error()}) + return + } + if err = keyInfo.IsValidCaps(); err != nil { + sendErrReply(w, r, &proto.HTTPAuthReply{Code: proto.ErrCodeParamError, Msg: err.Error()}) + return + } + case proto.MsgAuthOSGetCapsReq: + if err = keyInfo.IsValidAK(); err != nil { + sendErrReply(w, r, &proto.HTTPAuthReply{Code: proto.ErrCodeParamError, Msg: err.Error()}) + return + } + */ +} + func (m *Server) genTicket(key []byte, serviceID string, IP string, caps []byte) (ticket cryptoutil.Ticket) { currentTime := time.Now().Unix() ticket.Version = cryptoutil.TicketVersion @@ -354,14 +386,14 @@ func (m *Server) getSecretKey(id string) (key []byte, err error) { if keyInfo, err = m.getSecretKeyInfo(id); err != nil { return } - return keyInfo.Key, err + return keyInfo.AuthKey, err } func (m *Server) getSecretKeyInfo(id string) (keyInfo *keystore.KeyInfo, err error) { if id == proto.AuthServiceID { keyInfo = &keystore.KeyInfo{ - Key: m.cluster.AuthSecretKey, - Caps: []byte(`{"API": ["*:*:*"]}`), + AuthKey: m.cluster.AuthSecretKey, + Caps: []byte(`{"API": ["*:*:*"]}`), } } else { if keyInfo, err = m.cluster.GetKey(id); err != nil { @@ -417,7 +449,7 @@ func (m *Server) genGetTicketAuthResp(req *proto.AuthGetTicketReq, ts int64, r * if keyInfo, err = m.getSecretKeyInfo(resp.ClientID); err != nil { return } - clientKey = keyInfo.Key + clientKey = keyInfo.AuthKey if message, err = cryptoutil.EncodeMessage(jresp, clientKey); err != nil { return } @@ -459,7 +491,7 @@ func genAuthAPIAccessResp(req *proto.APIAccessReq, keyInfo *keystore.KeyInfo, ts } if message, err = cryptoutil.EncodeMessage(jresp, key); err != nil { - err = fmt.Errorf("encdoe message for response failed %s", err.Error()) + err = fmt.Errorf("encode message for response failed %s", err.Error()) return } diff --git a/authnode/authnode_manager.go b/authnode/authnode_manager.go index 0b1a0a9a6..955dbb0a0 100644 --- a/authnode/authnode_manager.go +++ b/authnode/authnode_manager.go @@ -40,6 +40,9 @@ func (m *Server) handleLeaderChange(leader uint64) { if err := m.cluster.loadKeystore(); err != nil { panic(err) } + if err := m.cluster.loadAKstore(); err != nil { + panic(err) + } m.metaReady = true } } diff --git a/authnode/cluster.go b/authnode/cluster.go index 83cb1f9c2..56262df23 100644 --- a/authnode/cluster.go +++ b/authnode/cluster.go @@ -17,6 +17,7 @@ package authnode import ( "encoding/json" "fmt" + "github.com/chubaofs/chubaofs/util" "time" "github.com/chubaofs/chubaofs/proto" @@ -57,6 +58,7 @@ func newCluster(name string, leaderInfo *LeaderInfo, fsm *KeystoreFsm, partition c.fsm = fsm c.partition = partition c.fsm.keystore = make(map[string]*keystore.KeyInfo, 0) + c.fsm.accessKeystore = make(map[string]*keystore.AccessKeyInfo, 0) return } @@ -88,17 +90,28 @@ func (c *Cluster) checkLeaderAddr() { func (c *Cluster) CreateNewKey(id string, keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) { c.fsm.opKeyMutex.Lock() defer c.fsm.opKeyMutex.Unlock() + accessKeyInfo := &keystore.AccessKeyInfo{ + AccessKey: keyInfo.AccessKey, + ID: keyInfo.ID, + } if _, err = c.fsm.GetKey(id); err == nil { err = proto.ErrDuplicateKey goto errHandler } keyInfo.Ts = time.Now().Unix() - keyInfo.Key = cryptoutil.GenSecretKey([]byte(c.AuthRootKey), keyInfo.Ts, id) + keyInfo.AuthKey = cryptoutil.GenSecretKey([]byte(c.AuthRootKey), keyInfo.Ts, id) + //TODO check duplicate + keyInfo.AccessKey = util.RandomString(16, util.Numeric|util.LowerLetter|util.UpperLetter) + keyInfo.SecretKey = util.RandomString(32, util.Numeric|util.LowerLetter|util.UpperLetter) if err = c.syncAddKey(keyInfo); err != nil { goto errHandler } + if err = c.syncAddAccessKey(accessKeyInfo); err != nil { + goto errHandler + } res = keyInfo c.fsm.PutKey(keyInfo) + c.fsm.PutAKInfo(accessKeyInfo) return errHandler: err = fmt.Errorf("action[CreateNewKey], clusterID[%v] ID:%v, err:%v ", c.Name, keyInfo, err.Error()) @@ -138,6 +151,19 @@ errHandler: return } +// GetKey get a key from the AKstore +func (c *Cluster) GetAKInfo(accessKey string) (akInfo *keystore.AccessKeyInfo, err error) { + if akInfo, err = c.fsm.GetAKInfo(accessKey); err != nil { + err = proto.ErrAccessKeyNotExists + goto errHandler + } + return +errHandler: + err = fmt.Errorf("action[GetAKInfo], clusterID[%v] ID:%v, err:%v ", c.Name, accessKey, err.Error()) + log.LogError(errors.Stack(err)) + return +} + // AddCaps add caps to the key func (c *Cluster) AddCaps(id string, keyInfo *keystore.KeyInfo) (res *keystore.KeyInfo, err error) { var ( diff --git a/authnode/const.go b/authnode/const.go index 7e3070663..1085bdc10 100644 --- a/authnode/const.go +++ b/authnode/const.go @@ -33,4 +33,7 @@ const ( idSeparator = "$" // To seperate ID of server that submits raft changes keyAcronym = "key" ksPrefix = keySeparator + keyAcronym + keySeparator + + akAcronym = "ak" + akPrefix = keySeparator + akAcronym + keySeparator ) diff --git a/authnode/http_server.go b/authnode/http_server.go index 4c636d1f3..1a33d1e18 100644 --- a/authnode/http_server.go +++ b/authnode/http_server.go @@ -92,6 +92,12 @@ func (m *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { fallthrough case proto.AdminRemoveRaftNode: m.raftNodeOp(w, r) + case proto.OSAddCaps: + fallthrough + case proto.OSDeleteCaps: + fallthrough + case proto.OSGetCaps: + m.osCapsOp(w, r) default: sendErrReply(w, r, &proto.HTTPAuthReply{Code: proto.ErrCodeParamError, Msg: "Invalid requst URL"}) } @@ -107,7 +113,9 @@ func (m *Server) handleFunctions() { http.Handle(proto.AdminGetCaps, m.handlerWithInterceptor()) http.Handle(proto.AdminAddRaftNode, m.handlerWithInterceptor()) http.Handle(proto.AdminRemoveRaftNode, m.handlerWithInterceptor()) - + http.Handle(proto.OSAddCaps, m.handlerWithInterceptor()) + http.Handle(proto.OSDeleteCaps, m.handlerWithInterceptor()) + http.Handle(proto.OSGetCaps, m.handlerWithInterceptor()) return } diff --git a/authnode/keystore_cache_op.go b/authnode/keystore_cache_op.go index 0694fd909..ab715a08e 100644 --- a/authnode/keystore_cache_op.go +++ b/authnode/keystore_cache_op.go @@ -29,6 +29,31 @@ func (mf *KeystoreFsm) GetKey(id string) (u *keystore.KeyInfo, err error) { func (mf *KeystoreFsm) DeleteKey(id string) { mf.ksMutex.Lock() defer mf.ksMutex.Unlock() - delete((mf.keystore), id) + delete(mf.keystore, id) + return +} + +func (mf *KeystoreFsm) PutAKInfo(akInfo *keystore.AccessKeyInfo) { + mf.aksMutex.Lock() + defer mf.aksMutex.Unlock() + if _, ok := (mf.accessKeystore)[akInfo.AccessKey]; !ok { + (mf.accessKeystore)[akInfo.AccessKey] = akInfo + } +} + +func (mf *KeystoreFsm) GetAKInfo(accessKey string) (akInfo *keystore.AccessKeyInfo, err error) { + mf.aksMutex.RLock() + defer mf.aksMutex.RUnlock() + akInfo, ok := (mf.accessKeystore)[accessKey] + if !ok { + err = proto.ErrAccessKeyNotExists + } + return +} + +func (mf *KeystoreFsm) DeleteAKInfo(accessKey string) { + mf.aksMutex.Lock() + defer mf.aksMutex.Unlock() + //TODO return } diff --git a/authnode/keystore_fsm.go b/authnode/keystore_fsm.go index 1610ca3a2..3c7abbfc3 100644 --- a/authnode/keystore_fsm.go +++ b/authnode/keystore_fsm.go @@ -51,10 +51,12 @@ type KeystoreFsm struct { peerChangeHandler raftPeerChangeHandler snapshotHandler raftApplySnapshotHandler - keystore map[string]*keystore.KeyInfo - ksMutex sync.RWMutex // keystore mutex - opKeyMutex sync.RWMutex // operations on key mutex - id uint64 // current id of server + keystore map[string]*keystore.KeyInfo + accessKeystore map[string]*keystore.AccessKeyInfo + ksMutex sync.RWMutex // keystore mutex + aksMutex sync.RWMutex //accesskeystore mutex + opKeyMutex sync.RWMutex // operations on key mutex + id uint64 // current id of server } func newKeystoreFsm(store *raftstore.RocksDBStore, retainsLog uint64, rs *raft.RaftServer) (fsm *KeystoreFsm) { @@ -144,7 +146,7 @@ func (mf *KeystoreFsm) Apply(command []byte, index uint64) (resp interface{}, er // of cache may happen in newly demoted leader node. Therefore, we use the following // statement: "id" indicates which server has changed keystore cache (typical leader). if mf.id != leader { - mf.DeleteKey(string(keyInfo.Key)) + mf.DeleteKey(keyInfo.ID) log.LogInfof("action[Apply], Successfully delete key in node[%d]", mf.id) } else { log.LogInfof("action[Apply], Already delete key in node[%d]", mf.id) diff --git a/authnode/keystore_fsm_op.go b/authnode/keystore_fsm_op.go index 307b229eb..dc40d354c 100644 --- a/authnode/keystore_fsm_op.go +++ b/authnode/keystore_fsm_op.go @@ -52,6 +52,8 @@ func (m *RaftCmd) setOpType() { switch keyArr[1] { case keyAcronym: m.Op = opSyncAddKey + case akAcronym: + m.Op = opSyncAddKey default: log.LogWarnf("action[setOpType] unknown opCode[%v]", keyArr[1]) } @@ -73,6 +75,10 @@ func (c *Cluster) syncAddKey(keyInfo *keystore.KeyInfo) (err error) { return c.syncPutKeyInfo(opSyncAddKey, keyInfo) } +func (c *Cluster) syncAddAccessKey(akInfo *keystore.AccessKeyInfo) (err error) { + return c.syncPutAccessKeyInfo(opSyncAddKey, akInfo) +} + func (c *Cluster) syncAddCaps(keyInfo *keystore.KeyInfo) (err error) { return c.syncPutKeyInfo(opSyncAddCaps, keyInfo) } @@ -96,6 +102,17 @@ func (c *Cluster) syncPutKeyInfo(opType uint32, keyInfo *keystore.KeyInfo) (err return c.submit(keydata) } +func (c *Cluster) syncPutAccessKeyInfo(opType uint32, accessKeyInfo *keystore.AccessKeyInfo) (err error) { + keydata := new(RaftCmd) + keydata.Op = opType + keydata.K = akPrefix + accessKeyInfo.AccessKey + idSeparator + strconv.FormatUint(c.fsm.id, 10) + vv := *accessKeyInfo + if keydata.V, err = json.Marshal(vv); err != nil { + return errors.New(err.Error()) + } + return c.submit(keydata) +} + func (c *Cluster) loadKeystore() (err error) { ks := make(map[string]*keystore.KeyInfo, 0) log.LogInfof("action[loadKeystore]") @@ -128,6 +145,38 @@ func (c *Cluster) clearKeystore() { c.fsm.keystore = nil } +func (c *Cluster) loadAKstore() (err error) { + aks := make(map[string]*keystore.AccessKeyInfo, 0) + log.LogInfof("action[loadAccessKeystore]") + result, err := c.fsm.store.SeekForPrefix([]byte(akPrefix)) + if err != nil { + err = fmt.Errorf("action[loadAccessKeystore], err: %v", err.Error()) + return err + } + for _, value := range result { + ak := &keystore.AccessKeyInfo{} + if err = json.Unmarshal(value, ak); err != nil { + err = fmt.Errorf("action[loadAccessKeystore], value: %v, unmarshal err: %v", string(value), err) + return err + } + if _, ok := aks[ak.AccessKey]; !ok { + aks[ak.AccessKey] = ak + } + log.LogInfof("action[loadAccessKeystore], access key[%v]", ak) + } + c.fsm.aksMutex.Lock() + defer c.fsm.aksMutex.Unlock() + c.fsm.accessKeystore = aks + + return +} + +func (c *Cluster) clearAKstore() { + c.fsm.aksMutex.Lock() + defer c.fsm.aksMutex.Unlock() + c.fsm.accessKeystore = nil +} + func (c *Cluster) addRaftNode(nodeID uint64, addr string) (err error) { peer := proto.Peer{ID: nodeID} _, err = c.partition.ChangeMember(proto.ConfAddNode, peer, []byte(addr)) diff --git a/authnode/server.go b/authnode/server.go index ab9ee9cf1..c976837ff 100644 --- a/authnode/server.go +++ b/authnode/server.go @@ -189,7 +189,10 @@ func (m *Server) Start(cfg *config.Config) (err error) { log.LogError(errors.Stack(err)) return } - m.rocksDBStore = raftstore.NewRocksDBStore(m.storeDir, LRUCacheSize, WriteBufferSize) + if m.rocksDBStore, err = raftstore.NewRocksDBStore(m.storeDir, LRUCacheSize, WriteBufferSize); err != nil { + log.LogErrorf("Start: init RocksDB fail: err(%v)", err) + return + } if err = m.createRaftServer(); err != nil { log.LogError(errors.Stack(err)) return diff --git a/authtool/authtool.go b/authtool/authtool.go index 5c3639b82..3b366f99c 100644 --- a/authtool/authtool.go +++ b/authtool/authtool.go @@ -181,7 +181,7 @@ func getTicket() { if err1 != nil { panic(err1) } - key, err2 := cryptoutil.Base64Decode(cfg.GetString("key")) + key, err2 := cryptoutil.Base64Decode(cfg.GetString("auth_key")) if err2 != nil { panic(err2) } @@ -464,11 +464,11 @@ func main() { panic(err) } keyInfo := keystore.KeyInfo{ - ID: "AuthService", - Key: random, - Ts: time.Now().Unix(), - Role: "AuthService", - Caps: []byte(`{"*"}`), + ID: "AuthService", + AuthKey: random, + Ts: time.Now().Unix(), + Role: "AuthService", + Caps: []byte(`{"*"}`), } keyInfo.DumpJSONFile(*output[i]) } diff --git a/build/build.sh b/build/build.sh index 2aa70ce83..7a2ae5ed3 100755 --- a/build/build.sh +++ b/build/build.sh @@ -12,38 +12,25 @@ BuildTime=$(date +%Y-%m-%d\ %H:%M) LDFlags="-X main.CommitID=${CommitID} -X main.BranchName=${BranchName} -X 'main.BuildTime=${BuildTime}'" MODFLAGS="" -RM="rm -rf" -[[ -x "/usr/bin/rm" ]] && RM="/usr/bin/rm -rf" -[[ -x "/bin/rm" ]] && RM="/bin/rm -rf" - NPROC=$(nproc 2>/dev/null) NPROC=${NPROC:-"1"} GCC_LIBRARY_PATH="/lib /lib64 /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64" +cgo_cflags="" +cgo_ldflags="-lstdc++ -lm" [[ $(uname -s) != "Linux" ]] && { echo "ChubaoFS only support Linux os"; exit 1; } -TMPDIR=${HOME}/tmp/$$ -mkdir -p ${TMPDIR} - -set_go_path() { - export GOPATH=${TMPDIR} - mkdir -p $GOPATH/src/github.com/chubaofs - SrcPath=$GOPATH/src/github.com/chubaofs/chubaofs - if [[ ! -e "$SrcPath" ]] ; then - ln -s $RootPath $SrcPath 2>/dev/null - fi -} - build_snappy() { - SnappySrcPath=${RocksdbBuildPath}/third-party/snappy-1.1.7 - SnappyBuildPath=${SnappySrcPath}/build + found=$(find ${GCC_LIBRARY_PATH} -name libsnappy.a -o -name libsnappy.so 2>/dev/null | wc -l) + if [[ ${found} -gt 0 ]] ; then + cgo_ldflags="${cgo_ldflags} -lsnappy" + return + fi + SnappySrcPath=${VendorPath}/snappy-1.1.7 + SnappyBuildPath=${BuildOutPath}/snappy found=$(find ${SnappyBuildPath} -name libsnappy.a 2>/dev/null | wc -l) if [[ ${found} -eq 0 ]] ; then - if [[ ! -d ${RocksdbBuildPath} ]] ; then - mkdir -p ${RocksdbBuildPath} - cp -rf ${RocksdbSrcPath}/* ${RocksdbBuildPath} - fi mkdir -p ${SnappyBuildPath} echo "build snappy..." pushd ${SnappyBuildPath} >/dev/null @@ -55,11 +42,13 @@ build_snappy() { } build_rocksdb() { + found=$(find ${GCC_LIBRARY_PATH} -name librocksdb.a -o -name librocksdb.so 2>/dev/null | wc -l) + if [[ ${found} -gt 0 ]] ; then + cgo_ldflags="${cgo_ldflags} -lrocksdb" + return + fi RocksdbSrcPath=${VendorPath}/rocksdb-5.9.2 RocksdbBuildPath=${BuildOutPath}/rocksdb - - build_snappy - found=$(find ${RocksdbBuildPath} -name librocksdb.a 2>/dev/null | wc -l) if [[ ${found} -eq 0 ]] ; then if [[ ! -d ${RocksdbBuildPath} ]] ; then @@ -76,20 +65,32 @@ build_rocksdb() { cgo_ldflags="${cgo_ldflags} -L${RocksdbBuildPath} -lrocksdb" } -set_server_deps() { - cgo_cflags="" - cgo_ldflags="" - +pre_build() { + build_snappy build_rocksdb + rocksdb_libs=( z bz2 lz4 zstd ) + for p in ${rocksdb_libs[*]} ; do + found=$(find /usr -name lib${p}.so 2>/dev/null | wc -l) + if [[ ${found} -gt 0 ]] ; then + cgo_ldflags="${cgo_ldflags} -l${p}" + fi + done + export CGO_CFLAGS=${cgo_cflags} export CGO_LDFLAGS="${cgo_ldflags}" export GO111MODULE=off + export GOPATH=/tmp/cfs/go + + mkdir -p $GOPATH/src/github.com/chubaofs + SrcPath=$GOPATH/src/github.com/chubaofs/chubaofs + if [[ ! -e "$SrcPath" ]] ; then + ln -s $RootPath $SrcPath 2>/dev/null + fi } run_test() { - set_go_path - set_server_deps + pre_build pushd $SrcPath >/dev/null echo "run test " go test -ldflags "${LDFlags}" ./... @@ -97,17 +98,15 @@ run_test() { } build_server() { - set_go_path - set_server_deps + pre_build pushd $SrcPath >/dev/null echo -n "build cfs-server " go build $MODFLAGS -ldflags "${LDFlags}" -o ${BuildBinPath}/cfs-server ${SrcPath}/cmd/*.go && echo "success" || echo "failed" popd >/dev/null - unset CGO_LDFLAGS CGO_CFLAGS } build_client() { - set_go_path + pre_build pushd $SrcPath >/dev/null echo -n "build cfs-client " go build $MODFLAGS -ldflags "${LDFlags}" -o ${BuildBinPath}/cfs-client ${SrcPath}/client/*.go && echo "success" || echo "failed" @@ -115,7 +114,7 @@ build_client() { } build_client2() { - set_go_path + pre_build pushd $SrcPath >/dev/null echo -n "build cfs-client2 " go build $MODFLAGS -ldflags "${LDFlags}" -o ${BuildBinPath}/cfs-client2 ${SrcPath}/clientv2/*.go && echo "success" || echo "failed" @@ -123,7 +122,7 @@ build_client2() { } build_authtool() { - set_go_path + pre_build pushd $SrcPath >/dev/null echo -n "build cfs-authtool " go build $MODFLAGS -ldflags "${LDFlags}" -o ${BuildBinPath}/cfs-authtool ${SrcPath}/authtool/*.go && echo "success" || echo "failed" @@ -131,12 +130,12 @@ build_authtool() { } clean() { - ${RM} ${BuildBinPath} + rm -rf ${BuildBinPath} } dist_clean() { - ${RM} ${BuildBinPath} - ${RM} ${BuildOutPath} + rm -rf ${BuildBinPath} + rm -rf ${BuildOutPath} } cmd=${1:-"all"} @@ -170,5 +169,3 @@ case "$cmd" in *) ;; esac - -${RM} ${TMPDIR} diff --git a/client/fs/super.go b/client/fs/super.go index 1465facff..517f4ade9 100644 --- a/client/fs/super.go +++ b/client/fs/super.go @@ -81,7 +81,7 @@ var ( // NewSuper returns a new Super. func NewSuper(opt *MountOption) (s *Super, err error) { s = new(Super) - s.mw, err = meta.NewMetaWrapper(opt.Volname, opt.Owner, opt.Master, opt.Authenticate, opt.TicketMess) + s.mw, err = meta.NewMetaWrapper(opt.Volname, opt.Owner, opt.Master, opt.Authenticate, true, &opt.TicketMess) if err != nil { return nil, errors.Trace(err, "NewMetaWrapper failed!") } diff --git a/clientv2/fs/super.go b/clientv2/fs/super.go index 86def2cb5..948e0583d 100644 --- a/clientv2/fs/super.go +++ b/clientv2/fs/super.go @@ -75,7 +75,7 @@ var ( func NewSuper(opt *MountOption) (s *Super, err error) { s = new(Super) - s.mw, err = meta.NewMetaWrapper(opt.Volname, opt.Owner, opt.Master, opt.Authenticate, opt.TicketMess) + s.mw, err = meta.NewMetaWrapper(opt.Volname, opt.Owner, opt.Master, opt.Authenticate, true, &opt.TicketMess) if err != nil { return nil, errors.Trace(err, "NewMetaWrapper failed!") } diff --git a/cmd/build.sh b/cmd/build.sh old mode 100644 new mode 100755 diff --git a/cmd/cfg/meta.json b/cmd/cfg/meta.json index 101dddde1..7a0033c2a 100644 --- a/cmd/cfg/meta.json +++ b/cmd/cfg/meta.json @@ -7,7 +7,7 @@ "logDir": "/export/Logs/metanode", "raftDir": "/export/Data/metanode/raft", "raftHeartbeatPort": "9093", - "raftReplicatePort": "9094", + "raftReplicaPort": "9094", "totalMem": "17179869184", "warnLogDir":"/export/home/tomcat/UMP-Monitor/logs/", "consulAddr": "http://consul.prometheus-cfs.local", diff --git a/cmd/cfg/objectnode.json b/cmd/cfg/objectnode.json new file mode 100644 index 000000000..5b6306999 --- /dev/null +++ b/cmd/cfg/objectnode.json @@ -0,0 +1,15 @@ +{ + "role": "objectnode", + "domains": [ + "object.cfs.local" + ], + "listen": 80, + "masters": [ + "master1.cfs.local:80", + "master2.cfs.local:80", + "master3.cfs.local:80" + ], + "logLevel": "info", + "logDir": "/export/Logs/objectnode", + "region": "cn_bj" +} \ No newline at end of file diff --git a/cmd/cmd.go b/cmd/cmd.go index dcefa727e..7c59c43ed 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -28,6 +28,8 @@ import ( "strings" "syscall" + "github.com/chubaofs/chubaofs/objectnode" + "github.com/jacobsa/daemonize" "github.com/chubaofs/chubaofs/authnode" @@ -58,6 +60,7 @@ const ( RoleMeta = "metanode" RoleData = "datanode" RoleAuth = "authnode" + RoleObject = "objectnode" ) const ( @@ -65,6 +68,7 @@ const ( ModuleMeta = "metaNode" ModuleData = "dataNode" ModuleAuth = "authNode" + ModuleObject = "objectNode" ) const ( @@ -173,6 +177,9 @@ func main() { case RoleAuth: server = authnode.NewServer() module = ModuleAuth + case RoleObject: + server = objectnode.NewServer() + module = ModuleObject default: daemonize.SignalOutcome(fmt.Errorf("Fatal: role mismatch: %v", role)) os.Exit(1) diff --git a/datanode/data_partition_repair.go b/datanode/data_partition_repair.go index 06c8512ea..be9d597eb 100644 --- a/datanode/data_partition_repair.go +++ b/datanode/data_partition_repair.go @@ -126,7 +126,7 @@ func (dp *DataPartition) repair(extentType uint8) { log.LogInfof("action[repair] partition(%v) GoodTinyExtents(%v) BadTinyExtents(%v)"+ " finish cost[%vms] masterAddr(%v).", dp.partitionID, dp.extentStore.AvailableTinyExtentCnt(), - dp.extentStore.BrokenTinyExtentCnt(), (end-start)/int64(time.Millisecond), MasterHelper.Nodes()) + dp.extentStore.BrokenTinyExtentCnt(), (end-start)/int64(time.Millisecond), MasterClient.Nodes()) } func (dp *DataPartition) buildDataPartitionRepairTask(repairTasks []*DataPartitionRepairTask, extentType uint8, tinyExtents []uint64) (err error) { diff --git a/datanode/partition.go b/datanode/partition.go index aad7e1c6a..f2e61e22f 100644 --- a/datanode/partition.go +++ b/datanode/partition.go @@ -587,28 +587,16 @@ func (dp *DataPartition) compareReplicas(v1, v2 []string) (equals bool) { // Fetch the replica information from the master. func (dp *DataPartition) fetchReplicasFromMaster() (isLeader bool, replicas []string, err error) { - var ( - bufs []byte - ) - params := make(map[string]string) - params["id"] = strconv.Itoa(int(dp.partitionID)) - params["name"] = dp.volumeID - if bufs, err = MasterHelper.Request("GET", proto.AdminGetDataPartition, params, nil); err != nil { + var partition *master.DataPartition + if partition, err = MasterClient.AdminAPI().GetDataPartition(dp.volumeID, dp.partitionID); err != nil { isLeader = false return } - response := &master.DataPartition{} - replicas = make([]string, 0) - if err = json.Unmarshal(bufs, &response); err != nil { - isLeader = false - replicas = nil - return - } - for _, host := range response.Hosts { + for _, host := range partition.Hosts { replicas = append(replicas, host) } - if response.Hosts != nil && len(response.Hosts) >= 1 { - leaderAddr := strings.Split(response.Hosts[0], ":") + if partition.Hosts != nil && len(partition.Hosts) >= 1 { + leaderAddr := strings.Split(partition.Hosts[0], ":") if len(leaderAddr) == 2 && strings.TrimSpace(leaderAddr[0]) == LocalIP { isLeader = true } diff --git a/datanode/server.go b/datanode/server.go index 0e6411bf2..31b512db6 100644 --- a/datanode/server.go +++ b/datanode/server.go @@ -15,8 +15,8 @@ package datanode import ( - "encoding/json" "fmt" + "github.com/chubaofs/chubaofs/master" "net" "net/http" "regexp" @@ -34,6 +34,7 @@ import ( "github.com/chubaofs/chubaofs/proto" "github.com/chubaofs/chubaofs/raftstore" "github.com/chubaofs/chubaofs/repl" + masterSDK "github.com/chubaofs/chubaofs/sdk/master" "github.com/chubaofs/chubaofs/util" "github.com/chubaofs/chubaofs/util/config" "github.com/chubaofs/chubaofs/util/exporter" @@ -47,7 +48,7 @@ var ( LocalIP string gConnPool = util.NewConnectPool() - MasterHelper = util.NewMasterHelper() + MasterClient = masterSDK.NewMasterClient(nil, false) ) const ( @@ -87,6 +88,7 @@ type DataNode struct { raftHeartbeat string raftReplica string raftStore raftstore.RaftStore + tcpListener net.Listener stopC chan bool state uint32 @@ -194,13 +196,13 @@ func (s *DataNode) parseConfig(cfg *config.Config) (err error) { return fmt.Errorf("Err:masterAddr unavalid") } for _, ip := range cfg.GetArray(ConfigKeyMasterAddr) { - MasterHelper.AddNode(ip.(string)) + MasterClient.AddNode(ip.(string)) } s.cellName = cfg.GetString(ConfigKeyCell) if s.cellName == "" { s.cellName = DefaultCellName } - log.LogDebugf("action[parseConfig] load masterAddrs(%v).", MasterHelper.Nodes()) + log.LogDebugf("action[parseConfig] load masterAddrs(%v).", MasterClient.Nodes()) log.LogDebugf("action[parseConfig] load port(%v).", s.port) log.LogDebugf("action[parseConfig] load cellName(%v).", s.cellName) return @@ -258,7 +260,6 @@ func (s *DataNode) startSpaceManager(cfg *config.Config) (err error) { func (s *DataNode) register(cfg *config.Config) { var ( err error - data []byte ) timer := time.NewTimer(0) @@ -267,20 +268,18 @@ func (s *DataNode) register(cfg *config.Config) { for { select { case <-timer.C: - data, err = MasterHelper.Request(http.MethodGet, proto.AdminGetIP, nil, nil) - masterAddr := MasterHelper.Leader() - if err != nil { + var ci *proto.ClusterInfo + if ci, err = MasterClient.AdminAPI().GetClusterInfo(); err != nil { log.LogErrorf("action[registerToMaster] cannot get ip from master(%v) err(%v).", - masterAddr, err) + MasterClient.Leader(), err) timer.Reset(2 * time.Second) continue } - cInfo := new(proto.ClusterInfo) - json.Unmarshal(data, cInfo) + masterAddr := MasterClient.Leader() + s.clusterID = ci.Cluster if LocalIP == "" { - LocalIP = string(cInfo.Ip) + LocalIP = string(ci.Ip) } - s.clusterID = cInfo.Cluster s.localServerAddr = fmt.Sprintf("%s:%v", LocalIP, s.port) if !util.IsIPV4(LocalIP) { log.LogErrorf("action[registerToMaster] got an invalid local ip(%v) from master(%v).", @@ -290,21 +289,16 @@ func (s *DataNode) register(cfg *config.Config) { } // register this data node on the master - params := make(map[string]string) - params["addr"] = fmt.Sprintf("%s:%v", LocalIP, s.port) - data, err = MasterHelper.Request(http.MethodPost, proto.AddDataNode, params, nil) - if err != nil { + var nodeID uint64 + if nodeID, err = MasterClient.NodeAPI().AddDataNode(fmt.Sprintf("%s:%v", LocalIP, s.port)); err != nil { log.LogErrorf("action[registerToMaster] cannot register this node to master[%v] err(%v).", masterAddr, err) timer.Reset(2 * time.Second) continue } - exporter.RegistConsul(s.clusterID, ModuleName, cfg) - - nodeID := strings.TrimSpace(string(data)) - s.nodeID, err = strconv.ParseUint(nodeID, 10, 64) - log.LogDebugf("[tempDebug] nodeID(%v)", s.nodeID) + s.nodeID = nodeID + log.LogDebugf("register: register DataNode: nodeID(%v)", s.nodeID) return case <-s.stopC: timer.Stop() @@ -319,23 +313,21 @@ type DataNodeInfo struct { } func (s *DataNode) checkLocalPartitionMatchWithMaster() (err error) { - params := make(map[string]string) - params["addr"] = s.localServerAddr - var data interface{} + var convert = func(node *master.DataNode) *DataNodeInfo { + result := &DataNodeInfo{} + result.Addr = node.Addr + result.PersistenceDataPartitions = node.PersistenceDataPartitions + return result + } + var dataNode *master.DataNode for i := 0; i < 3; i++ { - data, err = MasterHelper.Request(http.MethodGet, proto.GetDataNode, params, nil) - if err != nil { + if dataNode, err = MasterClient.NodeAPI().GetDataNode(s.localServerAddr); err != nil { log.LogErrorf("checkLocalPartitionMatchWithMaster error %v", err) continue } break } - dinfo := new(DataNodeInfo) - if err = json.Unmarshal(data.([]byte), dinfo); err != nil { - err = fmt.Errorf("checkLocalPartitionMatchWithMaster jsonUnmarsh failed %v", err) - log.LogErrorf(err.Error()) - return - } + dinfo := convert(dataNode) if len(dinfo.PersistenceDataPartitions) == 0 { return } diff --git a/datanode/wrap_operator.go b/datanode/wrap_operator.go index 51f5bac9e..38f955986 100644 --- a/datanode/wrap_operator.go +++ b/datanode/wrap_operator.go @@ -187,10 +187,7 @@ func (s *DataNode) handlePacketToCreateDataPartition(p *repl.Packet) { // Handle OpHeartbeat packet. func (s *DataNode) handleHeartbeatPacket(p *repl.Packet) { - var ( - data []byte - err error - ) + var err error task := &proto.AdminTask{} err = json.Unmarshal(p.Data, task) defer func() { @@ -210,8 +207,8 @@ func (s *DataNode) handleHeartbeatPacket(p *repl.Packet) { s.buildHeartBeatResponse(response) if task.OpCode == proto.OpDataNodeHeartbeat { - bytes, _ := json.Marshal(task.Request) - json.Unmarshal(bytes, request) + marshaled, _ := json.Marshal(task.Request) + _ = json.Unmarshal(marshaled, request) response.Status = proto.TaskSucceeds } else { response.Status = proto.TaskFailed @@ -219,11 +216,7 @@ func (s *DataNode) handleHeartbeatPacket(p *repl.Packet) { response.Result = err.Error() } task.Response = response - if data, err = json.Marshal(task); err != nil { - return - } - _, err = MasterHelper.Request("POST", proto.GetDataNodeTaskResponse, nil, data) - if err != nil { + if err = MasterClient.NodeAPI().ResponseDataNodeTask(task); err != nil { err = errors.Trace(err, "heartbeat to master(%v) failed.", request.MasterAddr) log.LogErrorf(err.Error()) return @@ -312,15 +305,7 @@ func (s *DataNode) asyncLoadDataPartition(task *proto.AdminTask) { response.Result = err.Error() } task.Response = response - data, err := json.Marshal(task) - if err != nil { - response.PartitionId = uint64(request.PartitionId) - response.Status = proto.TaskFailed - response.Result = err.Error() - err = fmt.Errorf("from master Task(%v) failed,error(%v)", task.ToString(), response.Result) - } - _, err = MasterHelper.Request("POST", proto.GetDataNodeTaskResponse, nil, data) - if err != nil { + if err = MasterClient.NodeAPI().ResponseDataNodeTask(task); err != nil { err = errors.Trace(err, "load DataPartition failed,PartitionID(%v)", request.PartitionId) log.LogError(errors.Stack(err)) } diff --git a/docker/Dockerfile-cfs b/docker/Dockerfile-cfs deleted file mode 100644 index 18b4a130d..000000000 --- a/docker/Dockerfile-cfs +++ /dev/null @@ -1,11 +0,0 @@ -FROM centos:7 AS base -RUN curl -o /etc/yum.repos.d/epel-7.repo http://mirrors.aliyun.com/repo/epel-7.repo && \ - yum install -y bind-utils xfsprogs jq fuse -RUN mkdir -p /cfs/bin /cfs/conf /cfs/logs /cfs/data - -FROM base AS server -COPY build/bin/cfs-server /cfs/bin/ - -FROM base AS client -COPY build/bin/cfs-client /cfs/bin/ - diff --git a/docker/authnode/authnode1.json b/docker/authnode/authnode1.json index 3f980c551..bbd5e6373 100644 --- a/docker/authnode/authnode1.json +++ b/docker/authnode/authnode1.json @@ -4,7 +4,7 @@ "port": "8080", "prof":"10088", "id":"1", - "peers": "1:192.168.0.14:8080,2:192.168.0.15:8081", + "peers": "1:192.168.0.14:8080,2:192.168.0.15:8081,3:192.168.0.16:8082", "retainLogs":"2", "logDir": "/export/Logs/authnode", "logLevel":"info", diff --git a/docker/authnode/authnode2.json b/docker/authnode/authnode2.json index d360bc79c..bd0cd7f09 100644 --- a/docker/authnode/authnode2.json +++ b/docker/authnode/authnode2.json @@ -4,7 +4,7 @@ "port": "8081", "prof":"10088", "id":"2", - "peers": "1:192.168.0.14:8080,2:192.168.0.15:8081", + "peers": "1:192.168.0.14:8080,2:192.168.0.15:8081,3:192.168.0.16:8082", "retainLogs":"2", "logDir": "/export/Logs/authnode", "logLevel":"info", diff --git a/docker/authnode/run_docker4auth.sh b/docker/authnode/run_docker4auth.sh index f8010e329..933e77998 100644 --- a/docker/authnode/run_docker4auth.sh +++ b/docker/authnode/run_docker4auth.sh @@ -3,8 +3,8 @@ #write authkey to authnode.json cd .. cd .. -cp ./build/bin/cfs-server /home/wuwenjia/gocode/src/github.com/chubaofs/chubaofs/docker/authnode/ -cp ./build/bin/cfs-authtool /home/wuwenjia/gocode/src/github.com/chubaofs/chubaofs/docker/authnode/ +cp ./build/bin/cfs-server /home/wuwenjia/gocode/src/github.com/chubaofs/chubaofs/docker/authnode/. +cp ./build/bin/cfs-authtool /home/wuwenjia/gocode/src/github.com/chubaofs/chubaofs/docker/authnode/. cd docker/authnode ./cfs-authtool authkey authnodeKey=$(sed -n '3p' authservice.json | sed 's/key/authServiceKey/g') diff --git a/docker/build_docker_cfs.sh b/docker/build_docker_cfs.sh deleted file mode 100755 index c822e614f..000000000 --- a/docker/build_docker_cfs.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -Version="1.0.0" -if [[ -n "$1" ]] ;then - # docker image tag of ChubaoFS - Version=$1 -fi - -RootPath=$(cd $(dirname $0)/..; pwd) - -source ${RootPath}/build/build.sh -CfsServer="chubaofs/cfs-server:$Version" -CfsClient="chubaofs/cfs-client:$Version" -docker build -t ${CfsServer} -f ${RootPath}/docker/Dockerfile-cfs --target server ${RootPath} -docker build -t ${CfsClient} -f ${RootPath}/docker/Dockerfile-cfs --target client ${RootPath} diff --git a/docker/conf/metanode.json b/docker/conf/metanode.json index 2feaf73b4..9e69d08aa 100644 --- a/docker/conf/metanode.json +++ b/docker/conf/metanode.json @@ -8,9 +8,15 @@ "exporterPort": 9500, "logLevel": "info", "logDir": "/cfs/log", + "warnLogDir":"/cfs/log", "totalMem":"536870912", "metadataDir": "/cfs/data/meta", "raftDir": "/cfs/data/raft", + "masterAddrs": [ + "192.168.0.11:17010", + "192.168.0.12:17010", + "192.168.0.13:17010" + ], "masterAddr": [ "192.168.0.11:17010", "192.168.0.12:17010", diff --git a/docker/conf/objectnode.json b/docker/conf/objectnode.json new file mode 100644 index 000000000..cdb1e5c28 --- /dev/null +++ b/docker/conf/objectnode.json @@ -0,0 +1,12 @@ +{ + "role": "objectnode", + "hosts": "127.0.0.1", + "logDir": "/cfs/log/", + "logLevel": "debug", + "listen": 80, + "masters": [ + "192.168.0.11:17010", + "192.168.0.12:17010", + "192.168.0.13:17010" + ] +} diff --git a/docker/docker-compose-auth.yml b/docker/docker-compose-auth.yml index e9b061e13..f0c6755a2 100644 --- a/docker/docker-compose-auth.yml +++ b/docker/docker-compose-auth.yml @@ -27,6 +27,7 @@ services: - datanode2 - datanode3 - datanode4 + - s3node1 networks: authnode_extnetwork: @@ -103,6 +104,7 @@ services: privileged: true networks: authnode_extnetwork: + ipv4_address: 192.168.0.21 metanode2: image: chubaofs/cfs-base:1.0 @@ -121,6 +123,7 @@ services: privileged: true networks: authnode_extnetwork: + ipv4_address: 192.168.0.22 metanode3: image: chubaofs/cfs-base:1.0 @@ -139,6 +142,7 @@ services: privileged: true networks: authnode_extnetwork: + ipv4_address: 192.168.0.23 datanode1: image: chubaofs/cfs-base:1.0 @@ -158,6 +162,7 @@ services: privileged: true networks: authnode_extnetwork: + ipv4_address: 192.168.0.31 datanode2: image: chubaofs/cfs-base:1.0 @@ -177,6 +182,7 @@ services: privileged: true networks: authnode_extnetwork: + ipv4_address: 192.168.0.32 datanode3: image: chubaofs/cfs-base:1.0 @@ -196,6 +202,7 @@ services: privileged: true networks: authnode_extnetwork: + ipv4_address: 192.168.0.33 datanode4: image: chubaofs/cfs-base:1.0 @@ -215,6 +222,22 @@ services: privileged: true networks: authnode_extnetwork: + ipv4_address: 192.168.0.34 + + s3node1: + image: chubaofs/cfs-base:1.0 + ports: + - "80" + volumes: + - ./bin:/cfs/bin + - ./conf/s3node.json:/cfs/conf/s3node.json + - ./script/start_s3node.sh:/cfs/script/start.sh + command: /bin/sh /cfs/script/start.sh + restart: on-failure + privileged: true + networks: + authnode_extnetwork: + ipv4_address: 192.168.0.31 client: image: chubaofs/cfs-base:1.0 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2bc9e4426..774e01ccc 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -30,6 +30,7 @@ services: - datanode2 - datanode3 - datanode4 + - objectnode1 networks: extnetwork: @@ -226,6 +227,21 @@ services: extnetwork: ipv4_address: 192.168.0.34 + objectnode1: + image: chubaofs/cfs-base:1.0 + ports: + - "80" + volumes: + - ./bin:/cfs/bin + - ./conf/objectnode.json:/cfs/conf/objectnode.json + - ./script/start_objectnode.sh:/cfs/script/start.sh + command: /bin/sh /cfs/script/start.sh + restart: on-failure + privileged: true + networks: + extnetwork: + ipv4_address: 192.168.0.41 + client: image: chubaofs/cfs-base:1.0 ports: diff --git a/docker/run_docker.sh b/docker/run_docker.sh index c9adca6d2..0880c1d00 100755 --- a/docker/run_docker.sh +++ b/docker/run_docker.sh @@ -11,10 +11,11 @@ help() { Usage: ./run_docker.sh [ -h | --help ] [ -d | --disk ] [ -l | --ltptest ] -h, --help show help info - -d, --disk set datanode local disk path - -b, --build build chubaofs server and cliente - -s, --server start chubaofs servers docker image - -c, --client start chubaofs client docker image + -d, --disk set ChubaoFS DataNode local disk path + -b, --build build ChubaoFS server and client + -s, --server start ChubaoFS servers docker image + -o, --object-node start ChubaoFS + -c, --client start ChubaoFS client docker image -m, --monitor start monitor web ui -l, --ltptest run ltp test -r, --run run servers, client and monitor @@ -44,6 +45,10 @@ start_client() { docker-compose -f ${RootPath}/docker/docker-compose.yml run client bash -c "/cfs/script/start_client.sh ; /bin/bash" } +start_objectnode() { + docker-compose -f ${RootPath}/docker/docker-compose.yml up -d objectnode +} + start_monitor() { docker-compose -f ${RootPath}/docker/docker-compose.yml up -d monitor } @@ -62,6 +67,7 @@ run() { build start_monitor start_servers + start_objectnode start_client } @@ -137,6 +143,7 @@ case "-$cmd" in -build) build ;; -run_servers) start_servers ;; -run_client) start_client ;; + -run_s3node) start_s3node ;; -run_monitor) start_monitor ;; -run_ltptest) run_ltptest ;; -clean) clean ;; diff --git a/docker/run_docker_auth.sh b/docker/run_docker_auth.sh index 01a3c73bb..88050aff9 100644 --- a/docker/run_docker_auth.sh +++ b/docker/run_docker_auth.sh @@ -4,6 +4,8 @@ RootPath=$(cd $(dirname $0)/..; pwd) GOPATH=/go export DiskPath="$RootPath/docker/disk" +MIN_DNDISK_AVAIL_SIZE_GB=10 + help() { cat < ] [ -l | -b, --build build chubaofs server and cliente -s, --server start chubaofs servers docker image -c, --client start chubaofs client docker image + -s3, --s3node start chubaofs s3node -m, --monitor start monitor web ui -l, --ltptest run ltp test -r, --run run servers, client and monitor @@ -33,6 +36,7 @@ build() { # start server start_servers() { + isDiskAvailable $DiskPath mkdir -p ${DiskPath}/{1..4} docker-compose -f ${RootPath}/docker/docker-compose-auth.yml up -d servers } @@ -41,6 +45,10 @@ start_client() { docker-compose -f ${RootPath}/docker/docker-compose-auth.yml run client bash -c "/cfs/script/start_client.sh ; /bin/bash" } +start_s3node() { + docker-compose -f ${RootPath}/docker/docker-compose-auth.yml up -d s3node1 +} + start_monitor() { docker-compose -f ${RootPath}/docker/docker-compose-auth.yml up -d monitor } @@ -82,6 +90,9 @@ for opt in ${ARGS[*]} ; do -s|--server) cmd=run_servers ;; + -s3|--s3node) + cmd=run_s3node + ;; -c|--client) cmd=run_client ;; @@ -96,12 +107,27 @@ for opt in ${ARGS[*]} ; do esac done +function isDiskAvailable() { + Disk=${1:-"need diskpath"} + [[ -d $Disk ]] || mkdir -p $Disk + if [[ ! -d $Disk ]] ; then + echo "error: $DiskPath must be exist and at least 10GB free size" + exit 1 + fi + avail_sectors=$(df $Disk | tail -1 | awk '{print $4}') + avail_GB=$(( $avail_sectors / 1024 / 1024 / 2 )) + if (( $avail_GB < $MIN_DNDISK_AVAIL_SIZE_GB )) ; then + echo "$Disk: avaible size $avail_GB GB < Min Disk avaible size $MIN_DNDISK_AVAIL_SIZE_GB GB" ; + exit 1 + fi +} + for opt in ${ARGS[*]} ; do case "-$1" in --d|---disk) shift export DiskPath=${1:?"need disk dir path"} - [[ -d $DiskPath ]] || { echo "error: $DiskPath must be exist and at least 30GB free size"; exit 1; } + isDiskAvailable $DiskPath shift ;; -) @@ -119,9 +145,9 @@ case "-$cmd" in -build) build ;; -run_servers) start_servers ;; -run_client) start_client ;; + -run_s3node) start_s3node ;; -run_monitor) start_monitor ;; -run_ltptest) run_ltptest ;; -clean) clean ;; *) help ;; esac - diff --git a/docker/script/start_client.sh b/docker/script/start_client.sh index a2f3686f4..eaf42a4c1 100755 --- a/docker/script/start_client.sh +++ b/docker/script/start_client.sh @@ -94,8 +94,8 @@ print_error_info() { start_client() { echo -n "start client " + /cfs/bin/cfs-client -c /cfs/conf/client.json for((i=0; i<$TryTimes; i++)) ; do - nohup /cfs/bin/cfs-client -c /cfs/conf/client.json >/cfs/log/cfs.out 2>&1 & sleep 2 sta=$(stat $MntPoint 2>/dev/null | tr ":：" " " | awk '/Inode/{print $4}') if [[ "x$sta" == "x1" ]] ; then diff --git a/docker/script/start_client_auth.sh b/docker/script/start_client_auth.sh index 1116ae148..87fa2b4a7 100644 --- a/docker/script/start_client_auth.sh +++ b/docker/script/start_client_auth.sh @@ -94,8 +94,8 @@ print_error_info() { start_client() { echo -n "start client " + /cfs/bin/cfs-client -c /cfs/conf/client.json for((i=0; i<$TryTimes; i++)) ; do - nohup /cfs/bin/cfs-client -c /cfs/conf/client.json >/cfs/log/cfs.out 2>&1 & sleep 2 sta=$(stat $MntPoint 2>/dev/null | tr ":：" " " | awk '/Inode/{print $4}') if [[ "x$sta" == "x1" ]] ; then @@ -112,5 +112,4 @@ getLeaderAddr check_status "MetaNode" check_status "DataNode" create_vol -start_client - +start_client \ No newline at end of file diff --git a/docker/script/start_objectnode.sh b/docker/script/start_objectnode.sh new file mode 100755 index 000000000..9ff92ccdb --- /dev/null +++ b/docker/script/start_objectnode.sh @@ -0,0 +1,7 @@ +#!/bin/sh +rm -rf /cfs/disk/* /cfs/log/* +mkdir -p /cfs/bin /cfs/log + +echo "start objectnode" +/cfs/bin/cfs-server -f -c /cfs/conf/objectnode.json ; sleep 99999999d + diff --git a/docs/source/index.rst b/docs/source/index.rst index e0ebc7b1d..664ff79a1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -24,6 +24,7 @@ Welcome to ChubaoFS(Chubao File System) user-guide/master user-guide/metanode user-guide/datanode + user-guide/objectnode user-guide/client user-guide/monitor user-guide/fuse diff --git a/docs/source/user-guide/objectnode.rst b/docs/source/user-guide/objectnode.rst new file mode 100644 index 000000000..8cdfaf4d5 --- /dev/null +++ b/docs/source/user-guide/objectnode.rst @@ -0,0 +1,57 @@ +Object Storage Service +============================== + +How To Provide Object Storage Service with ObjectNode +----------------------- + +Start a ObjectNode process by execute the server binary of ChubaoFS you built with ``-c`` argument and specify configuration file. + +.. code-block:: bash + + nohup cfs-server -c s3gateway.json & + + +Configurations +----------------------- + +.. csv-table:: Properties + :header: "Key", "Type", "Description", "Mandatory" + + "role", "string", "Role of process and must be set to *objectnode*", "Yes" + "listen", "string", "Listen and accept port of the server. Default: 80", "Yes" + "region", "string", "Region of this gateway. Used by S3-like interface signature validation. Default: cfs_default", "No" + "domains", "string slice", " + | Format: *DOMAIN*. + | DOMAIN: Domain of S3-like interface which makes wildcard domain support", "No" + "logDir", "string", "Log directory", "Yes" + "logLevel", "string", "Level operation for logging. Default is *error*", "No" + "masters", "string slice", " + | Format: *HOST:PORT*. + | HOST: Hostname, domain or IP address of master (resource manager). + | PORT: port number which listened by this master", "Yes" + "exporterPort", "string", "Port for monitor system", "No" + "prof", "string", "Pprof port", "Yes" + + +**Example:** + +.. code-block:: json + + { + "role": "objectnode", + "listen": 80, + "region": "test", + "domains": [ + "object.cfs.local" + ], + "logDir": "/opt/cfs/objectnode/logs", + "logLevel": "debug", + "masters": [ + "172.20.240.95:7002", + "172.20.240.94:7002", + "172.20.240.67:7002" + ], + "exporterPort": 9512, + "prof": "7013" + } + diff --git a/master/api_service.go b/master/api_service.go index f16d571db..818c19e67 100644 --- a/master/api_service.go +++ b/master/api_service.go @@ -22,13 +22,14 @@ import ( "time" "bytes" + "io/ioutil" + "strings" + "github.com/chubaofs/chubaofs/proto" "github.com/chubaofs/chubaofs/util" "github.com/chubaofs/chubaofs/util/cryptoutil" "github.com/chubaofs/chubaofs/util/errors" "github.com/chubaofs/chubaofs/util/log" - "io/ioutil" - "strings" ) // ClusterView provides the view of a cluster. @@ -903,8 +904,43 @@ func parseRequestToGetTaskResponse(r *http.Request) (tr *proto.AdminTask, err er return } -func parseRequestToGetVol(r *http.Request) (name, authKey string, err error) { - return parseVolNameAndAuthKey(r) +func parseVolName(r *http.Request) (name string, err error) { + if err = r.ParseForm(); err != nil { + return + } + if name, err = extractName(r); err != nil { + return + } + return +} + +type getVolParameter struct { + name string + authKey string + skipOwnerValidation bool +} + +func parseGetVolParameter(r *http.Request) (p *getVolParameter, err error) { + p = &getVolParameter{} + skipOwnerValidationVal := r.Header.Get(proto.SkipOwnerValidation) + if len(skipOwnerValidationVal) > 0 { + if p.skipOwnerValidation, err = strconv.ParseBool(skipOwnerValidationVal); err != nil { + return + } + } + if p.name = r.FormValue(nameKey); p.name == "" { + err = keyNotFound(nameKey) + return + } + if !volNameRegexp.MatchString(p.name) { + err = errors.New("name can only be number and letters") + return + } + if p.authKey = r.FormValue(volAuthKey); !p.skipOwnerValidation && len(p.authKey) == 0 { + err = keyNotFound(volAuthKey) + return + } + return } func parseVolNameAndAuthKey(r *http.Request) (name, authKey string, err error) { @@ -1333,24 +1369,23 @@ func (m *Server) getDataPartitions(w http.ResponseWriter, r *http.Request) { func (m *Server) getVol(w http.ResponseWriter, r *http.Request) { var ( - err error - name string - authKey string - vol *Vol - checkMessage string - jobj proto.APIAccessReq - ticket cryptoutil.Ticket - ts int64 + err error + vol *Vol + message string + jobj proto.APIAccessReq + ticket cryptoutil.Ticket + ts int64 + param *getVolParameter ) - if name, authKey, err = parseRequestToGetVol(r); err != nil { + if param, err = parseGetVolParameter(r); err != nil { sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeParamError, Msg: err.Error()}) return } - if vol, err = m.cluster.getVol(name); err != nil { + if vol, err = m.cluster.getVol(param.name); err != nil { sendErrReply(w, r, newErrHTTPReply(proto.ErrVolNotExists)) return } - if !matchKey(vol.Owner, authKey) { + if !param.skipOwnerValidation && !matchKey(vol.Owner, param.authKey) { sendErrReply(w, r, newErrHTTPReply(proto.ErrVolAuthKeyNotMatch)) return } @@ -1360,7 +1395,7 @@ func (m *Server) getVol(w http.ResponseWriter, r *http.Request) { viewCache = vol.getViewCache() } if vol.authenticate { - if jobj, ticket, ts, err = parseAndCheckTicket(r, m.cluster.MasterSecretKey, name); err != nil { + if jobj, ticket, ts, err = parseAndCheckTicket(r, m.cluster.MasterSecretKey, param.name); err != nil { if err == proto.ErrExpiredTicket { sendErrReply(w, r, newErrHTTPReply(err)) return @@ -1368,15 +1403,11 @@ func (m *Server) getVol(w http.ResponseWriter, r *http.Request) { sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeInvalidTicket, Msg: err.Error()}) return } - if checkMessage, err = genCheckMessage(&jobj, ts, ticket.SessionKey.Key); err != nil { + if message, err = genRespMessage(viewCache, &jobj, ts, ticket.SessionKey.Key); err != nil { sendErrReply(w, r, &proto.HTTPReply{Code: proto.ErrCodeMasterAPIGenRespError, Msg: err.Error()}) return } - resp := &proto.GetVolResponse{ - VolViewCache: viewCache, - CheckMsg: checkMessage, - } - sendOkReply(w, r, newSuccessHTTPReply(resp)) + sendOkReply(w, r, newSuccessHTTPReply(message)) } else { send(w, r, viewCache) } @@ -1565,23 +1596,25 @@ func extractTicketMess(req *proto.APIAccessReq, key []byte, volName string) (tic err = fmt.Errorf("CheckAPIAccessCaps failed: %s", err.Error()) return } - if err = proto.CheckVOLAccessCaps(&ticket, proto.VOLRsc, volName, proto.VOLAccess, proto.MasterNode); err != nil { + if err = proto.CheckVOLAccessCaps(&ticket, proto.OwnerVOLRsc, volName, proto.VOLAccess, proto.MasterNode); err != nil { err = fmt.Errorf("CheckVOLAccessCaps failed: %s", err.Error()) return } return } -func genCheckMessage(req *proto.APIAccessReq, ts int64, key []byte) (message string, err error) { +func genRespMessage(data []byte, req *proto.APIAccessReq, ts int64, key []byte) (message string, err error) { var ( jresp []byte - resp proto.APIAccessResp + resp proto.MasterAPIAccessResp ) - resp.Type = req.Type + 1 - resp.ClientID = req.ClientID - resp.ServiceID = req.ServiceID - resp.Verifier = ts + 1 // increase ts by one for client verify server + resp.Data = data + + resp.APIResp.Type = req.Type + 1 + resp.APIResp.ClientID = req.ClientID + resp.APIResp.ServiceID = req.ServiceID + resp.APIResp.Verifier = ts + 1 // increase ts by one for client verify server if jresp, err = json.Marshal(resp); err != nil { err = fmt.Errorf("json marshal for response failed %s", err.Error()) diff --git a/master/api_service_test.go b/master/api_service_test.go index 4697e656e..862df268b 100644 --- a/master/api_service_test.go +++ b/master/api_service_test.go @@ -67,7 +67,6 @@ func createDefaultMasterServerForTest() *Server { "electionTick":6, "logDir": "/export/chubaofs/Logs", "logLevel":"DEBUG", - "warnLogDir": "/export/chubaofs/Logs", "walDir":"/export/chubaofs/raft", "storeDir":"/export/chubaofs/rocksdbstore", "clusterName":"chubaofs" diff --git a/master/cluster.go b/master/cluster.go index 888bde59d..64a3acbaf 100644 --- a/master/cluster.go +++ b/master/cluster.go @@ -1168,6 +1168,8 @@ func (c *Cluster) doCreateVol(name, owner string, dpSize, capacity uint64, dpRep goto errHandler } vol = newVol(id, name, owner, dpSize, capacity, uint8(dpReplicaNum), defaultReplicaNum, followerRead, authenticate) + // refresh oss secure + vol.refreshOSSSecure() if err = c.syncAddVol(vol); err != nil { goto errHandler } diff --git a/master/cluster_test.go b/master/cluster_test.go index 21928b267..3bf681fd6 100644 --- a/master/cluster_test.go +++ b/master/cluster_test.go @@ -19,7 +19,7 @@ func buildPanicVol() *Vol { if err != nil { return nil } - vol := newVol(id, commonVol.Name, commonVol.Owner, commonVol.dataPartitionSize, commonVol.Capacity, defaultReplicaNum, defaultReplicaNum, false, false) + vol := newVol(id, commonVol.Name, commonVol.Owner, commonVol.dataPartitionSize, commonVol.Capacity, defaultReplicaNum, defaultReplicaNum, false) vol.dataPartitions = nil return vol } diff --git a/master/data_partition_check.go b/master/data_partition_check.go index 69787a5bd..452476a1b 100644 --- a/master/data_partition_check.go +++ b/master/data_partition_check.go @@ -219,12 +219,12 @@ func (partition *DataPartition) missingReplicaAddress(dataPartitionSize uint64) } // go through all the hosts to find the missing replica - for _, addr := range partition.Hosts { - if _, ok := partition.hasReplica(addr); !ok { + for _, host := range partition.Hosts { + if _, ok := partition.hasReplica(host); !ok { log.LogError(fmt.Sprintf("action[missingReplicaAddress],partitionID:%v lack replication:%v", - partition.PartitionID, addr)) + partition.PartitionID, host)) err = proto.ErrMissingReplica - addr = addr + addr = host break } } diff --git a/master/http_server.go b/master/http_server.go index e03ffb560..653cf1146 100644 --- a/master/http_server.go +++ b/master/http_server.go @@ -64,7 +64,7 @@ func (m *Server) handleFunctions() { http.Handle(proto.ClientMetaPartition, m.handlerWithInterceptor()) http.Handle(proto.GetDataNodeTaskResponse, m.handlerWithInterceptor()) http.Handle(proto.GetMetaNodeTaskResponse, m.handlerWithInterceptor()) - http.Handle(proto.AdminCreateMP, m.handlerWithInterceptor()) + http.Handle(proto.AdminCreateMetaPartition, m.handlerWithInterceptor()) http.Handle(proto.ClientVolStat, m.handlerWithInterceptor()) http.Handle(proto.AddRaftNode, m.handlerWithInterceptor()) http.Handle(proto.RemoveRaftNode, m.handlerWithInterceptor()) @@ -165,7 +165,7 @@ func (m *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.loadMetaPartition(w, r) case proto.AdminDecommissionMetaPartition: m.decommissionMetaPartition(w, r) - case proto.AdminCreateMP: + case proto.AdminCreateMetaPartition: m.createMetaPartition(w, r) case proto.AdminAddMetaReplica: m.addMetaReplica(w, r) diff --git a/master/master_manager_test.go b/master/master_manager_test.go index 3988282b4..18b43136d 100644 --- a/master/master_manager_test.go +++ b/master/master_manager_test.go @@ -61,6 +61,7 @@ func TestRaft(t *testing.T) { } func snapshotTest(t *testing.T) { + var err error mdSnapshot, err := server.cluster.fsm.Snapshot() if err != nil { t.Error(err) @@ -68,7 +69,12 @@ func snapshotTest(t *testing.T) { } t.Logf("snapshot apply index[%v]\n", mdSnapshot.ApplyIndex()) s := &Server{} - dbStore := raftstore.NewRocksDBStore("/export/chubaofs/raft2", LRUCacheSize, WriteBufferSize) + + var dbStore *raftstore.RocksDBStore + dbStore, err = raftstore.NewRocksDBStore("/export/chubaofs/raft2", LRUCacheSize, WriteBufferSize) + if err != nil { + t.Fatalf("ioen rocks db store fail cause: %v", err) + } fsm := &MetadataFsm{ rs: server.fsm.rs, store: dbStore, diff --git a/master/meta_partition_test.go b/master/meta_partition_test.go index 58348afe5..ecc980afe 100644 --- a/master/meta_partition_test.go +++ b/master/meta_partition_test.go @@ -37,7 +37,7 @@ func createMetaPartition(vol *Vol, t *testing.T) { var start uint64 start = mp.Start + defaultMetaPartitionInodeIDStep reqURL := fmt.Sprintf("%v%v?name=%v&start=%v", - hostAddr, proto.AdminCreateMP, vol.Name, start) + hostAddr, proto.AdminCreateMetaPartition, vol.Name, start) fmt.Println(reqURL) process(reqURL, t) if start < mp.MaxInodeID { diff --git a/master/metadata_fsm_op.go b/master/metadata_fsm_op.go index da3d40edf..406ac1699 100644 --- a/master/metadata_fsm_op.go +++ b/master/metadata_fsm_op.go @@ -116,6 +116,13 @@ type volValue struct { Owner string FollowerRead bool Authenticate bool + OSSAccessKey string + OSSSecretKey string +} + +func (v *volValue) Bytes() (raw []byte, err error) { + raw, err = json.Marshal(v) + return } func newVolValue(vol *Vol) (vv *volValue) { @@ -130,10 +137,20 @@ func newVolValue(vol *Vol) (vv *volValue) { Owner: vol.Owner, FollowerRead: vol.FollowerRead, Authenticate: vol.authenticate, + OSSAccessKey: vol.OSSAccessKey, + OSSSecretKey: vol.OSSSecretKey, } return } +func newVolValueFromBytes(raw []byte) (*volValue, error) { + vv := &volValue{} + if err := json.Unmarshal(raw, vv); err != nil { + return nil, err + } + return vv, nil +} + type dataNodeValue struct { ID uint64 NodeSetID uint64 @@ -519,12 +536,12 @@ func (c *Cluster) loadVols() (err error) { return err } for _, value := range result { - vv := &volValue{} - if err = json.Unmarshal(value, vv); err != nil { + var vv *volValue + if vv, err = newVolValueFromBytes(value); err != nil { err = fmt.Errorf("action[loadVols],value:%v,unmarshal err:%v", string(value), err) return err } - vol := newVol(vv.ID, vv.Name, vv.Owner, vv.DataPartitionSize, vv.Capacity, vv.DpReplicaNum, vv.ReplicaNum, vv.FollowerRead, vv.Authenticate) + vol := newVolFromVolValue(vv) vol.Status = vv.Status c.putVol(vol) log.LogInfof("action[loadVols],vol[%v]", vol.Name) diff --git a/master/server.go b/master/server.go index 5fd5f871f..8faa6300a 100644 --- a/master/server.go +++ b/master/server.go @@ -101,7 +101,10 @@ func (m *Server) Start(cfg *config.Config) (err error) { log.LogError(err) return } - m.rocksDBStore = raftstore.NewRocksDBStore(m.storeDir, LRUCacheSize, WriteBufferSize) + if m.rocksDBStore, err = raftstore.NewRocksDBStore(m.storeDir, LRUCacheSize, WriteBufferSize); err != nil { + return + } + if err = m.createRaftServer(); err != nil { log.LogError(errors.Stack(err)) return diff --git a/master/vol.go b/master/vol.go index 8c6130bfa..e24594d3d 100644 --- a/master/vol.go +++ b/master/vol.go @@ -29,6 +29,8 @@ type Vol struct { ID uint64 Name string Owner string + OSSAccessKey string + OSSSecretKey string dpReplicaNum uint8 mpReplicaNum uint8 Status uint8 @@ -76,6 +78,29 @@ func newVol(id uint64, name, owner string, dpSize, capacity uint64, dpReplicaNum return } +func newVolFromVolValue(vv *volValue) (vol *Vol) { + vol = newVol( + vv.ID, + vv.Name, + vv.Owner, + vv.DataPartitionSize, + vv.Capacity, + vv.DpReplicaNum, + vv.ReplicaNum, + vv.FollowerRead, + vv.Authenticate) + // overwrite oss secure + vol.OSSAccessKey, vol.OSSSecretKey = vv.OSSAccessKey, vv.OSSSecretKey + vol.Status = vv.Status + return vol +} + +func (vol *Vol) refreshOSSSecure() (key, secret string) { + vol.OSSAccessKey = util.RandomString(16, util.Numeric|util.LowerLetter|util.UpperLetter) + vol.OSSSecretKey = util.RandomString(32, util.Numeric|util.LowerLetter|util.UpperLetter) + return vol.OSSAccessKey, vol.OSSSecretKey +} + func (vol *Vol) addMetaPartition(mp *MetaPartition) { vol.mpsLock.Lock() defer vol.mpsLock.Unlock() @@ -372,6 +397,8 @@ func (vol *Vol) totalUsedSpace() uint64 { func (vol *Vol) updateViewCache(c *Cluster) { view := proto.NewVolView(vol.Name, vol.Status, vol.FollowerRead) + view.SetOwner(vol.Owner) + view.SetOSSSecure(vol.OSSAccessKey, vol.OSSSecretKey) mpViews := vol.getMetaPartitionsView() view.MetaPartitions = mpViews mpViewsReply := newSuccessHTTPReply(mpViews) diff --git a/master/vol_test.go b/master/vol_test.go index 33df1a21a..e0a087835 100644 --- a/master/vol_test.go +++ b/master/vol_test.go @@ -219,8 +219,12 @@ func TestConcurrentReadWriteDataPartitionMap(t *testing.T) { mp2.Status = proto.ReadOnly vol.addMetaPartition(mp2) vol.updateViewCache(server.cluster) + for id := 0; id < 30000; id++ { + dp := newDataPartition(uint64(id), 3, name, volID) + vol.dataPartitions.put(dp) + } go func() { - var id uint64 + var id uint64 = 30000 for { id++ dp := newDataPartition(id, 3, name, volID) diff --git a/metanode/api_handler.go b/metanode/api_handler.go index ba1e09d1d..2633634e0 100644 --- a/metanode/api_handler.go +++ b/metanode/api_handler.go @@ -29,7 +29,7 @@ import ( type APIResponse struct { Code int `json:"code"` Msg string `json:"msg"` - Data interface{} `json:"data, omitempty"` + Data interface{} `json:"data,omitempty"` } // NewAPIResponse returns a new API response. diff --git a/metanode/const.go b/metanode/const.go index 4ae8ef99d..383dccd0a 100644 --- a/metanode/const.go +++ b/metanode/const.go @@ -77,7 +77,7 @@ type ( // Client -> MetaNode EvictInodeReq = proto.EvictInodeRequest - // Client -> MetaNOde + // Client -> MetaNode SetattrRequest = proto.SetAttrRequest ) @@ -102,6 +102,11 @@ const ( opFSMInternalDelExtentFile opFSMInternalDelExtentCursor opExtentFileSnapshot + opFSMSetXAttr + opFSMRemoveXAttr + opFSMCreateMultipart + opFSMRemoveMultipart + opFSMAppendMultipart ) var ( diff --git a/metanode/extend.go b/metanode/extend.go new file mode 100644 index 000000000..c677048a8 --- /dev/null +++ b/metanode/extend.go @@ -0,0 +1,169 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package metanode + +import ( + "bytes" + "encoding/binary" + "sync" + + "github.com/chubaofs/chubaofs/util/btree" +) + +type Extend struct { + inode uint64 + dataMap map[string][]byte + mu sync.RWMutex +} + +func NewExtend(inode uint64) *Extend { + return &Extend{inode: inode, dataMap: make(map[string][]byte)} +} + +func NewExtendFromBytes(raw []byte) (*Extend, error) { + var err error + var buffer = bytes.NewBuffer(raw) + // decode inode + var inode uint64 + if inode, err = binary.ReadUvarint(buffer); err != nil { + return nil, err + } + var ext = NewExtend(inode) + // decode number of key-value pairs + var numKV uint64 + if numKV, err = binary.ReadUvarint(buffer); err != nil { + return nil, err + } + var readBytes = func() ([]byte, error) { + var length uint64 + if length, err = binary.ReadUvarint(buffer); err != nil { + return nil, err + } + var data = make([]byte, length) + if _, err = buffer.Read(data); err != nil { + return nil, err + } + return data, nil + } + for i := 0; i < int(numKV); i++ { + var k, v []byte + if k, err = readBytes(); err != nil { + return nil, err + } + if v, err = readBytes(); err != nil { + return nil, err + } + ext.Put(k, v) + } + return ext, nil +} + +func (e *Extend) Less(than btree.Item) bool { + ext, is := than.(*Extend) + return is && e.inode < ext.inode +} + +func (e *Extend) Put(key, value []byte) { + e.mu.Lock() + defer e.mu.Unlock() + e.dataMap[string(key)] = value +} + +func (e *Extend) Get(key []byte) (value []byte, exist bool) { + e.mu.RLock() + defer e.mu.RUnlock() + value, exist = e.dataMap[string(key)] + return +} + +func (e *Extend) Remove(key []byte) { + e.mu.Lock() + defer e.mu.Unlock() + delete(e.dataMap, string(key)) + return +} + +func (e *Extend) Range(visitor func(key, value []byte) bool) { + e.mu.RLock() + defer e.mu.RUnlock() + for k, v := range e.dataMap { + if !visitor([]byte(k), v) { + return + } + } +} + +func (e *Extend) Merge(o *Extend, override bool) { + e.mu.Lock() + defer e.mu.Unlock() + o.Range(func(key, value []byte) bool { + strKey := string(key) + if _, exist := e.dataMap[strKey]; override || !exist { + copied := make([]byte, len(value)) + copy(copied, value) + e.dataMap[strKey] = copied + } + return true + }) +} + +func (e *Extend) Copy() btree.Item { + newExt := NewExtend(e.inode) + for k, v := range e.dataMap { + newExt.dataMap[k] = v + } + return newExt +} + +func (e *Extend) Bytes() ([]byte, error) { + var err error + e.mu.RLock() + defer e.mu.RUnlock() + var n int + var tmp = make([]byte, binary.MaxVarintLen64) + var buffer = bytes.NewBuffer(nil) + // write inode with varint codec + n = binary.PutUvarint(tmp, e.inode) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + // write number of key-value pairs + n = binary.PutUvarint(tmp, uint64(len(e.dataMap))) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + // write key-value paris + var writeBytes = func(val []byte) error { + n = binary.PutUvarint(tmp, uint64(len(val))) + if _, err = buffer.Write(tmp[:n]); err != nil { + return err + } + if _, err = buffer.Write(val); err != nil { + return err + } + return nil + } + for k, v := range e.dataMap { + // key + if err = writeBytes([]byte(k)); err != nil { + return nil, err + } + // value + if err = writeBytes(v); err != nil { + return nil, err + } + } + return buffer.Bytes(), nil +} diff --git a/metanode/extend_test.go b/metanode/extend_test.go new file mode 100644 index 000000000..320a2c300 --- /dev/null +++ b/metanode/extend_test.go @@ -0,0 +1,57 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package metanode + +import ( + "math/rand" + "reflect" + "testing" + "time" + + "github.com/chubaofs/chubaofs/util" +) + +func TestExtend_Bytes(t *testing.T) { + var err error + const numSamples = 100 + + var random = rand.New(rand.NewSource(time.Now().UnixNano())) + + extends := make([]*Extend, numSamples) + for i := 0; i < numSamples; i++ { + extend := NewExtend(random.Uint64()) + extend.Put([]byte("msg"), []byte(util.RandomString(16, util.Numeric|util.LowerLetter|util.UpperLetter))) + extends[i] = extend + } + + outputs := make([][]byte, numSamples) + for i := 0; i < numSamples; i++ { + if outputs[i], err = extends[i].Bytes(); err != nil { + t.Fatalf("encode extend to bytes fail cause: %v", err) + } + } + + // validate result + for i := 0; i < numSamples; i++ { + var e *Extend + if e, err = NewExtendFromBytes(outputs[i]); err != nil { + t.Fatalf("decode bytes to extend fail cause: %v", err) + } + if !reflect.DeepEqual(e, extends[i]) { + t.Fatalf("result mismatch") + } + } + +} diff --git a/metanode/manager.go b/metanode/manager.go index a9378fbc0..d8e1192b7 100644 --- a/metanode/manager.go +++ b/metanode/manager.go @@ -122,6 +122,32 @@ func (m *metadataManager) HandleMetadataOperation(conn net.Conn, p *Packet, err = m.opMetaPartitionTryToLeader(conn, p, remoteAddr) case proto.OpMetaBatchInodeGet: err = m.opMetaBatchInodeGet(conn, p, remoteAddr) + case proto.OpMetaDeleteInode: + err = m.opMetaDeleteInode(conn, p, remoteAddr) + case proto.OpMetaBatchExtentsAdd: + err = m.opMetaBatchExtentsAdd(conn, p, remoteAddr) + // operations for extend attributes + case proto.OpMetaSetXAttr: + err = m.opMetaSetXAttr(conn, p, remoteAddr) + case proto.OpMetaGetXAttr: + err = m.opMetaGetXAttr(conn, p, remoteAddr) + case proto.OpMetaBatchGetXAttr: + err = m.opMetaBatchGetXAttr(conn, p, remoteAddr) + case proto.OpMetaRemoveXAttr: + err = m.opMetaRemoveXAttr(conn, p, remoteAddr) + case proto.OpMetaListXAttr: + err = m.opMetaListXAttr(conn, p, remoteAddr) + // operations for multipart session + case proto.OpCreateMultipart: + err = m.opCreateMultipart(conn, p, remoteAddr) + case proto.OpListMultiparts: + err = m.opListMultipart(conn, p, remoteAddr) + case proto.OpRemoveMultipart: + err = m.opRemoveMultipart(conn, p, remoteAddr) + case proto.OpAddMultipartPart: + err = m.opAppendMultipart(conn, p, remoteAddr) + case proto.OpGetMultipart: + err = m.opGetMultipart(conn, p, remoteAddr) default: err = fmt.Errorf("%s unknown Opcode: %d, reqId: %d", remoteAddr, p.Opcode, p.GetReqID()) diff --git a/metanode/manager_op.go b/metanode/manager_op.go index e359c8539..a70aeee9b 100644 --- a/metanode/manager_op.go +++ b/metanode/manager_op.go @@ -117,7 +117,7 @@ func (m *metadataManager) opCreateMetaPartition(conn net.Conn, p *Packet, " struct: %s", err.Error()) return } - log.LogDebugf("%s [opCreateMetaPartition] [remoteAddr=%s]accept a from"+ + log.LogDebugf("[opCreateMetaPartition] [remoteAddr=%s]accept a from"+ " master message: %v", remoteAddr, adminTask) // create a new meta partition. if err = m.createPartition(req.PartitionID, req.VolName, @@ -363,7 +363,7 @@ func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet, if err != nil { p.PacketErrorWithBody(proto.OpNotExistErr, nil) m.respondToClient(conn, p) - err = errors.NewErrorf("[opMetaEvictInode] req: %s, resp: %v", req, err.Error()) + err = errors.NewErrorf("[opMetaEvictInode] req: %v, resp: %v", req, err.Error()) return } if !m.serveProxy(conn, mp, p) { @@ -371,7 +371,7 @@ func (m *metadataManager) opMetaEvictInode(conn net.Conn, p *Packet, } if err = mp.EvictInode(req, p); err != nil { - err = errors.NewErrorf("[opMetaEvictInode] req: %s, resp: %v", req, err.Error()) + err = errors.NewErrorf("[opMetaEvictInode] req: %v, resp: %v", req, err.Error()) } m.respondToClient(conn, p) log.LogDebugf("%s [opMetaEvictInode] req: %d - %v, resp: %v, body: %s", @@ -838,3 +838,255 @@ errDeal: m.respondToClient(conn, p) return } + +func (m *metadataManager) opMetaDeleteInode(conn net.Conn, p *Packet, + remoteAddr string) (err error) { + req := &proto.DeleteInodeRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.DeleteInode(req, p) + _ = m.respondToClient(conn, p) + log.LogDebugf("%s [opMetaDeleteInode] req: %d - %v, resp: %v, body: %s", + remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data) + return +} + +func (m *metadataManager) opMetaSetXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) { + req := &proto.SetXAttrRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.SetXAttr(req, p) + _ = m.respondToClient(conn, p) + log.LogDebugf("%s [opMetaSetXAttr] req: %d - %v, resp: %v, body: %s", + remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data) + return +} + +func (m *metadataManager) opMetaGetXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) { + req := &proto.GetXAttrRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.GetXAttr(req, p) + _ = m.respondToClient(conn, p) + log.LogDebugf("%s [opMetaGetXAttr] req: %d - %v, resp: %v, body: %s", + remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data) + return +} + +func (m *metadataManager) opMetaBatchGetXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) { + req := &proto.BatchGetXAttrRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.BatchGetXAttr(req, p) + _ = m.respondToClient(conn, p) + log.LogDebugf("%s [opMetaBatchGetXAttr req: %d - %v, resp: %v, body: %s", + remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data) + return +} + +func (m *metadataManager) opMetaRemoveXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) { + req := &proto.RemoveXAttrRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.RemoveXAttr(req, p) + _ = m.respondToClient(conn, p) + log.LogDebugf("%s [opMetaGetXAttr] req: %d - %v, resp: %v, body: %s", + remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data) + return +} + +func (m *metadataManager) opMetaListXAttr(conn net.Conn, p *Packet, remoteAddr string) (err error) { + req := &proto.ListXAttrRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.ListXAttr(req, p) + _ = m.respondToClient(conn, p) + log.LogDebugf("%s [opMetaGetXAttr] req: %d - %v, resp: %v, body: %s", + remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data) + return +} + +func (m *metadataManager) opMetaBatchExtentsAdd(conn net.Conn, p *Packet, remoteAddr string) (err error) { + req := &proto.AppendExtentKeysRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpNotExistErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.BatchExtentAppend(req, p) + _ = m.respondToClient(conn, p) + log.LogDebugf("%s [opMetaBatchExtentsAdd] req: %d - %v, resp: %v, body: %s", + remoteAddr, p.GetReqID(), req, p.GetResultMsg(), p.Data) + return +} + +func (m *metadataManager) opCreateMultipart(conn net.Conn, p *Packet, remote string) (err error) { + req := &proto.CreateMultipartRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.CreateMultipart(req, p) + _ = m.respondToClient(conn, p) + return +} + +func (m *metadataManager) opRemoveMultipart(conn net.Conn, p *Packet, remote string) (err error) { + req := &proto.RemoveMultipartRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.RemoveMultipart(req, p) + _ = m.respondToClient(conn, p) + return +} + +func (m *metadataManager) opGetMultipart(conn net.Conn, p *Packet, remote string) (err error) { + req := &proto.GetMultipartRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + _ = m.respondToClient(conn, p) + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.GetMultipart(req, p) + _ = m.respondToClient(conn, p) + return +} + +func (m *metadataManager) opAppendMultipart(conn net.Conn, p *Packet, remote string) (err error) { + defer func() { + if err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + } + _ = m.respondToClient(conn, p) + }() + req := &proto.AddMultipartPartRequest{} + if err = json.Unmarshal(p.Data, req); err != nil { + return + } + mp, err := m.getPartition(req.PartitionId) + if err != nil { + return + } + if !m.serveProxy(conn, mp, p) { + return + } + err = mp.AppendMultipart(req, p) + return +} + +func (m *metadataManager) opListMultipart(conn net.Conn, p *Packet, remote string) (err error) { + // TODO: implement method 'opListMultipart' + return +} diff --git a/metanode/manager_resp.go b/metanode/manager_resp.go index 8897301f9..2b2704284 100644 --- a/metanode/manager_resp.go +++ b/metanode/manager_resp.go @@ -15,20 +15,15 @@ package metanode import ( - "encoding/json" + "github.com/chubaofs/chubaofs/proto" "net" "github.com/chubaofs/chubaofs/util/errors" "github.com/chubaofs/chubaofs/util/log" ) -const ( - masterResponsePath = "/metaNode/response" // Method: 'POST', - // ContentType: 'application/json' -) - // Reply operation results to the master. -func (m *metadataManager) respondToMaster(data interface{}) (err error) { +func (m *metadataManager) respondToMaster(task *proto.AdminTask) (err error) { // handle panic defer func() { if r := recover(); r != nil { @@ -40,14 +35,7 @@ func (m *metadataManager) respondToMaster(data interface{}) (err error) { } } }() - - // process data and send reply though http specified remote address. - jsonBytes, err := json.Marshal(data) - if err != nil { - return - } - _, err = masterHelper.Request("POST", masterResponsePath, nil, jsonBytes) - if err != nil { + if err = masterClient.NodeAPI().ResponseMetaNodeTask(task); err != nil { err = errors.Trace(err, "try respondToMaster failed") } return diff --git a/metanode/metanode.go b/metanode/metanode.go index 0d04592c8..d1339e672 100644 --- a/metanode/metanode.go +++ b/metanode/metanode.go @@ -15,7 +15,8 @@ package metanode import ( - "encoding/json" + "github.com/chubaofs/chubaofs/master" + masterSDK "github.com/chubaofs/chubaofs/sdk/master" "os" "strings" "sync" @@ -23,7 +24,6 @@ import ( "time" "fmt" - "net/http" "strconv" "github.com/chubaofs/chubaofs/proto" @@ -37,7 +37,7 @@ import ( var ( clusterInfo *proto.ClusterInfo - masterHelper util.MasterHelper + masterClient *masterSDK.MasterClient configTotalMem uint64 ) @@ -97,30 +97,20 @@ type MetaNodeInfo struct { } func (m *MetaNode) checkLocalPartitionMatchWithMaster() (err error) { - params := make(map[string]string) - params["addr"] = m.localAddr + ":" + m.listen - var data interface{} + var metaNodeInfo *master.MetaNode for i := 0; i < 3; i++ { - data, err = masterHelper.Request(http.MethodGet, proto.GetMetaNode, params, nil) - if err != nil { - log.LogErrorf("checkLocalPartitionMatchWithMaster error %v", err) + if metaNodeInfo, err = masterClient.NodeAPI().GetMetaNode(fmt.Sprintf("%s:%s", m.localAddr, m.listen)); err != nil { + log.LogErrorf("checkLocalPartitionMatchWithMaster: get MetaNode info fail: err(%v)", err) continue } break } - minfo := new(MetaNodeInfo) - if err = json.Unmarshal(data.([]byte), minfo); err != nil { - err = fmt.Errorf("checkLocalPartitionMatchWithMaster jsonUnmarsh failed %v", err) - log.LogErrorf(err.Error()) - return - } - - if len(minfo.PersistenceMetaPartitions) == 0 { + if len(metaNodeInfo.PersistenceMetaPartitions) == 0 { return } lackPartitions := make([]uint64, 0) - for _, partitionID := range minfo.PersistenceMetaPartitions { + for _, partitionID := range metaNodeInfo.PersistenceMetaPartitions { _, err := m.metadataManager.GetPartition(partitionID) if err != nil { lackPartitions = append(lackPartitions, partitionID) @@ -226,14 +216,12 @@ func (m *MetaNode) parseConfig(cfg *config.Config) (err error) { log.LogInfof("[parseConfig] load raftHeartbeatPort[%v].", m.raftHeartbeatPort) log.LogInfof("[parseConfig] load raftReplicatePort[%v].", m.raftReplicatePort) - addrs := cfg.GetArray(cfgMasterAddr) - if len(addrs) == 0 { - addrs = cfg.GetArray(cfgMasterAddrs) - } - masterHelper = util.NewMasterHelper() + addrs := cfg.GetArray(cfgMasterAddrs) + masters := make([]string, 0, len(addrs)) for _, addr := range addrs { - masterHelper.AddNode(addr.(string)) + masters = append(masters, addr.(string)) } + masterClient = masterSDK.NewMasterClient(masters, false) err = m.validConfig() return } @@ -249,7 +237,7 @@ func (m *MetaNode) validConfig() (err error) { if m.raftDir == "" { m.raftDir = defaultRaftDir } - if len(masterHelper.Nodes()) == 0 { + if len(masterClient.Nodes()) == 0 { err = errors.New("master address list is empty") return } @@ -283,10 +271,10 @@ func (m *MetaNode) stopMetaManager() { func (m *MetaNode) register() (err error) { step := 0 - reqParam := make(map[string]string) + var nodeAddress string for { if step < 1 { - clusterInfo, err = getClusterInfo() + clusterInfo, err = getClientIP() if err != nil { log.LogErrorf("[register] %s", err.Error()) continue @@ -295,28 +283,16 @@ func (m *MetaNode) register() (err error) { m.localAddr = clusterInfo.Ip } m.clusterId = clusterInfo.Cluster - reqParam["addr"] = m.localAddr + ":" + m.listen + nodeAddress = m.localAddr + ":" + m.listen step++ } - var respBody []byte - respBody, err = masterHelper.Request("POST", proto.AddMetaNode, reqParam, nil) - if err != nil { - log.LogErrorf("[register] %s", err.Error()) - time.Sleep(3 * time.Second) - continue - } - nodeIDStr := strings.TrimSpace(string(respBody)) - if nodeIDStr == "" { - log.LogErrorf("[register] master respond empty body") - time.Sleep(3 * time.Second) - continue - } - m.nodeId, err = strconv.ParseUint(nodeIDStr, 10, 64) - if err != nil { - log.LogErrorf("[register] parse to nodeID: %s", err.Error()) + var nodeID uint64 + if nodeID, err = masterClient.NodeAPI().AddMetaNode(nodeAddress); err != nil { + log.LogErrorf("register: register to master fail: address(%v) err(%s)", nodeAddress, err) time.Sleep(3 * time.Second) continue } + m.nodeId = nodeID return } } @@ -326,14 +302,7 @@ func NewServer() *MetaNode { return &MetaNode{} } -func getClusterInfo() (*proto.ClusterInfo, error) { - respBody, err := masterHelper.Request("GET", proto.AdminGetIP, nil, nil) - if err != nil { - return nil, err - } - cInfo := &proto.ClusterInfo{} - if err = json.Unmarshal(respBody, cInfo); err != nil { - return nil, err - } - return cInfo, nil +func getClientIP() (ci *proto.ClusterInfo, err error) { + ci, err = masterClient.AdminAPI().GetClusterInfo() + return } diff --git a/metanode/multipart.go b/metanode/multipart.go new file mode 100644 index 000000000..549fdf179 --- /dev/null +++ b/metanode/multipart.go @@ -0,0 +1,337 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package metanode + +import ( + "bytes" + "encoding/binary" + "github.com/chubaofs/chubaofs/util/btree" + "sort" + "sync" + "time" +) + +// Part defined necessary fields for multipart part management. +type Part struct { + ID uint16 + UploadTime time.Time + MD5 string + Size uint64 + Inode uint64 +} + +func (m Part) Bytes() ([]byte, error) { + var err error + var buffer = bytes.NewBuffer(nil) + var tmp = make([]byte, binary.MaxVarintLen64) + var n int + // ID + n = binary.PutUvarint(tmp, uint64(m.ID)) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + // upload time + n = binary.PutVarint(tmp, m.UploadTime.UnixNano()) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + // MD5 + n = binary.PutUvarint(tmp, uint64(len(m.MD5))) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + if _, err = buffer.WriteString(m.MD5); err != nil { + return nil, err + } + // size + n = binary.PutUvarint(tmp, m.Size) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + // inode + n = binary.PutUvarint(tmp, m.Inode) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + return buffer.Bytes(), nil +} + +func PartFromBytes(raw []byte) *Part { + var offset, n int + // decode ID + var u64ID uint64 + u64ID, n = binary.Uvarint(raw) + offset += n + // decode upload time + var uploadTimeI64 int64 + uploadTimeI64, n = binary.Varint(raw[offset:]) + offset += n + // decode MD5 + var md5Len uint64 + md5Len, n = binary.Uvarint(raw[offset:]) + offset += n + var md5Content = string(raw[offset : offset+int(md5Len)]) + offset += int(md5Len) + // decode size + var sizeU64 uint64 + sizeU64, n = binary.Uvarint(raw[offset:]) + offset += n + // decode inode + var inode uint64 + inode, n = binary.Uvarint(raw[offset:]) + + var muPart = &Part{ + ID: uint16(u64ID), + UploadTime: time.Unix(0, uploadTimeI64), + MD5: md5Content, + Size: sizeU64, + Inode: inode, + } + return muPart +} + +type Parts []*Part + +func (m Parts) Len() int { + return len(m) +} + +func (m Parts) Less(i, j int) bool { + return m[i].ID < m[j].ID +} + +func (m Parts) Swap(i, j int) { + m[i], m[j] = m[j], m[i] +} + +func (m Parts) Sort() { + sort.Sort(m) +} + +func (m *Parts) Hash(part *Part) (has bool) { + i := sort.Search(len(*m), func(i int) bool { + return (*m)[i].ID >= part.ID + }) + has = i < len(*m) && (*m)[i].ID == part.ID + return +} + +func (m *Parts) Insert(part *Part, replace bool) (success bool) { + i := sort.Search(len(*m), func(i int) bool { + return (*m)[i].ID >= part.ID + }) + if i < len(*m) && (*m)[i].ID == part.ID { + if replace { + (*m)[i] = part + return true + } + return false + } + *m = append(*m, part) + m.Sort() + return true +} + +func (m *Parts) Remove(id uint16) { + i := sort.Search(len(*m), func(i int) bool { + return (*m)[i].ID >= id + }) + if i < len(*m) && (*m)[i].ID == id { + if len(*m) > i+1 { + *m = append((*m)[:i], (*m)[i+1:]...) + } else { + *m = (*m)[:i] + } + } +} + +func (m Parts) Search(id uint16) (part *Part, found bool) { + i := sort.Search(len(m), func(i int) bool { + return m[i].ID >= id + }) + if i < len(m) && m[i].ID == id { + return m[i], true + } + return nil, false +} + +func (m Parts) Bytes() ([]byte, error) { + var err error + var n int + var buffer = bytes.NewBuffer(nil) + var tmp = make([]byte, binary.MaxVarintLen64) + n = binary.PutUvarint(tmp, uint64(len(m))) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + var marshaled []byte + for _, p := range m { + marshaled, err = p.Bytes() + if err != nil { + return nil, err + } + // write part length + n = binary.PutUvarint(tmp, uint64(len(marshaled))) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + // write part bytes + if _, err = buffer.Write(marshaled); err != nil { + return nil, err + } + } + return buffer.Bytes(), nil +} + +func PartsFromBytes(raw []byte) Parts { + var offset, n int + var numPartsU64 uint64 + numPartsU64, n = binary.Uvarint(raw) + offset += n + var muParts = make([]*Part, int(numPartsU64)) + for i := 0; i < int(numPartsU64); i++ { + var partLengthU64 uint64 + partLengthU64, n = binary.Uvarint(raw[offset:]) + offset += n + part := PartFromBytes(raw[offset : offset+int(partLengthU64)]) + muParts[i] = part + offset += int(partLengthU64) + } + return muParts +} + +// Multipart defined necessary fields for multipart session management. +type Multipart struct { + // session fields + id string + key string + initTime time.Time + parts Parts + + mu sync.RWMutex +} + +func (m *Multipart) Less(than btree.Item) bool { + thanMultipart, is := than.(*Multipart) + return is && m.id < thanMultipart.id +} + +func (m *Multipart) Copy() btree.Item { + return &Multipart{ + id: m.id, + key: m.key, + initTime: m.initTime, + parts: append(Parts{}, m.parts...), + } +} + +func (m *Multipart) ID() string { + return m.id +} + +func (m *Multipart) InsertPart(part *Part, replace bool) (success bool) { + m.mu.Lock() + defer m.mu.Unlock() + if m.parts == nil { + m.parts = PartsFromBytes(nil) + } + success = m.parts.Insert(part, replace) + return +} + +func (m *Multipart) Parts() []*Part { + m.mu.RLock() + defer m.mu.RUnlock() + return append([]*Part{}, m.parts...) +} + +func (m *Multipart) Bytes() ([]byte, error) { + var n int + var buffer = bytes.NewBuffer(nil) + var err error + tmp := make([]byte, binary.MaxVarintLen64) + // marshal id + var marshalStr = func(src string) error { + n = binary.PutUvarint(tmp, uint64(len(src))) + if _, err = buffer.Write(tmp[:n]); err != nil { + return err + } + if _, err = buffer.WriteString(src); err != nil { + return err + } + return nil + } + // marshal id + if err = marshalStr(m.id); err != nil { + return nil, err + } + // marshal key + if err = marshalStr(m.key); err != nil { + return nil, err + } + // marshal init time + n = binary.PutVarint(tmp, m.initTime.UnixNano()) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + // marshal parts + var marshaledParts []byte + if marshaledParts, err = m.parts.Bytes(); err != nil { + return nil, err + } + n = binary.PutUvarint(tmp, uint64(len(marshaledParts))) + if _, err = buffer.Write(tmp[:n]); err != nil { + return nil, err + } + if _, err = buffer.Write(marshaledParts); err != nil { + return nil, err + } + return buffer.Bytes(), nil +} + +func MultipartFromBytes(raw []byte) *Multipart { + var unmarshalStr = func(data []byte) (string, int) { + var n int + var lengthU64 uint64 + lengthU64, n = binary.Uvarint(data) + return string(data[n : n+int(lengthU64)]), n + int(lengthU64) + } + var offset, n int + // decode id + var id string + id, n = unmarshalStr(raw) + offset += n + // decode key + var key string + key, n = unmarshalStr(raw[offset:]) + offset += n + // decode init time + var initTimeI64 int64 + initTimeI64, n = binary.Varint(raw[offset:]) + offset += n + // decode parts + var partsLengthU64 uint64 + partsLengthU64, n = binary.Uvarint(raw[offset:]) + offset += n + var parts = PartsFromBytes(raw[offset : offset+int(partsLengthU64)]) + + var muSession = &Multipart{ + id: id, + key: key, + initTime: time.Unix(0, initTimeI64), + parts: parts, + } + return muSession +} diff --git a/metanode/multipart_test.go b/metanode/multipart_test.go new file mode 100644 index 000000000..133dc9189 --- /dev/null +++ b/metanode/multipart_test.go @@ -0,0 +1,151 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package metanode + +import ( + "math/rand" + "reflect" + "testing" + "time" + + "github.com/chubaofs/chubaofs/util" +) + +func TestMUPart_Bytes(t *testing.T) { + var ( + id uint16 = 1 + uploadTime = time.Now().Local() + md5 = util.RandomString(16, util.UpperLetter|util.Numeric) + size uint64 = 65536 + inode uint64 = 12345 + err error + ) + part1 := &Part{ + ID: id, + UploadTime: uploadTime, + MD5: md5, + Size: size, + Inode: inode, + } + var partBytes []byte + if partBytes, err = part1.Bytes(); err != nil { + t.Fatalf("get bytes of part fail cause: %v", err) + } + part2 := PartFromBytes(partBytes) + if !reflect.DeepEqual(part1, part2) { + t.Fatalf("result mismatch:\n\tpart1:%v\n\tpart2:%v", part1, part2) + } + t.Logf("encoded length: %v", len(partBytes)) +} + +func TestMUParts_Bytes(t *testing.T) { + var err error + var random = rand.New(rand.NewSource(time.Now().UnixNano())) + var parts1 = PartsFromBytes(nil) + for i := 0; i < 100; i++ { + part := &Part{ + ID: uint16(i), + UploadTime: time.Now().Local(), + MD5: util.RandomString(16, util.UpperLetter|util.Numeric), + Size: random.Uint64(), + Inode: random.Uint64(), + } + parts1.Insert(part, false) + } + var partsBytes []byte + partsBytes, err = parts1.Bytes() + if partsBytes, err = parts1.Bytes(); err != nil { + t.Fatalf("get bytes of part fail cause: %v", err) + } + parts2 := PartsFromBytes(partsBytes) + if !reflect.DeepEqual(parts1, parts2) { + t.Fatalf("result mismatch:\n\tpart1:%v\n\tpart2:%v", parts1, parts2) + } + t.Logf("encoded length: %v", len(partsBytes)) +} + +func TestMUParts_Modify(t *testing.T) { + var random = rand.New(rand.NewSource(time.Now().UnixNano())) + var parts = PartsFromBytes(nil) + for i := 0; i < 100; i++ { + part := &Part{ + ID: uint16(i), + UploadTime: time.Now().Local(), + MD5: util.RandomString(16, util.UpperLetter|util.Numeric), + Size: random.Uint64(), + Inode: random.Uint64(), + } + parts.Insert(part, false) + } + if parts.Len() != 100 { + t.Fatalf("parts length mismatch: except 100 actual %v", parts.Len()) + } + // validate before modify + if _, found := parts.Search(0); !found { + t.Fatalf("part id[0] not found before modify") + } + if _, found := parts.Search(50); !found { + t.Fatalf("part id[50] not found before modify") + } + if _, found := parts.Search(99); !found { + t.Fatalf("part id[99] not found before modify") + } + // modify + parts.Remove(0) + parts.Remove(50) + parts.Remove(99) + if parts.Len() != 97 { + t.Fatalf("parts length mismatch: expect 97 actual %v", parts.Len()) + } + // validate after modify + if _, found := parts.Search(0); found { + t.Fatalf("part id[0] not found before modify") + } + if _, found := parts.Search(50); found { + t.Fatalf("part id[50] not found before modify") + } + if _, found := parts.Search(99); found { + t.Fatalf("part id[99] not found before modify") + } +} + +func TestMUSession_Bytes(t *testing.T) { + var err error + var random = rand.New(rand.NewSource(time.Now().UnixNano())) + var session1 = MultipartFromBytes(nil) + for i := 0; i < 100; i++ { + id := uint16(i) + md5 := util.RandomString(16, util.UpperLetter|util.Numeric) + size := random.Uint64() + inode := random.Uint64() + session1.InsertPart(&Part{ + ID: id, + MD5: md5, + Size: size, + Inode: inode, + UploadTime: time.Now().Local(), + }, false) + } + var sessionBytes []byte + sessionBytes, err = session1.Bytes() + if err != nil { + t.Fatalf("encode session to bytes fail caue: %v", err) + } + session2 := MultipartFromBytes(sessionBytes) + if !reflect.DeepEqual(session1, session2) { + t.Fatalf("result mismatch:\n\tsession1:%v\n\tsession2:%v", session1, session2) + } + t.Logf("encoded session length: %v", len(sessionBytes)) +} diff --git a/metanode/partition.go b/metanode/partition.go index b2d220396..8441c714f 100644 --- a/metanode/partition.go +++ b/metanode/partition.go @@ -15,6 +15,7 @@ package metanode import ( + "bytes" "encoding/json" "sort" "strconv" @@ -113,6 +114,15 @@ type OpInode interface { EvictInode(req *EvictInodeReq, p *Packet) (err error) SetAttr(reqData []byte, p *Packet) (err error) GetInodeTree() *BTree + DeleteInode(req *proto.DeleteInodeRequest, p *Packet) (err error) +} + +type OpExtend interface { + SetXAttr(req *proto.SetXAttrRequest, p *Packet) (err error) + GetXAttr(req *proto.GetXAttrRequest, p *Packet) (err error) + BatchGetXAttr(req *proto.BatchGetXAttrRequest, p *Packet) (err error) + RemoveXAttr(req *proto.RemoveXAttrRequest, p *Packet) (err error) + ListXAttr(req *proto.ListXAttrRequest, p *Packet) (err error) } // OpDentry defines the interface for the dentry operations. @@ -130,6 +140,14 @@ type OpExtent interface { ExtentAppend(req *proto.AppendExtentKeyRequest, p *Packet) (err error) ExtentsList(req *proto.GetExtentsRequest, p *Packet) (err error) ExtentsTruncate(req *ExtentsTruncateReq, p *Packet) (err error) + BatchExtentAppend(req *proto.AppendExtentKeysRequest, p *Packet) (err error) +} + +type OpMultipart interface { + GetMultipart(req *proto.GetMultipartRequest, p *Packet) (err error) + CreateMultipart(req *proto.CreateMultipartRequest, p *Packet) (err error) + AppendMultipart(req *proto.AddMultipartPartRequest, p *Packet) (err error) + RemoveMultipart(req *proto.RemoveMultipartRequest, p *Packet) (err error) } // OpMeta defines the interface for the metadata operations. @@ -138,6 +156,8 @@ type OpMeta interface { OpDentry OpExtent OpPartition + OpExtend + OpMultipart } // OpPartition defines the interface for the partition operations. @@ -175,6 +195,8 @@ type metaPartition struct { applyID uint64 // Inode/Dentry max applyID, this index will be update after restoring from the dumped data. dentryTree *BTree inodeTree *BTree // btree for inodes + extendTree *BTree // btree for inode extend (XAttr) management + multipartTree *BTree // collection for multipart management raftPartition raftstore.Partition stopC chan bool storeChan chan *storeMsg @@ -333,16 +355,18 @@ func (mp *metaPartition) getRaftPort() (heartbeat, replica int, err error) { // NewMetaPartition creates a new meta partition with the specified configuration. func NewMetaPartition(conf *MetaPartitionConfig, manager *metadataManager) MetaPartition { mp := &metaPartition{ - config: conf, - dentryTree: NewBtree(), - inodeTree: NewBtree(), - stopC: make(chan bool), - storeChan: make(chan *storeMsg, 5), - freeList: newFreeList(), - extDelCh: make(chan BtreeItem, 10000), - extReset: make(chan struct{}), - vol: NewVol(), - manager: manager, + config: conf, + dentryTree: NewBtree(), + inodeTree: NewBtree(), + extendTree: NewBtree(), + multipartTree: NewBtree(), + stopC: make(chan bool), + storeChan: make(chan *storeMsg, 5), + freeList: newFreeList(), + extDelCh: make(chan BtreeItem, 10000), + extReset: make(chan struct{}), + vol: NewVol(), + manager: manager, } return mp } @@ -393,14 +417,20 @@ func (mp *metaPartition) load() (err error) { if err = mp.loadMetadata(); err != nil { return } - loadSnapshotDir := path.Join(mp.config.RootDir, snapshotDir) - if err = mp.loadInode(loadSnapshotDir); err != nil { + snapshotPath := path.Join(mp.config.RootDir, snapshotDir) + if err = mp.loadInode(snapshotPath); err != nil { return } - if err = mp.loadDentry(loadSnapshotDir); err != nil { + if err = mp.loadDentry(snapshotPath); err != nil { return } - err = mp.loadApplyID(loadSnapshotDir) + if err = mp.loadExtend(snapshotPath); err != nil { + return + } + if err = mp.loadMultipart(snapshotPath); err != nil { + return + } + err = mp.loadApplyID(snapshotPath) return } @@ -421,21 +451,28 @@ func (mp *metaPartition) store(sm *storeMsg) (err error) { os.RemoveAll(tmpDir) } }() - var ( - inoCRC, denCRC uint32 - ) - if inoCRC, err = mp.storeInode(tmpDir, sm); err != nil { - return + var crcBuffer = bytes.NewBuffer(make([]byte, 0, 16)) + var storeFuncs = []func(dir string, sm *storeMsg) (uint32, error){ + mp.storeInode, + mp.storeDentry, + mp.storeExtend, + mp.storeMultipart, } - if denCRC, err = mp.storeDentry(tmpDir, sm); err != nil { - return + for _, storeFunc := range storeFuncs { + var crc uint32 + if crc, err = storeFunc(tmpDir, sm); err != nil { + return + } + if crcBuffer.Len() != 0 { + crcBuffer.WriteString(" ") + } + crcBuffer.WriteString(fmt.Sprintf("%d", crc)) } if err = mp.storeApplyID(tmpDir, sm); err != nil { return } // write crc to file - if err = ioutil.WriteFile(path.Join(tmpDir, SnapshotSign), - []byte(fmt.Sprintf("%d %d", inoCRC, denCRC)), 0775); err != nil { + if err = ioutil.WriteFile(path.Join(tmpDir, SnapshotSign), crcBuffer.Bytes(), 0775); err != nil { return } snapshotDir := path.Join(mp.config.RootDir, snapshotDir) @@ -457,7 +494,7 @@ func (mp *metaPartition) store(sm *storeMsg) (err error) { err = nil if err = os.Rename(tmpDir, snapshotDir); err != nil { - os.Rename(backupDir, snapshotDir) + _ = os.Rename(backupDir, snapshotDir) return } err = os.RemoveAll(backupDir) @@ -589,9 +626,15 @@ func (mp *metaPartition) Reset() (err error) { mp.dentryTree.Reset() mp.config.Cursor = 0 mp.applyID = 0 - // delete ino/dentry applyID file - mp.deleteApplyFile() - mp.deleteDentryFile() - mp.deleteInodeFile() + + // remove files + filenames := []string{applyIDFile, dentryFile, inodeFile, extendFile, multipartFile} + for _, filename := range filenames { + filepath := path.Join(mp.config.RootDir, filename) + if err = os.Remove(filepath); err != nil { + return + } + } + return } diff --git a/metanode/partition_free_list.go b/metanode/partition_free_list.go index 4e4375c83..450af4bd5 100644 --- a/metanode/partition_free_list.go +++ b/metanode/partition_free_list.go @@ -15,7 +15,6 @@ package metanode import ( - "encoding/json" "fmt" "github.com/chubaofs/chubaofs/proto" "github.com/chubaofs/chubaofs/util/errors" @@ -52,25 +51,34 @@ func (mp *metaPartition) startFreeList() (err error) { func (mp *metaPartition) updateVolWorker() { t := time.NewTicker(UpdateVolTicket) - reqURL := fmt.Sprintf("%s?name=%s", proto.ClientDataPartitions, mp.config.VolName) + var convert = func(view *proto.DataPartitionsView) *DataPartitionsView { + newView := &DataPartitionsView{ + DataPartitions: make([]*DataPartition, len(view.DataPartitions)), + } + for i:=0; i 0 && marker < multipart.key { + // skip and continue + return true + } + // prefix is enabled + if len(prefix) > 0 && !strings.HasPrefix(multipart.key, prefix) { + // skip and continue + return true + } + matches = append(matches, multipart) + return !(len(matches) >= max) + } + if len(multipartIdMarker) > 0 { + mp.multipartTree.AscendGreaterOrEqual(&Multipart{id: multipartIdMarker}, walkTreeFunc) + } else { + mp.multipartTree.Ascend(walkTreeFunc) + } + multipartInfos := make([]*proto.MultipartInfo, len(matches)) + + var convertPartFunc = func(part *Part) *proto.MultipartPartInfo { + return &proto.MultipartPartInfo{ + ID: part.ID, + Inode: part.Inode, + MD5: part.MD5, + Size: part.Size, + UploadTime: part.UploadTime, + } + } + + var convertMultipartFunc = func(multipart *Multipart) *proto.MultipartInfo { + partInfos := make([]*proto.MultipartPartInfo, len(multipart.parts)) + for i := 0; i < len(multipart.parts); i++ { + partInfos[i] = convertPartFunc(multipart.parts[i]) + } + return &proto.MultipartInfo{ + ID: multipart.id, + Path: multipart.key, + InitTime: multipart.initTime, + Parts: partInfos, + } + } + + for i := 0; i < len(matches); i++ { + multipartInfos[i] = convertMultipartFunc(matches[i]) + } + + resp := &proto.ListMultipartResponse{ + Multiparts: multipartInfos, + } + + var reply []byte + if reply, err = json.Marshal(resp); err != nil { + p.PacketErrorWithBody(proto.OpErr, []byte(err.Error())) + return + } + p.PacketOkWithBody(reply) + return +} + +// SendMultipart replicate specified multipart operation to raft. +func (mp *metaPartition) putMultipart(op uint32, multipart *Multipart) (resp interface{}, err error) { + var encoded []byte + if encoded, err = multipart.Bytes(); err != nil { + return + } + resp, err = mp.Put(op, encoded) + return +} diff --git a/metanode/partition_store.go b/metanode/partition_store.go index ee5aebb2d..b70cf8442 100644 --- a/metanode/partition_store.go +++ b/metanode/partition_store.go @@ -19,8 +19,6 @@ import ( "encoding/binary" "encoding/json" "fmt" - "github.com/chubaofs/chubaofs/proto" - "github.com/chubaofs/chubaofs/util/errors" "hash/crc32" "io" "io/ioutil" @@ -28,6 +26,12 @@ import ( "path" "strings" "sync/atomic" + + "github.com/chubaofs/chubaofs/util/log" + + "github.com/chubaofs/chubaofs/proto" + "github.com/chubaofs/chubaofs/util/errors" + mmap "github.com/edsrzf/mmap-go" ) const ( @@ -36,6 +40,8 @@ const ( snapshotBackup = ".snapshot_backup" inodeFile = "inode" dentryFile = "dentry" + extendFile = "extend" + multipartFile = "multipart" applyIDFile = "apply" SnapshotSign = ".sign" metadataFile = "meta" @@ -72,10 +78,20 @@ func (mp *metaPartition) loadMetadata() (err error) { mp.config.End = mConf.End mp.config.Peers = mConf.Peers mp.config.Cursor = mp.config.Start + + log.LogInfof("loadMetadata: load complete: partitionID(%v) volume(%v) range(%v,%v) cursor(%v)", + mp.config.PartitionId, mp.config.VolName, mp.config.Start, mp.config.End, mp.config.Cursor) return } func (mp *metaPartition) loadInode(rootDir string) (err error) { + var numInodes uint64 + defer func() { + if err == nil { + log.LogInfof("loadInode: load complete: partitonID(%v) volume(%v) numInodes(%v)", + mp.config.PartitionId, mp.config.VolName, numInodes) + } + }() filename := path.Join(rootDir, inodeFile) if _, err = os.Stat(filename); err != nil { err = nil @@ -124,11 +140,19 @@ func (mp *metaPartition) loadInode(rootDir string) (err error) { if mp.config.Cursor < ino.Inode { mp.config.Cursor = ino.Inode } + numInodes += 1 } } // Load dentry from the dentry snapshot. func (mp *metaPartition) loadDentry(rootDir string) (err error) { + var numDentries uint64 + defer func() { + if err == nil { + log.LogInfof("loadDentry: load complete: partitonID(%v) volume(%v) numDentries(%v)", + mp.config.PartitionId, mp.config.VolName, numDentries) + } + }() filename := path.Join(rootDir, dentryFile) if _, err = os.Stat(filename); err != nil { err = nil @@ -179,13 +203,98 @@ func (mp *metaPartition) loadDentry(rootDir string) (err error) { return } if status := mp.fsmCreateDentry(dentry, true); status != proto.OpOk { - err = errors.NewErrorf("[loadDentry] createDentry dentry: %v, "+ - "resp code: %d", status) + err = errors.NewErrorf("[loadDentry] createDentry dentry: %v, resp code: %d", dentry, status) return } + numDentries += 1 } } +func (mp *metaPartition) loadExtend(rootDir string) error { + var err error + filename := path.Join(rootDir, extendFile) + if _, err = os.Stat(filename); err != nil { + return nil + } + fp, err := os.OpenFile(filename, os.O_RDONLY, 0644) + if err != nil { + return err + } + defer func() { + _ = fp.Close() + }() + var mem mmap.MMap + if mem, err = mmap.Map(fp, mmap.RDONLY, 0); err != nil { + return err + } + defer func() { + _ = mem.Unmap() + }() + var offset, n int + // read number of extends + var numExtends uint64 + numExtends, n = binary.Uvarint(mem) + offset += n + for i := uint64(0); i < numExtends; i++ { + // read length + var numBytes uint64 + numBytes, n = binary.Uvarint(mem[offset:]) + offset += n + var extend *Extend + if extend, err = NewExtendFromBytes(mem[offset : offset+int(numBytes)]); err != nil { + return err + } + log.LogDebugf("loadExtend: new extend from bytes: partitionID（%v) volume(%v) inode(%v)", + mp.config.PartitionId, mp.config.VolName, extend.inode) + _ = mp.fsmSetXAttr(extend) + offset += int(numBytes) + } + log.LogInfof("loadExtend: load complete: partitionID(%v) volume(%v) numExtends(%v) filename(%v)", + mp.config.PartitionId, mp.config.VolName, numExtends, filename) + return nil +} + +func (mp *metaPartition) loadMultipart(rootDir string) error { + var err error + filename := path.Join(rootDir, multipartFile) + if _, err = os.Stat(filename); err != nil { + return nil + } + fp, err := os.OpenFile(filename, os.O_RDONLY, 0644) + if err != nil { + return err + } + defer func() { + _ = fp.Close() + }() + var mem mmap.MMap + if mem, err = mmap.Map(fp, mmap.RDONLY, 0); err != nil { + return err + } + defer func() { + _ = mem.Unmap() + }() + var offset, n int + // read number of extends + var numMultiparts uint64 + numMultiparts, n = binary.Uvarint(mem) + offset += n + for i := uint64(0); i < numMultiparts; i++ { + // read length + var numBytes uint64 + numBytes, n = binary.Uvarint(mem[offset:]) + offset += n + var multipart *Multipart + multipart = MultipartFromBytes(mem[offset : offset+int(numBytes)]) + log.LogDebugf("loadMultipart: create multipart from bytes: partitionID（%v) multipartID(%v)", mp.config.PartitionId, multipart.id) + mp.fsmCreateMultipart(multipart) + offset += int(numBytes) + } + log.LogInfof("loadMultipart: load complete: partitionID(%v) numMultiparts(%v) filename(%v)", + mp.config.PartitionId, numMultiparts, filename) + return nil +} + func (mp *metaPartition) loadApplyID(rootDir string) (err error) { filename := path.Join(rootDir, applyIDFile) if _, err = os.Stat(filename); err != nil { @@ -219,7 +328,8 @@ func (mp *metaPartition) loadApplyID(rootDir string) (err error) { if cursor > atomic.LoadUint64(&mp.config.Cursor) { atomic.StoreUint64(&mp.config.Cursor, cursor) } - + log.LogInfof("loadApplyID: load complete: partitionID(%v) volume(%v) applyID(%v) filename(%v)", + mp.config.PartitionId, mp.config.VolName, mp.applyID, filename) return } @@ -250,7 +360,11 @@ func (mp *metaPartition) persistMetadata() (err error) { if _, err = fp.Write(data); err != nil { return } - err = os.Rename(filename, path.Join(mp.config.RootDir, metadataFile)) + if err = os.Rename(filename, path.Join(mp.config.RootDir, metadataFile)); err != nil { + return + } + log.LogInfof("persistMetata: persist complete: partitionID(%v) volume(%v) range(%v,%v) cursor(%v)", + mp.config.PartitionId, mp.config.VolName, mp.config.Start, mp.config.End, mp.config.Cursor) return } @@ -268,6 +382,8 @@ func (mp *metaPartition) storeApplyID(rootDir string, sm *storeMsg) (err error) if _, err = fp.WriteString(fmt.Sprintf("%d|%d", sm.applyIndex, atomic.LoadUint64(&mp.config.Cursor))); err != nil { return } + log.LogInfof("storeApplyID: store complete: partitionID(%v) volume(%v) applyID(%v)", + mp.config.PartitionId, mp.config.VolName, sm.applyIndex) return } @@ -310,6 +426,8 @@ func (mp *metaPartition) storeInode(rootDir string, return true }) crc = sign.Sum32() + log.LogInfof("storeInode: store complete: partitoinID(%v) volume(%) numInodes(%v) crc(%v)", + mp.config.PartitionId, mp.config.VolName, sm.inodeTree.Len(), crc) return } @@ -352,22 +470,137 @@ func (mp *metaPartition) storeDentry(rootDir string, return true }) crc = sign.Sum32() + log.LogInfof("storeDentry: store complete: partitoinID(%v) volume(%) numDentries(%v) crc(%v)", + mp.config.PartitionId, mp.config.VolName, sm.dentryTree.Len(), crc) return } -func (mp *metaPartition) deleteInodeFile() { - filename := path.Join(mp.config.RootDir, inodeFile) - // TODO Unhandled errors - os.Remove(filename) -} -func (mp *metaPartition) deleteDentryFile() { - filename := path.Join(mp.config.RootDir, dentryFile) - // TODO Unhandled errors - os.Remove(filename) +func (mp *metaPartition) storeExtend(rootDir string, sm *storeMsg) (crc uint32, err error) { + var extendTree = sm.extendTree + var fp = path.Join(rootDir, extendFile) + var f *os.File + f, err = os.OpenFile(fp, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return + } + defer func() { + closeErr := f.Close() + if err == nil && closeErr != nil { + err = closeErr + } + }() + var writer = bufio.NewWriterSize(f, 4*1024*1024) + var crc32 = crc32.NewIEEE() + var varintTmp = make([]byte, binary.MaxVarintLen64) + var n int + // write number of extends + n = binary.PutUvarint(varintTmp, uint64(extendTree.Len())) + if _, err = writer.Write(varintTmp[:n]); err != nil { + return + } + if _, err = crc32.Write(varintTmp[:n]); err != nil { + return + } + extendTree.Ascend(func(i BtreeItem) bool { + e := i.(*Extend) + var raw []byte + if raw, err = e.Bytes(); err != nil { + return false + } + // write length + n = binary.PutUvarint(varintTmp, uint64(len(raw))) + if _, err = writer.Write(varintTmp[:n]); err != nil { + return false + } + if _, err = crc32.Write(varintTmp[:n]); err != nil { + return false + } + // write raw + if _, err = writer.Write(raw); err != nil { + return false + } + if _, err = crc32.Write(raw); err != nil { + return false + } + return true + }) + if err != nil { + return + } + if err = writer.Flush(); err != nil { + return + } + if err = f.Sync(); err != nil { + return + } + crc = crc32.Sum32() + log.LogInfof("storeExtend: store complete: partitoinID(%v) volume(%) numExtends(%v) crc(%v)", + mp.config.PartitionId, mp.config.VolName, extendTree.Len(), crc) + return } -func (mp *metaPartition) deleteApplyFile() { - filename := path.Join(mp.config.RootDir, applyIDFile) - // TODO Unhandled errors - os.Remove(filename) + +func (mp *metaPartition) storeMultipart(rootDir string, sm *storeMsg) (crc uint32, err error) { + var multipartTree = sm.multipartTree + var fp = path.Join(rootDir, multipartFile) + var f *os.File + f, err = os.OpenFile(fp, os.O_RDWR|os.O_TRUNC|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return + } + defer func() { + closeErr := f.Close() + if err == nil && closeErr != nil { + err = closeErr + } + }() + var writer = bufio.NewWriterSize(f, 4*1024*1024) + var crc32 = crc32.NewIEEE() + var varintTmp = make([]byte, binary.MaxVarintLen64) + var n int + // write number of extends + n = binary.PutUvarint(varintTmp, uint64(multipartTree.Len())) + if _, err = writer.Write(varintTmp[:n]); err != nil { + return + } + if _, err = crc32.Write(varintTmp[:n]); err != nil { + return + } + multipartTree.Ascend(func(i BtreeItem) bool { + m := i.(*Multipart) + var raw []byte + if raw, err = m.Bytes(); err != nil { + return false + } + // write length + n = binary.PutUvarint(varintTmp, uint64(len(raw))) + if _, err = writer.Write(varintTmp[:n]); err != nil { + return false + } + if _, err = crc32.Write(varintTmp[:n]); err != nil { + return false + } + // write raw + if _, err = writer.Write(raw); err != nil { + return false + } + if _, err = crc32.Write(raw); err != nil { + return false + } + return true + }) + if err != nil { + return + } + + if err = writer.Flush(); err != nil { + return + } + if err = f.Sync(); err != nil { + return + } + crc = crc32.Sum32() + log.LogInfof("storeMultipart: store complete: partitoinID(%v) volume(%) numMultiparts(%v) crc(%v)", + mp.config.PartitionId, mp.config.VolName, multipartTree.Len(), crc) + return } diff --git a/metanode/partition_store_ticket.go b/metanode/partition_store_ticket.go index 87489ccd8..6ffe96dd0 100644 --- a/metanode/partition_store_ticket.go +++ b/metanode/partition_store_ticket.go @@ -23,10 +23,12 @@ import ( ) type storeMsg struct { - command uint32 - applyIndex uint64 - inodeTree *BTree - dentryTree *BTree + command uint32 + applyIndex uint64 + inodeTree *BTree + dentryTree *BTree + extendTree *BTree + multipartTree *BTree } func (mp *metaPartition) startSchedule(curIndex uint64) { diff --git a/objectnode/acl.go b/objectnode/acl.go new file mode 100644 index 000000000..ba11d4c86 --- /dev/null +++ b/objectnode/acl.go @@ -0,0 +1,266 @@ +package objectnode + +// https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/S3_ACLs_UsingACLs.html + +import ( + "encoding/xml" + "errors" + + "github.com/chubaofs/chubaofs/util/log" +) + +const ( + maxGrantCount = 100 //ACL 可以拥有最多 100 个授权。 + bucketRootPath = "/" +) + +const ( + //Permission Value + ReadPermission Permission = "READ" + WritePermission = "WRITE" + ReadACPPermission = "READ_ACP" + WriteACPPermission = "WRITE_ACP" + FullControlPermission = "FULL_CONTROL" +) + +// https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/acl-overview.html +var ( + aclBucketPermissionActions = map[Permission][]Action{ + ReadPermission: {ListBucketAction, ListBucketVersionsAction, ListBucketMultipartUploadsAction}, + WritePermission: {PutObjectAction, DeleteObjectAction}, + ReadACPPermission: {GetBucketAclAction}, + WriteACPPermission: {PutBucketAclAction}, + FullControlPermission: { + ListBucketAction, ListBucketVersionsAction, ListBucketMultipartUploadsAction, + PutObjectAction, DeleteObjectAction, + GetBucketAclAction, + PutBucketAclAction}, + } + aclObjectPermissionActions = map[Permission][]Action{ + ReadPermission: {GetObjectAction, GetObjectVersionAction, GetObjectTorrentAction}, + WritePermission: {}, + ReadACPPermission: {GetObjectAclAction, GetObjectVersionAclAction}, + WriteACPPermission: {PutObjectAclAction, PutObjectVersionAclAction}, + FullControlPermission: { + GetObjectAction, GetObjectVersionAction, GetObjectTorrentAction, + GetObjectAclAction, GetObjectVersionAclAction, + PutObjectAclAction, PutObjectVersionAclAction}, + } +) + +type StandardACL string + +const ( + PrivateACL StandardACL = "private" + PublicReadACL = "public-read" + PubliceReadWriteACL = "public-read-write" + AwsExecReadACL = "aws-exec-read" + AuthenticatedReadACL = "authenticated-read" + BucketOwnerReadACL = "bucket-owner-read" + BucketOwnerFullControlACL = "bucket-owner-full-control" + LogDeliveryWriteACL = "log-delivery-write" +) + +type ResourceType string + +const ( + bucketResource ResourceType = "bucket" + objectResource = "object" +) + +type AclRole = string + +const ( + objectOwnerRole AclRole = "owner" + bucketOwnerRole = "bucket-owner" + allUsersRole = "AllUsers" + LogDeliveryRole = "LogDelivery" +) + +var ( + aclPermissions = map[StandardACL]map[ResourceType]map[string][]Permission{ + PrivateACL: {"bucket": {"owner": {FullControlPermission}}, "object": {"owner": {FullControlPermission}}}, + PublicReadACL: {"bucket": {"owner": {FullControlPermission}, "AllUsers": {ReadPermission}}, "object": {"owner": {FullControlPermission}, "AllUsers": {ReadPermission}}}, + PubliceReadWriteACL: {"bucket": {"owner": {FullControlPermission}, "AllUsers": {ReadPermission, WritePermission}}, "object": {"owner": {FullControlPermission}, "AllUsers": {ReadPermission, WritePermission}}}, + AwsExecReadACL: {"bucket": {"owner": {FullControlPermission}}, "object": {"owner": {FullControlPermission}}}, + AuthenticatedReadACL: {"bucket": {"owner": {FullControlPermission}}, "object": {"owner": {FullControlPermission}}}, + BucketOwnerReadACL: {"object": {"owner": {FullControlPermission}, "bucket-owner": {ReadPermission}}}, + BucketOwnerFullControlACL: {"object": {"owner": {FullControlPermission}, "bucket-owner": {FullControlPermission}}}, + LogDeliveryWriteACL: {"bucket": {"LogDelivery": {WriteACPPermission, ReadACPPermission}}}, + } +) + +//grant permission +type Permission string + +// grantee +type Grantee struct { + Xmlns string `xmlns:si,attr,omitempty` + Xmlsi string `xsi:type,attr,omitempty` + Id string `xml:"ID,omitempty"` + URI string `xml:"URI,omitempty"` + Type string `xml:"Type,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + EmailAddress string `xml:"EmailAddress,omitempty"` +} + +// grant +type Grant struct { + Grantee Grantee `xml:"Grantee,omitempty"` + Permission Permission `xml:"Permission,omitempty"` +} + +// access control list +type AccessControlList struct { + Grants []Grant `xml:"Grant,omitempty"` +} + +// owner +type Owner struct { + Id string `xml:"ID"` + DispalyName string `xml:"DisplayName"` +} + +// access control policy +type AccessControlPolicy struct { + Xmlns string `xml:"xmlns:xsi,attr"` + Owner Owner `xml:"Owner,omitempty"` + Acl AccessControlList `xml:"AccessControlList,omitempty"` +} + +func (acp *AccessControlPolicy) Validate(bucket string) (bool, error) { + for _, grant := range acp.Acl.Grants { + if !grant.Validate() { + return false, nil + } + } + + return true, nil +} + +func (acp *AccessControlPolicy) IsAllowed(param *RequestParam) bool { + log.LogInfof("acl is allowed ?") + if len(acp.Acl.Grants) == 0 { + return true + } + for _, grant := range acp.Acl.Grants { + if grant.IsAllowed(param) { + return true + } + } + return false +} + +var ( + aclGrantKeyPermissionMap = map[string]Permission{ + "x-amz-grant-full-control": FullControlPermission, + "x-amz-grant-read": ReadPermission, + "x-amz-grant-read-acp": ReadACPPermission, + "x-amz-grant-write": WritePermission, + "x-amz-grant-write-acp": WriteACPPermission, + } + aclRoleURIMap = map[string]string{ + "AllUsers": "http://acs.amazonaws.com/groups/global/AllUsers", + "LogDelivery": "http://acs.amazonaws.com/groups/s3/LogDelivery", + } +) + +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html +func (acp *AccessControlPolicy) SetBucketStandardACL(param *RequestParam, acl string) { + sacl := StandardACL(acl) + var ( + rolePermissionsMap map[string][]Permission + ok bool + ) + + if rolePermissionsMap, ok = aclPermissions[sacl]["bucket"]; !ok { + return + } + for role, permissions := range rolePermissionsMap { + grantee := Grantee{} + if uri, ok := aclRoleURIMap[role]; ok { + grantee.URI = uri + } else { + grantee.Id = param.account + grantee.DisplayName = param.account + } + for _, p := range permissions { + grant := Grant{ + Grantee: grantee, + Permission: p, + } + acp.Acl.Grants = append(acp.Acl.Grants, grant) + } + } +} + +func (acp *AccessControlPolicy) SetBucketGrantACL(param *RequestParam, permission Permission) { + grantee := Grantee{ + Id: param.account, + DisplayName: param.account, + } + grant := Grant{ + Grantee: grantee, + Permission: permission, + } + acp.Acl.Grants = append(acp.Acl.Grants, grant) +} + +func (acl *AccessControlPolicy) Marshal() ([]byte, error) { + data, err := xml.Marshal(acl) + if err != nil { + return nil, err + } + return append([]byte(xml.Header), data...), nil +} + +func ParseACL(bytes []byte, bucket string) (*AccessControlPolicy, error) { + acl := &AccessControlPolicy{} + err2 := xml.Unmarshal(bytes, acl) + if err2 != nil { + return nil, err2 + } + + ok, err3 := acl.Validate(bucket) + if err3 != nil { + return nil, err3 + } + if !ok { + return nil, errors.New("") + } + + return acl, nil +} + +func storeBucketACL(bytes []byte, vol *volume) (*AccessControlPolicy, error) { + store, err1 := vol.vm.GetStore() + if err1 != nil { + return nil, err1 + } + + acl, err3 := ParseACL(bytes, vol.name) + if err3 != nil { + return nil, err3 + } + + err4 := store.Put(vol.name, bucketRootPath, OSS_ACL_KEY, bytes) + if err4 != nil { + return nil, err4 + } + + vol.storeACL(acl) + + return acl, nil +} + +func (g Grant) Validate() bool { + return true +} + +func (g *Grant) IsAllowed(param *RequestParam) bool { + if param.account != g.Grantee.Id { + return false + } + actions := aclBucketPermissionActions[g.Permission] + return IsIntersectionActions(actions, param.actions) +} diff --git a/objectnode/acl_handler.go b/objectnode/acl_handler.go new file mode 100644 index 000000000..b59f840af --- /dev/null +++ b/objectnode/acl_handler.go @@ -0,0 +1,154 @@ +package objectnode + +// https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/acl-using-rest-api.html + +import ( + "encoding/xml" + "errors" + "io" + "io/ioutil" + "net/http" + + "github.com/chubaofs/chubaofs/util/log" +) + +const ( + OSS_ACL_KEY = "oss:acl" + XMLNS = "http://www.w3.org/2001/XMLSchema-instance" + XMLXSI = "CanonicalUser" + DEF_GRANTEE_TYPE = "CanonicalUser" // + +) + +var ( + defaultGrant = Grant{ + Grantee: Grantee{ + Xmlns: XMLNS, + Xmlsi: XMLXSI, + Type: DEF_GRANTEE_TYPE, + }, + Permission: FullControlPermission, + } +) + +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html +func (o *ObjectNode) getBucketACLHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("Get bucket acl") + var ( + err error + ec *ErrorCode + ) + defer o.errorResponse(w, r, err, ec) + + _, bucket, _, vol, err := o.parseRequestParams(r) + if bucket == "" { + ec = &NoSuchBucket + return + } + + //om := vol.OSSMeta() + acl := vol.loadACL() + var aclData []byte + if acl != nil { + aclData, err = xml.Marshal(acl) + if err != nil { + ec = &InternalError + return + } + + } else { + acl = &AccessControlPolicy{} + acl.Acl.Grants = append(acl.Acl.Grants, defaultGrant) + aclData, err = xml.Marshal(acl) + if err != nil { + ec = &InternalError + return + } + } + + w.Write(aclData) + + return +} + +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html#API_PutBucketAcl_RequestSyntax +// +func (o *ObjectNode) putBucketACLHandler(w http.ResponseWriter, r *http.Request) { + var ( + err error + ec *ErrorCode + ) + defer o.errorResponse(w, r, err, ec) + + log.LogInfof("Put bucket acl") + _, bucket, _, vol, err1 := o.parseRequestParams(r) + if err1 != nil { + err = err1 + return + } + if bucket == "" { + err = errors.New("") + ec = &NoSuchBucket + return + } + + bytes, err2 := ioutil.ReadAll(r.Body) + if err2 != nil && err2 != io.EOF { + err = err2 + return + } + + acl, err3 := ParseACL(bytes, vol.name) + if err3 != nil { + err = err3 + return + } + if acl == nil { + err = errors.New("") + return + } + + //add standard acl request header + // https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/acl-overview.html + p, err5 := o.parseRequestParam(r) + if err5 != nil { + err = err5 + return + } + if standardAcls, found := r.Header["x-amz-acl"]; found { + acl.SetBucketStandardACL(p, standardAcls[0]) + } else { + for grant, permission := range aclGrantKeyPermissionMap { + if _, found2 := r.Header[grant]; found2 { + acl.SetBucketGrantACL(p, permission) + } + } + } + + newBytes, err6 := acl.Marshal() + if err6 != nil { + err = err6 + return + } + + // store bucket acl + _, err4 := storeBucketACL(newBytes, vol) + if err4 != nil { + err = err4 + return + } + + return +} + +func (o *ObjectNode) getObjectACLHandler(w http.ResponseWriter, r *http.Request) { + //TODO: implement get object acl handler + + return +} + +func (o *ObjectNode) putObjectACLHandler(w http.ResponseWriter, r *http.Request) { + //TODO: implement get object acl handler + + return +} diff --git a/objectnode/acl_test.go b/objectnode/acl_test.go new file mode 100644 index 000000000..5206a1e41 --- /dev/null +++ b/objectnode/acl_test.go @@ -0,0 +1,2 @@ +// Package s3 provides ... +package objectnode diff --git a/objectnode/api_handler.go b/objectnode/api_handler.go new file mode 100644 index 000000000..ae30db683 --- /dev/null +++ b/objectnode/api_handler.go @@ -0,0 +1,129 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "errors" + "net/http" + "strings" + + "github.com/gorilla/mux" + + "github.com/chubaofs/chubaofs/util/log" +) + +type RequestParam struct { + account string + resource string + bucket string + object string + actions []Action + sourceIP string + vol *volume + condVals map[string][]string + isOwner bool + vars map[string]string +} + +func (o *ObjectNode) parseRequestParam(r *http.Request) (*RequestParam, error) { + p := new(RequestParam) + p.vars = mux.Vars(r) + p.bucket = p.vars["bucket"] + p.object = p.vars["object"] + p.vol, _ = o.getVol(p.bucket) + p.sourceIP = getRequestIP(r) + p.condVals = getCondtionValues(r) + if len(p.bucket) > 0 { + p.resource = p.bucket + if len(p.object) > 0 { + if strings.HasPrefix(p.object, "/") { + p.resource = p.bucket + p.object + } else { + p.resource = p.bucket + "/" + p.object + } + } + } + auth := parseRequestAuthInfo(r) + if auth != nil && p.vol != nil { + accessKey, _ := p.vol.OSSSecure() + p.account = accessKey + if auth.accessKey == accessKey { + p.isOwner = true + } + } + + return p, nil +} + +//Deprecated: +func (o *ObjectNode) parseRequestParams(r *http.Request) (vars map[string]string, bucket, object string, vl *volume, err error) { + vars = mux.Vars(r) + bucket = vars["bucket"] + object = vars["object"] + if bucket != "" { + if vm, ok := o.vm.(*volumeManager); ok { + vl, err = vm.loadVolume(bucket) + if err != nil { + log.LogErrorf("parseRequestParams: load volume fail, requestId(%o) bucket(%v) err(%v)", + RequestIDFromRequest(r), bucket, err) + } + } else { + log.LogErrorf("parseRequestParams: load volume fail, requestId(%o) bucket(%v) err(%v)", + RequestIDFromRequest(r), bucket, err) + } + } + return +} + +func (o *ObjectNode) getVol(bucket string) (vol *volume, err error) { + if bucket == "" { + return nil, errors.New("bucket name is empty") + } + vm, ok := o.vm.(*volumeManager) + if !ok { + return nil, errors.New("volumeManger is invalid") + } + + vol, err = vm.loadVolume(bucket) + if err != nil { + log.LogErrorf("parseRequestParams: load volume fail, bucket(%v) err(%v)", bucket, err) + return nil, err + } + + return vol, nil +} + +func (o *ObjectNode) errorResponse(w http.ResponseWriter, r *http.Request, err error, ec *ErrorCode) { + if err != nil || ec != nil { + if err != nil { + log.LogErrorf("Handler: parse request parameters fail, requestID(%o) err(%v)", + RequestIDFromRequest(r), err) + } + if ec == nil { + ec = &InternalError + } + _ = ec.ServeResponse(w, r) + } +} + +func (o *ObjectNode) unsupportedOperationHandler(w http.ResponseWriter, r *http.Request) { + var err error + if err = UnsupportedOperation.ServeResponse(w, r); err != nil { + log.LogErrorf("unsupportedOperationHandler: serve response fail: requestID(%v) err(%v)", + RequestIDFromRequest(r), err) + ServeInternalStaticErrorResponse(w, r) + } + return +} diff --git a/objectnode/api_handler_bucket.go b/objectnode/api_handler_bucket.go new file mode 100644 index 000000000..22c063416 --- /dev/null +++ b/objectnode/api_handler_bucket.go @@ -0,0 +1,40 @@ +package objectnode + +import ( + "net/http" + + "github.com/chubaofs/chubaofs/util/log" +) + +// Head bucket +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html +func (o *ObjectNode) headBucketHandler(w http.ResponseWriter, r *http.Request) { + // do nothing +} + +// List buckets +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html +func (o *ObjectNode) listBucketsHandler(w http.ResponseWriter, r *http.Request) { + // TODO: implement 'listBucketsHandler' +} + +// Get bucket location +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html +func (o *ObjectNode) getBucketLocation(w http.ResponseWriter, r *http.Request) { + log.LogInfof("getBucketLocation: get bucket location: requestID(%v)", RequestIDFromRequest(r)) + // TODO: implement 'getBucketLocation' + var output = &GetBucketLocationOutput{ + LocationConstraint: o.region, + } + var marshaled []byte + var err error + if marshaled, err = MarshalXMLEntity(output); err != nil { + log.LogErrorf("getBucketLocation: marshal result fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + ServeInternalStaticErrorResponse(w, r) + return + } + if _, err = w.Write(marshaled); err != nil { + log.LogErrorf("getBucketLocation: write response body fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + } + return +} diff --git a/objectnode/api_handler_multipart.go b/objectnode/api_handler_multipart.go new file mode 100644 index 000000000..8fa4370c9 --- /dev/null +++ b/objectnode/api_handler_multipart.go @@ -0,0 +1,373 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "net/http" + "strconv" + + "github.com/chubaofs/chubaofs/util/log" +) + +// Create multipart upload +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html +func (o *ObjectNode) createMultipleUploadHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("createMultipleUploadHandler: init multiple upload, requestID(%v) remote(%v)", + RequestIDFromRequest(r), r.RemoteAddr) + + _, bucket, object, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("createMultipleUploadHandler: parse request parameters fail, requestID(%o) err(%v)", + RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + uploadId, initErr := vl.InitMultipart(object) + if initErr != nil { + log.LogErrorf("createMultipleUploadHandler: init multipart fail, requestID(%o) err(%v)", + RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + initResult := InitMultipartResult{ + Bucket: bucket, + Key: object, + UploadId: uploadId, + } + + var bytes []byte + var marshalError error + if bytes, marshalError = MarshalXMLEntity(initResult); marshalError != nil { + log.LogErrorf("createMultipleUploadHandler: marshal result fail, requestID(%v) err(%v)", + RequestIDFromRequest(r), err) + _ = InternalError.ServeResponse(w, r) + return + } + + // set response header + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes))) + if _, err = w.Write(bytes); err != nil { + log.LogErrorf("createMultipleUploadHandler: write response body fail, requestID(%v) err(%v)", + RequestIDFromRequest(r), err) + } + return +} + +// Upload part +// Uploads a part in a multipart upload. +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html . +func (o *ObjectNode) uploadPartHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("uploadPartHandler: upload part, requestID(%v) remote(%v)", + RequestIDFromRequest(r), r.RemoteAddr) + // check args + params, _, object, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("uploadPartHandler: parse request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + //// get upload id and part number + uploadId := params[ParamUploadId] + partNumber := params[ParamPartNumber] + if uploadId == "" || partNumber == "" { + log.LogErrorf("uploadPartHandler: illegal uploadID or partNumber, requestID(%v)", RequestIDFromRequest(r)) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + var partNumberInt uint64 + if partNumberInt, err = strconv.ParseUint(partNumber, 10, 64); err != nil { + log.LogErrorf("uploadPartHandler: parse part number fail, requestID(%o) raw(%v) err(%v)", + RequestIDFromRequest(r), partNumber, err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + // handle exception + + var fsFileInfo *FSFileInfo + if fsFileInfo, err = vl.WritePart(object, uploadId, uint16(partNumberInt), r.Body); err != nil { + log.LogErrorf("uploadPartHandler: write part fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err) + _ = InternalError.ServeResponse(w, r) + return + } + log.LogDebugf("uploadPartHandler: write part, requestID(%v) fsFileInfo(%v)", RequestIDFromRequest(r), fsFileInfo) + + // write header to response + w.Header().Set(HeaderNameContentLength, "0") + w.Header().Set(HeaderNameETag, fsFileInfo.ETag) + return +} + +// List parts +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html +func (o *ObjectNode) listPartsHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("listPartsHandler: list parts, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr) + + // check args + params, bucket, object, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("listPartsHandler: parse request parameters fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + //// get upload id and part number + uploadId := params[ParamUploadId] + maxParts := params[ParamMaxParts] + partNoMarker := params[ParamPartNoMarker] + + var maxPartsInt uint64 + var partNoMarkerInt uint64 + + if uploadId == "" { + log.LogErrorf("listPartsHandler: illegal update ID, requestID(%o) err(%v)", RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + if maxParts == "" { + maxPartsInt = MaxParts + } else { + maxPartsInt, err = strconv.ParseUint(maxParts, 10, 64) + if err != nil { + log.LogErrorf("listPartsHandler: parse max parts fail, requestID(%v) raw(%v) err(%v)", RequestIDFromRequest(r), maxParts, err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + if maxPartsInt > MaxParts { + maxPartsInt = MaxParts + } + } + if partNoMarker != "" { + res, err := strconv.ParseUint(uploadId, 10, 64) + if err != nil { + log.LogErrorf("listPatsHandler: parse update ID fail, requestID(%v) raw(%v) err(%v)", RequestIDFromRequest(r), uploadId, err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + partNoMarkerInt = res + } + + fsParts, nextMarker, isTruncated, err := vl.ListParts(object, uploadId, maxPartsInt, partNoMarkerInt) + if err != nil { + log.LogErrorf("listPartsHandler: volume list parts fail, requestID(%v) uploadID(%v) maxParts(%v) partNoMarker(%v) err(%v)", + RequestIDFromRequest(r), uploadId, maxPartsInt, partNoMarkerInt, err) + _ = InternalError.ServeResponse(w, r) + return + } + log.LogDebugf("listPartsHandler: volume list parts, "+ + "requestID(%v) uploadID(%v) maxParts(%v) partNoMarker(%v) numFSParts(%v) nextMarker(%v) isTruncated(%v)", + RequestIDFromRequest(r), uploadId, maxPartsInt, partNoMarkerInt, len(fsParts), nextMarker, isTruncated) + + // get owner + accessKey, _ := vl.OSSSecure() + bucketOwner := NewBucketOwner(accessKey) + + // get parts + parts := NewParts(fsParts) + + listPartsResult := ListPartsResult{ + Bucket: bucket, + Key: object, + UploadId: uploadId, + StorageClass: StorageClassStandard, + NextMarker: int(nextMarker), + MaxParts: int(maxPartsInt), + IsTruncated: isTruncated, + Parts: parts, + Owner: bucketOwner, + } + + var bytes []byte + var marshalError error + if bytes, marshalError = MarshalXMLEntity(listPartsResult); marshalError != nil { + log.LogErrorf("listPartsHandler: marshal result fail, requestID(%o) err(%v)", + RequestIDFromRequest(r), err) + _ = InternalError.ServeResponse(w, r) + return + } + + // set response header + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes))) + if _, err = w.Write(bytes); err != nil { + log.LogErrorf("listPartsHandler: write response body fail, requestID(%v) err(%v)", + RequestIDFromRequest(r), err) + } + return +} + +// Complete multipart +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html +func (o *ObjectNode) completeMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("completeMultipartUploadHandler: complete multiple upload, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr) + + params, bucket, object, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("[UploadPart] request id [%o], Get volume failed cause : %o", r.URL, err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + // get upload id and part number + uploadId := params[ParamUploadId] + if uploadId == "" { + log.LogErrorf("[CompleteMultipartUpload] request id [%o], Upload id can not be empty", r.URL.String()) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + fsFileInfo, err := vl.CompleteMultipart(object, uploadId) + if err != nil { + log.LogErrorf("completeMultipartUploadHandler: complete multipart fail, requestID(%o) uploadID(%v) err(%v)", + RequestIDFromRequest(r), uploadId, err) + _ = InternalError.ServeResponse(w, r) + return + } + log.LogDebugf("completeMultipartUploadHandler: complete multipart, requestID(%o) uploadID(%v) path(%v)", + RequestIDFromRequest(r), uploadId, object) + + // write response + completeResult := CompleteMultipartResult{ + Bucket: bucket, + Key: object, + ETag: fsFileInfo.ETag, + } + + var bytes []byte + var marshalError error + if bytes, marshalError = MarshalXMLEntity(completeResult); marshalError != nil { + log.LogErrorf("completeMultipartUploadHandler: marshal result fail, requestID(%o) err(%v)", RequestIDFromRequest(r), marshalError) + _ = InternalError.ServeResponse(w, r) + return + } + + // set response header + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes))) + if _, err = w.Write(bytes); err != nil { + log.LogErrorf("completeMultipartUploadHandler: write response body fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err) + return + } + return +} + +// Abort multipart +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html . +func (o *ObjectNode) abortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("abortMultipartUploadHandler: abort multiple upload, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr) + + // check args + params, _, object, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("abortMultipartUploadHandler: parse request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + uploadId := params["uploadId"] + //// Abort multipart upload + err = vl.AbortMultipart(object, uploadId) + if err != nil { + log.LogErrorf("abortMultipartUploadHandler: volume abort multipart fail, requestID(%o) uploadID(%v) err(%v)", RequestIDFromRequest(r), uploadId, err) + _ = InternalError.ServeResponse(w, r) + return + } + log.LogDebugf("abortMultipartUploadHandler: volume abort multipart, requestID(%o) uploadID(%v) path(%v)", RequestIDFromRequest(r), uploadId, object) + return +} + +// List multipart uploads +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html +func (o *ObjectNode) listMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("abortMultipartUploadHandler: list multipart uploads, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr) + // check args + params, bucket, _, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("abortMultipartUploadHandler: parse request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + } + + // get list uploads parameter + prefix := params[ParamPrefix] + keyMarker := params[ParamKeyMarker] + delimiter := params[ParamPartDelimiter] + maxUploads := params[ParamPartMaxUploads] + uploadIdMarker := params[ParamUploadIdMarker] + + var maxUploadsInt uint64 + if maxUploads == "" { + maxUploadsInt = MaxUploads + } else { + maxUploadsInt, err = strconv.ParseUint(maxUploads, 10, 64) + if err != nil { + log.LogErrorf("[ListMultipartUploads] request id [%o], Param max-uploads is invalid, failed info : %o", r.URL, err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + if maxUploadsInt > MaxUploads { + maxUploadsInt = MaxUploads + } + } + + fsUploads, nextKeyMarker, nextUploadIdMarker, IsTruncated, prefixes, err := vl.ListMultipartUploads(prefix, delimiter, keyMarker, uploadIdMarker, maxUploadsInt) + if err != nil { + log.LogErrorf("[ListMultipartUploads] request id [%o], List multipart uploads failed cause : %o", r.URL, err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + accessKey, _ := vl.OSSSecure() + uploads := NewUploads(fsUploads, accessKey) + + var commonPrefixes = make([]*CommonPrefix, 0) + for _, prefix := range prefixes { + commonPrefix := &CommonPrefix{ + Prefix: prefix, + } + commonPrefixes = append(commonPrefixes, commonPrefix) + } + + listUploadsResult := ListUploadsResult{ + Bucket: bucket, + KeyMarker: keyMarker, + UploadIdMarker: uploadIdMarker, + NextKeyMarker: nextKeyMarker, + NextUploadIdMarker: nextUploadIdMarker, + Delimiter: delimiter, + Prefix: prefix, + MaxUploads: int(maxUploadsInt), + IsTruncated: IsTruncated, + Uploads: uploads, + CommonPrefixes: commonPrefixes, + } + + var bytes []byte + var marshalError error + if bytes, marshalError = MarshalXMLEntity(listUploadsResult); marshalError != nil { + log.LogErrorf("[ListMultipartUploads] request id [%o], Marshal list uploads result failed cause : %o", r.URL, err) + _ = InternalError.ServeResponse(w, r) + return + } + + // set response header + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes))) + w.Write(bytes) + return +} diff --git a/objectnode/api_handler_object.go b/objectnode/api_handler_object.go new file mode 100644 index 000000000..c3f16b01f --- /dev/null +++ b/objectnode/api_handler_object.go @@ -0,0 +1,790 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "net/http" + "regexp" + "strconv" + "strings" + + "sync" + "syscall" + + "github.com/chubaofs/chubaofs/util/log" +) + +var ( + rangeRegexp = regexp.MustCompile("^bytes=(\\d)+-(\\d)*$") +) + +// Get object +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html +func (o *ObjectNode) getObjectHandler(w http.ResponseWriter, r *http.Request) { + // TODO: range read support + log.LogInfof("getObjectHandler: get object, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr) + + _, _, object, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("getObjectHandler: parse request parameters fail, requestId(%o) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + // parse http range option + var rangeOpt = strings.TrimSpace(r.Header.Get(HeaderNameRange)) + var rangeLower uint64 + var rangeUpper uint64 + var isRangeRead bool + if len(rangeOpt) > 0 && rangeRegexp.MatchString(rangeOpt) { + + var hyphenIndex = strings.Index(rangeOpt, "-") + if hyphenIndex < 0 { + _ = InvalidArgument.ServeResponse(w, r) + return + } + + var lowerPart = rangeOpt[len("bytes="):hyphenIndex] + var upperPart = "" + if hyphenIndex+1 < len(rangeOpt) { + upperPart = rangeOpt[hyphenIndex+1:] + } + + if len(lowerPart) > 0 { + if rangeLower, err = strconv.ParseUint(lowerPart, 10, 64); err != nil { + log.LogErrorf("getObjectHandler: parse range lower fail: requestID(%v) rangeOpt(%v) err(%v)", + RequestIDFromRequest(r), rangeOpt, err) + ServeInternalStaticErrorResponse(w, r) + return + } + } + if len(upperPart) > 0 { + if rangeUpper, err = strconv.ParseUint(upperPart, 10, 64); err != nil { + log.LogErrorf("getObjectHandler: parse range upper fail: requestID(%v) rangeOpt(%v) err(%v)", + RequestIDFromRequest(r), rangeOpt, err) + ServeInternalStaticErrorResponse(w, r) + return + } + } + if rangeUpper > 0 && rangeUpper < rangeLower { + // upper enabled and lower than lower side + if err = InvalidArgument.ServeResponse(w, r); err != nil { + log.LogErrorf("getObjectHandler: serve response fail: requestID(%v) err(%v)", + RequestIDFromRequest(r), err) + return + } + } + + isRangeRead = true + log.LogDebugf("getObjectHandler: parse range option: requestID(%v) rangeOpt(%v) rangeLower(%v) rangeUpper(%v)", + RequestIDFromRequest(r), rangeOpt, rangeLower, rangeUpper) + } + + // get object meta + fileInfo, err := vl.FileInfo(object) + if err != nil { + log.LogErrorf("getObjectHandler: volume get file info fail, requestId(%o) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchKey.ServeResponse(w, r) + return + } + + // validate and fix range + if isRangeRead && rangeUpper > uint64(fileInfo.Size) { + rangeUpper = uint64(fileInfo.Size) + } + + // compute content length + var contentLength = uint64(fileInfo.Size) + if isRangeRead { + contentLength = rangeUpper - rangeLower + } + + // set response header for GetObject + w.Header().Set(HeaderNameETag, fileInfo.ETag) + w.Header().Set(HeaderNameAcceptRange, HeaderValueAcceptRange) + w.Header().Set(HeaderNameLastModified, formatTimeRFC1123(fileInfo.ModifyTime)) + w.Header().Set(HeaderNameContentType, HeaderValueTypeStream) + w.Header().Set(HeaderNameContentLength, strconv.FormatUint(contentLength, 10)) + + if isRangeRead { + w.Header().Set(HeaderNameContentRange, fmt.Sprintf("bytes %d-%d/%d", rangeLower, rangeUpper, fileInfo.Size)) + } + + // get object content + var offset = rangeLower + var size = uint64(fileInfo.Size) + if isRangeRead { + if rangeUpper == 0 { + size = uint64(fileInfo.Size) - rangeLower + } else { + size = rangeUpper - rangeLower + } + } + if err = vl.ReadFile(object, w, offset, size); err != nil { + log.LogErrorf("getObjectHandler: read from volume fail: requestId(%v) volume(%v) path(%v) offset(%v) size(%v) err(%v)", + RequestIDFromRequest(r), vl.name, object, offset, size, err) + _ = InternalError.ServeResponse(w, r) + return + } + log.LogDebugf("getObjectHandler: volume read file: requestID(%v) volume(%v) path(%v) offset(%v) size(%v)", + RequestIDFromRequest(r), vl.name, object, offset, size) + return +} + +// Head object +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html +func (o *ObjectNode) headObjectHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("headObjectHandler: get object meta, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr) + + // check args + _, _, object, vl, err := o.parseRequestParams(r) + log.LogInfof("headObjectHandler: parse request params result in header object handler, object : (%o), vl : (%o), err : (%o)", object, vl.name, err) + if err != nil { + log.LogErrorf("headObjectHandler: parse request parameters fail, requestId(%o) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + // get object meta + fileInfo, err := vl.FileInfo(object) + if err != nil && err == syscall.ENOENT { + log.LogErrorf("headObjectHandler: get file meta fail, requestId(%o), err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchKey.ServeResponse(w, r) + return + } + if err != nil { + log.LogErrorf("headObjectHandler: get file meta fail, requestId(%o), err(%v)", RequestIDFromRequest(r), err) + _ = InternalError.ServeResponse(w, r) + return + } + + // set response header + w.Header().Set(HeaderNameETag, fileInfo.ETag) + w.Header().Set(HeaderNameAcceptRange, HeaderValueAcceptRange) + w.Header().Set(HeaderNameContentType, HeaderValueTypeStream) + w.Header().Set(HeaderNameLastModified, formatTimeRFC1123(fileInfo.ModifyTime)) + w.Header().Set(HeaderNameContentLength, strconv.Itoa(int(fileInfo.Size))) + w.Header().Set(HeaderNameContentMD5, EmptyContentMD5String) + return +} + +// Delete objects (multiple objects) +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html +func (o *ObjectNode) deleteObjectsHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("deleteObjectsHandler: delete multiple objects, requestID(%v) remote(%v)", + RequestIDFromRequest(r), r.RemoteAddr) + // check args + _, _, _, vl, err := o.parseRequestParams(r) + + bytes, err := ioutil.ReadAll(r.Body) + if err != nil && err != io.EOF { + log.LogErrorf("deleteObjectsHandler: read request body fail: requestID(%o) err(%v)", RequestIDFromRequest(r), err) + _ = InternalError.ServeResponse(w, r) + return + } + + deleteReq := DeleteRequest{} + err = UnmarshalXMLEntity(bytes, &deleteReq) + if err != nil { + log.LogErrorf("deleteObjectsHandler: unmarshal xml fail: requestID(%o) err(%v)", + RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + if len(deleteReq.Objects) <= 0 { + log.LogDebugf("deleteObjectsHandler: non objects found in request: requestID(%o)", RequestIDFromRequest(r)) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + var wg sync.WaitGroup + deletesResult := DeletesResult{ + DeletedObjects: make([]Deleted, 0), + DeletedErrors: make([]Error, 0), + } + deletedObjectsCh := make(chan *Deleted, len(deleteReq.Objects)) + deletedErrorsCh := make(chan *Error, len(deleteReq.Objects)) + + for _, object := range deleteReq.Objects { + wg.Add(1) + go func(obj Object) { + defer func() { + wg.Done() + if errDelete := recover(); errDelete != nil { + deletedError := Error{ + Key: obj.Key, + Code: strconv.Itoa(InternalError.StatusCode), + Message: InternalError.ErrorMessage, + } + deletedErrorsCh <- &deletedError + } + }() + + err = vl.DeleteFile(obj.Key) + if err != nil { + ossError := transferError(obj.Key, err) + deletedErrorsCh <- &ossError + } else { + deleted := Deleted{Key: obj.Key} + deletedObjectsCh <- &deleted + log.LogDebugf("deleteObjectsHandler: delete object: requestID(%v) key(%o)", RequestIDFromRequest(r), + deleted.Key) + } + }(object) + } + + wg.Wait() + close(deletedObjectsCh) + close(deletedErrorsCh) + + for { + deletedObject := <-deletedObjectsCh + if deletedObject == nil { + break + } + deletesResult.DeletedObjects = append(deletesResult.DeletedObjects, *deletedObject) + } + for { + deletedError := <-deletedErrorsCh + if deletedError == nil { + break + } + deletesResult.DeletedErrors = append(deletesResult.DeletedErrors, *deletedError) + } + + log.LogDebugf("deleteObjectsHandler: delete objects: deletes(%v) errors(%v)", len(deletesResult.DeletedObjects), len(deletesResult.DeletedErrors)) + var bytesRes []byte + var marshalError error + if bytesRes, marshalError = MarshalXMLEntity(deletesResult); marshalError != nil { + log.LogErrorf("deleteObjectsHandler: marshal xml entity fail: requestID(%o) err(%v)", RequestIDFromRequest(r), err) + if respErr := InternalError.ServeResponse(w, r); respErr != nil { + log.LogErrorf("deleteObjectsHandler: write response fail: requestID(%v) err(%v)", RequestIDFromRequest(r), respErr) + return + } + return + } + + // set response header + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytesRes))) + if _, err = w.Write(bytesRes); err != nil { + log.LogErrorf("deleteObjectsHandler: write response body fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + } + return +} + +func parseCopySourceInfo(r *http.Request) (sourceBucket, sourceObject string) { + var copySource = r.Header.Get(HeaderNameCopySource) + if strings.HasPrefix(copySource, "/") { + copySource = copySource[1:] + } + position := strings.Index(copySource, "/") + var bucket, object string + if position >= 0 { + bucket = copySource[:position] + if position+1 <= len(copySource) { + object = copySource[position+1:] + } + } + sourceBucket = bucket + sourceObject = object + return +} + +// Copy object +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html . +func (o *ObjectNode) copyObjectHandler(w http.ResponseWriter, r *http.Request) { + // check args + _, bucket, object, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("copyObjectHandler: parse request params fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + sourceBucket, sourceObject := parseCopySourceInfo(r) + if bucket != sourceBucket { + log.LogDebugf("copyObjectHandler: source bucket is not same with bucket: requestID(%v) target(%v) source(%v)", + RequestIDFromRequest(r), bucket, sourceBucket) + _ = UnsupportedOperation.ServeResponse(w, r) + return + } + + if sourceObject == object { + log.LogErrorf("copyObjectHandler: source object same with target object: requestID(%v) target(%v) source(%v)", + RequestIDFromRequest(r), object, sourceObject) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + // get object meta + fileInfo, err := vl.FileInfo(sourceObject) + if err != nil { + log.LogErrorf("copyObjectHandler: volume get file info fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchKey.ServeResponse(w, r) + return + } + + // get header + copyMatch := r.Header.Get(HeaderNameCopyMatch) + noneMatch := r.Header.Get(HeaderNameCopyNoneMatch) + modified := r.Header.Get(HeaderNameCopyModified) + unModified := r.Header.Get(HeaderNameCopyUnModified) + + // response 412 + if modified != "" { + fileModTime := fileInfo.ModifyTime + modifiedTime, err := parseTimeRFC1123(modified) + if err != nil { + log.LogErrorf("copyObjectHandler: parse RFC1123 time fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + if fileModTime.Before(modifiedTime) { + log.LogInfof("copyObjectHandler: file modified time not after than specified time: requestID(%v)", RequestIDFromRequest(r)) + _ = PreconditionFailed.ServeResponse(w, r) + return + } + } + if unModified != "" { + fileModTime := fileInfo.ModifyTime + unmodifiedTime, err := parseTimeRFC1123(unModified) + if err != nil { + log.LogErrorf("copyObjectHandler: parse RFC1123 time fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + if fileModTime.After(unmodifiedTime) { + log.LogInfof("copyObjectHandler: file modified time not before than specified time: requestID(%v)", RequestIDFromRequest(r)) + _ = PreconditionFailed.ServeResponse(w, r) + return + } + } + if copyMatch != "" && fileInfo.ETag != copyMatch { + log.LogInfof("copyObjectHandler: eTag mismatched with specified: requestID(%v)", RequestIDFromRequest(r)) + _ = PreconditionFailed.ServeResponse(w, r) + return + } + if noneMatch != "" && fileInfo.ETag == noneMatch { + log.LogInfof("copyObjectHandler: eTag same with specified: requestID(%v)", RequestIDFromRequest(r)) + _ = PreconditionFailed.ServeResponse(w, r) + return + } + + fsFileInfo, err := vl.CopyFile(object, sourceObject) + if err != nil { + log.LogErrorf("copyObjectHandler: volume copy file fail: requestID(%o) volume(%v) source(%v) target(%v) err(%v)", + RequestIDFromRequest(r), vl.name, sourceObject, object, err) + _ = InternalError.ServeResponse(w, r) + return + } + + copyResult := CopyResult{ + ETag: fsFileInfo.ETag, + LastModified: formatTimeISO(fsFileInfo.ModifyTime), + } + + var bytes []byte + var marshalError error + if bytes, marshalError = MarshalXMLEntity(copyResult); marshalError != nil { + log.LogErrorf("copyObjectHandler: marshal xml entity fail: requestID(%o) err(%v)", RequestIDFromRequest(r), err) + _ = InternalError.ServeResponse(w, r) + return + } + + // set response header + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes))) + _, _ = w.Write(bytes) + + return +} + +// List objects v1 +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html +func (o *ObjectNode) getBucketV1Handler(w http.ResponseWriter, r *http.Request) { + // check args + _, bucket, _, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("getBucketV1Handler: parse request parameters fail, requestID(%o) err(%v)", + RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + // get options + marker := r.URL.Query().Get(ParamMarker) + prefix := r.URL.Query().Get(ParamPrefix) + maxKeys := r.URL.Query().Get(ParamMaxKeys) + delimiter := r.URL.Query().Get(ParamPartDelimiter) + + var maxKeysInt uint64 + if maxKeys != "" { + maxKeysInt, err = strconv.ParseUint(maxKeys, 10, 16) + if err != nil { + log.LogErrorf("getBucketV1Handler: parse max key fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + if maxKeysInt > MaxKeys { + maxKeysInt = MaxKeys + } + } else { + maxKeysInt = uint64(MaxKeys) + } + + listBucketRequest := &ListBucketRequestV1{ + prefix: prefix, + delimiter: delimiter, + marker: marker, + maxKeys: maxKeysInt, + } + + fsFileInfos, nextMarker, isTruncated, prefixes, err := vl.ListFilesV1(listBucketRequest) + if err != nil { + log.LogErrorf("getBucketV1Handler: list file fail, requestID(%o), err(%o)", r.URL, err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + // get owner + aceesKey, _ := vl.OSSSecure() + bucketOwner := NewBucketOwner(aceesKey) + var contents = make([]*Content, 0) + if len(fsFileInfos) > 0 { + for _, fsFileInfo := range fsFileInfos { + content := &Content{ + Key: fsFileInfo.Path, + LastModified: formatTimeISO(fsFileInfo.ModifyTime), + ETag: fsFileInfo.ETag, + Size: int(fsFileInfo.Size), + StorageClass: StorageClassStandard, + Owner: bucketOwner, + } + contents = append(contents, content) + } + } + + var commonPrefixes = make([]*CommonPrefix, 0) + for _, prefix := range prefixes { + commonPrefix := &CommonPrefix{ + Prefix: prefix, + } + commonPrefixes = append(commonPrefixes, commonPrefix) + } + + listBucketResult := &ListBucketResult{ + Bucket: bucket, + Prefix: prefix, + Marker: marker, + MaxKeys: int(maxKeysInt), + Delimiter: delimiter, + IsTruncated: isTruncated, + NextMarker: nextMarker, + Contents: contents, + CommonPrefixes: commonPrefixes, + } + + var bytes []byte + var marshalError error + if bytes, marshalError = MarshalXMLEntity(listBucketResult); marshalError != nil { + log.LogErrorf("getBucketV1Handler: marshal result fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + // set response header + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes))) + w.Write(bytes) + + return +} + +// List objects version 2 +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html +func (o *ObjectNode) getBucketV2Handler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("getBucketV2Handler: get bucket, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr) + + // check args + _, bucket, _, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("getBucketV2Handler: parse request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + // get options + prefix := r.URL.Query().Get(ParamPrefix) + maxKeys := r.URL.Query().Get(ParamMaxKeys) + delimiter := r.URL.Query().Get(ParamPartDelimiter) + contToken := r.URL.Query().Get(ParamContToken) + fetchOwner := r.URL.Query().Get(ParamFetchOwner) + startAfter := r.URL.Query().Get(ParamStartAfter) + + var maxKeysInt uint64 + if maxKeys != "" { + maxKeysInt, err = strconv.ParseUint(maxKeys, 10, 16) + if err != nil { + log.LogErrorf("getBucketV2Handler: parse max keys fail, requestID(%v) err(%v)", + RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + if maxKeysInt > MaxKeys { + maxKeysInt = MaxKeys + } + } else { + maxKeysInt = MaxKeys + } + + var fetchOwnerBool bool + if fetchOwner != "" { + fetchOwnerBool, err = strconv.ParseBool(fetchOwner) + if err != nil { + log.LogErrorf("getBucketV2Handler: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + } else { + fetchOwnerBool = false + } + + request := &ListBucketRequestV2{ + delimiter: delimiter, + maxKeys: maxKeysInt, + prefix: prefix, + contToken: contToken, + fetchOwner: fetchOwnerBool, + startAfter: startAfter, + } + + fsFileInfos, keyCount, nextToken, isTruncated, prefixes, err := vl.ListFilesV2(request) + if err != nil { + log.LogErrorf("getBucketV2Handler: request id [%o], Get files list failed cause : %o", r.URL, err) + _ = InternalError.ServeResponse(w, r) + return + } + // get owner + var bucketOwner *BucketOwner + if fetchOwnerBool { + accessKey, _ := vl.OSSSecure() + bucketOwner = NewBucketOwner(accessKey) + } + + var contents = make([]*Content, 0) + if len(fsFileInfos) > 0 { + for _, fsFileInfo := range fsFileInfos { + content := &Content{ + Key: fsFileInfo.Path, + LastModified: formatTimeISO(fsFileInfo.ModifyTime), + ETag: fsFileInfo.ETag, + Size: int(fsFileInfo.Size), + StorageClass: StorageClassStandard, + Owner: bucketOwner, + } + contents = append(contents, content) + } + } + + var commonPrefixes = make([]*CommonPrefix, 0) + for _, prefix := range prefixes { + commonPrefix := &CommonPrefix{ + Prefix: prefix, + } + commonPrefixes = append(commonPrefixes, commonPrefix) + } + + listBucketResult := ListBucketResultV2{ + Name: bucket, + Prefix: prefix, + Token: contToken, + NextToken: nextToken, + KeyCount: keyCount, + MaxKeys: maxKeysInt, + Delimiter: delimiter, + IsTruncated: isTruncated, + Contents: contents, + CommonPrefixes: commonPrefixes, + } + + var bytes []byte + var marshalError error + if bytes, marshalError = MarshalXMLEntity(listBucketResult); marshalError != nil { + log.LogErrorf("getBucketV2Handler: marshal result fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + // set response header + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.Header().Set(HeaderNameContentLength, strconv.Itoa(len(bytes))) + if _, err = w.Write(bytes); err != nil { + log.LogErrorf("getBucketVeHandler: write response body fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err) + } + return +} + +// Put object +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html +func (o *ObjectNode) putObjectHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("putObjectHandler: put object, requestID(%v) remote(%v)", RequestIDFromRequest(r), r.RemoteAddr) + + _, bucket, object, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("putObjectHandler: parser request parameters fail, requestID(%o) err(%v)", RequestIDFromRequest(r), err) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + // check args + if bucket == "" || object == "" { + log.LogErrorf("putObjectHandler: illegal bucket or object found: requestID(%o)", RequestIDFromRequest(r)) + _ = InvalidArgument.ServeResponse(w, r) + return + } + + // Get request MD5, if request MD5 is not empty, compute and verify it. + requestMD5 := r.Header.Get(HeaderNameContentMD5) + + var checkMD5 bool + if len(requestMD5) > 0 { + checkMD5 = true + } + + var multipartID string + if multipartID, err = vl.InitMultipart(object); err != nil { + log.LogErrorf("putObjectHandler: volume init multipart fail: requestID(%v) path(%v) err(%v)", + RequestIDFromRequest(r), object, err) + _ = InternalError.ServeResponse(w, r) + return + } + defer func() { + // rollback policy + if err != nil { + if abortErr := vl.AbortMultipart(object, multipartID); abortErr != nil { + log.LogErrorf("putObjectHandler: volume abort multipart fail: requestID(%v) path(%v) multipartID(%v) err(%v)", + RequestIDFromRequest(r), object, multipartID, err) + } + } + }() + const partID uint16 = 1 + if _, err = vl.WritePart(object, multipartID, partID, r.Body); err != nil { + log.LogErrorf("putObjectHandler: volume write part fail: requestID(%v) path(%v) multipartID(%v) err(%v)", + RequestIDFromRequest(r), object, multipartID, err) + _ = InternalError.ServeResponse(w, r) + return + } + var fsFileInfo *FSFileInfo + if fsFileInfo, err = vl.CompleteMultipart(object, multipartID); err != nil { + log.LogErrorf("putObjectHandler: volume complete multipart fail: requestID(%v) path(%v) multipartID(%v) err(%v)", + RequestIDFromRequest(r), object, multipartID, err) + _ = InternalError.ServeResponse(w, r) + return + } + + // validate content MD5 value + if strings.HasSuffix(requestMD5, "==") { + var decoded []byte + if decoded, err = base64.StdEncoding.DecodeString(requestMD5); err != nil { + log.LogErrorf("putObjectHandler: decode request MD5 value fail: requestID(%v) raw(%v) err(%v)", + RequestIDFromRequest(r), requestMD5, err) + _ = InternalError.ServeResponse(w, r) + return + } + requestMD5 = hex.EncodeToString(decoded) + } + // check content MD5 + if checkMD5 && requestMD5 != fsFileInfo.ETag { + log.LogErrorf("putObjectHandler: MD5 validate fail: requestID(%v) requestMD5(%o) serverMD5(%o)", + r.URL.EscapedPath(), requestMD5, fsFileInfo.ETag) + _ = BadDigest.ServeResponse(w, r) + return + } + + // set response header + w.Header().Set(HeaderNameETag, fsFileInfo.ETag) + w.Header().Set(HeaderNameContentLength, "0") + return +} + +// Delete object +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html . +func (o *ObjectNode) deleteObjectHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("Delete object...") + + _, _, object, vl, err := o.parseRequestParams(r) + if err != nil { + log.LogErrorf("[GetObject] request id [%o], Bucket or object can not be empty", r.URL) + _ = NoSuchBucket.ServeResponse(w, r) + return + } + + err = vl.DeleteFile(object) + if err != nil { + log.LogErrorf("[DeleteFile] request id [%o], Delete object failed cause : %o", r.URL.String(), err) + _ = InternalError.ServeResponse(w, r) + return + } + + return +} + +// Get object tagging +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html +func (o *ObjectNode) getObjectTagging(w http.ResponseWriter, r *http.Request) { + // TODO: implement handler 'GetObjectTagging' + return +} + +// Put object tagging +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html +func (o *ObjectNode) putObjectTagging(w http.ResponseWriter, r *http.Request) { + // TODO: implement handler 'PutObjectTagging' + return +} + +// Delete object tagging +// API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjectTagging.html +func (o *ObjectNode) deleteObjectTagging(w http.ResponseWriter, r *http.Request) { + // TODO: implement handler 'DeleteObjectTagging' + return +} + +// Put object extend attribute (xattr) +func (o *ObjectNode) putObjectXAttr(w http.ResponseWriter, r *http.Request) { + // TODO: implement 'putObjectXAttr' +} + +// Get object extend attribute (xattr) +func (o *ObjectNode) getObjectXAttr(w http.ResponseWriter, r *http.Request) { + // TODO: implement 'getObjectXAttr' +} + +// Delete object extend attribute (xattr) +func (o *ObjectNode) deleteObjectXAttr(w http.ResponseWriter, r *http.Request) { + // TODO: implement 'deleteObjectXAttr' +} + +// List object xattrs +func (o *ObjectNode) listObjectXAttrs(w http.ResponseWriter, r *http.Request) { + // TODO: implement 'listObjectXAttrs' +} diff --git a/objectnode/api_middleware.go b/objectnode/api_middleware.go new file mode 100644 index 000000000..46c09f39c --- /dev/null +++ b/objectnode/api_middleware.go @@ -0,0 +1,125 @@ +// Package s3 provides ... +package objectnode + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/chubaofs/chubaofs/util/log" + + "github.com/google/uuid" + "github.com/gorilla/mux" +) + +const ( + ctxKeyRequestID = "ctx_request_id" +) + +func RequestIDFromRequest(r *http.Request) (id string) { + return mux.Vars(r)[ctxKeyRequestID] +} + +func (o *ObjectNode) traceMiddleware(next http.Handler) http.Handler { + var generateRequestID = func() (string, error) { + var uUID uuid.UUID + var err error + if uUID, err = uuid.NewRandom(); err != nil { + return "", err + } + return strings.ReplaceAll(uUID.String(), "-", ""), nil + } + var handlerFunc http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + + var err error + var requestID string + if requestID, err = generateRequestID(); err != nil { + log.LogErrorf("traceMiddleware: generate request ID fail, remote(%v) url(%v) err(%v)", + r.RemoteAddr, r.URL.String(), err) + _ = InternalError.ServeResponse(w, r) + return + } + mux.Vars(r)[ctxKeyRequestID] = requestID + w.Header().Set(HeaderNameRequestId, requestID) + + var startTime = time.Now() + + next.ServeHTTP(w, r) + + var headerToString = func(header http.Header) string { + var sb = strings.Builder{} + for k := range header { + if sb.Len() != 0 { + sb.WriteString(",") + } + sb.WriteString(fmt.Sprintf("%o:[%o]", k, header.Get(k))) + } + return "{" + sb.String() + "}" + } + + log.LogDebugf("traceMiddleware: trace request:\n"+ + " requestID(%v) host(%v) method(%v) url(%v)\n"+ + " header(%v)\n"+ + " remote(%v) cost(%v)", + requestID, r.Host, r.Method, r.URL.String(), headerToString(r.Header), getRequestIP(r), time.Since(startTime)) + + } + return handlerFunc +} + +func (o *ObjectNode) authMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + // 1. check auth type + if isSignaturedV4(r) { + if ok, _ := o.checkSignatureV4(r); !ok { + if err := AccessDenied.ServeResponse(w, r); err != nil { + log.LogErrorf("authMiddleware: serve access denied response fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err) + } + return + } + } else if isSignaturedV2(r) { + if ok, _ := o.checkSignatureV2(r); !ok { + if err := AccessDenied.ServeResponse(w, r); err != nil { + log.LogErrorf("authMiddleware: serve access denied response fail, requestID(%v) err(%v)", RequestIDFromRequest(r), err) + } + return + } + } else if isPresignedSignaturedV2(r) { + if ok, _ := o.checkPresignedSignatureV2(r); !ok { + log.LogDebugf("authMiddleware: presigned v2 denied: requestID(%v)", RequestIDFromRequest(r)) + if err := AccessDenied.ServeResponse(w, r); err != nil { + log.LogErrorf("authMiddleware: serve response fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + } + return + } + } else if isPresignedSignaturedV4(r) { + if ok, _ := o.checkPresignedSignatureV4(r); !ok { + log.LogDebugf("authMiddleware: presigned v4 denied: requestID(%v)", RequestIDFromRequest(r)) + if err := AccessDenied.ServeResponse(w, r); err != nil { + log.LogErrorf("authMiddleware: serve response fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + } + return + } + } else { + if err := AccessDenied.ServeResponse(w, r); err != nil { + log.LogErrorf("authMiddleware: serve response fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + } + return + } + + next.ServeHTTP(w, r) + }) +} + +func (o *ObjectNode) contentMiddleware(next http.Handler) http.Handler { + var handlerFunc http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + if len(r.Header) > 0 && len(r.Header.Get(http.CanonicalHeaderKey(HeaderNameDecodeContentLength))) > 0 { + r.Body = NewChunkedReader(r.Body) + log.LogDebugf("contentMiddleware: chunk reader inited: requestID(%v)", RequestIDFromRequest(r)) + } + next.ServeHTTP(w, r) + } + return handlerFunc +} diff --git a/objectnode/auth.go b/objectnode/auth.go new file mode 100644 index 000000000..f98bcff75 --- /dev/null +++ b/objectnode/auth.go @@ -0,0 +1,51 @@ +// Package s3 provides ... +package objectnode + +import "net/http" + +//https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader + +type AuthType string + +const ( + SignatrueV2 AuthType = "signature_v2" + SignatrueV4 = "signature_v4" + PresignedV2 = "presigned_v2" + PresignedV4 = "presigned_v4" +) + +type RequestAuthInfo struct { + authType AuthType + accessKey string +} + +func parseRequestAuthInfo(r *http.Request) *RequestAuthInfo { + auth := new(RequestAuthInfo) + if isSignaturedV2(r) { + auth.authType = SignatrueV2 + ai, _ := parseRequestAuthInfoV2(r) + if ai != nil { + auth.accessKey = ai.accessKeyId + } + } else if isSignaturedV4(r) { + auth.authType = SignatrueV4 + ai, _ := parseRequestV4(r) + if ai != nil { + auth.accessKey = ai.Credential.AccessKey + } + } else if isPresignedSignaturedV2(r) { + auth.authType = PresignedV2 + ai, _ := parsePresignedV2AuthInfo(r) + if ai != nil { + auth.accessKey = ai.accessKeyId + } + } else if isPresignedSignaturedV4(r) { + auth.authType = PresignedV4 + ai, _ := parseRequestV4(r) + if ai != nil { + auth.accessKey = ai.Credential.AccessKey + } + } + + return auth +} diff --git a/objectnode/auth_signature_v2.go b/objectnode/auth_signature_v2.go new file mode 100644 index 000000000..7da573f03 --- /dev/null +++ b/objectnode/auth_signature_v2.go @@ -0,0 +1,394 @@ +// Package s3 provides ... +package objectnode + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "errors" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/chubaofs/chubaofs/util" + "github.com/chubaofs/chubaofs/util/log" + "github.com/gorilla/mux" +) + +//https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader + +const ( + RequestHeaderV2Authorization = "Authorization" + RequestHeaderV2AuthorizationScheme = "AWS" + RequestHeaderV2XAmzDate = "X-Amz-Date" +) + +var PresignedSignatureV2Queries = []string{ + "AWSAccessKeyId", + "Signature", +} + +var SignatureV2WhiteQueries = map[string]struct{}{ + "acl": struct{}{}, + "delete": struct{}{}, + "lifecycle": struct{}{}, + "location": struct{}{}, + "logging": struct{}{}, + "notification": struct{}{}, + "partNumber": struct{}{}, + "policy": struct{}{}, + "requestPayment": struct{}{}, + "response-cache-control": struct{}{}, + "response-content-disposition": struct{}{}, + "response-content-encoding": struct{}{}, + "response-content-language": struct{}{}, + "response-content-type": struct{}{}, + "response-expires": struct{}{}, + "torrent": struct{}{}, + "uploadId": struct{}{}, + "uploads": struct{}{}, + "versionId": struct{}{}, + "versioning": struct{}{}, + "versions": struct{}{}, +} + +// +type requestAuthInfoV2 struct { + r *http.Request + authType AuthType + bucket string + accessKeyId string + signature string + expires string +} + +// http://127.0.0.1:33032/ltptest/b.txt +// ?AWSAccessKeyId=Yqnqp4v6q1fzNM2e +// &Expires=1573369185 +// &Signature=GJCqOY0ahf1BdzJDjNnFWB7vfSc%3D +// +func parsePresignedV2AuthInfo(r *http.Request) (*requestAuthInfoV2, error) { + // + ai := new(requestAuthInfoV2) + uris := strings.SplitN(r.RequestURI, "?", 2) + if len(uris) < 2 { + log.LogInfof("checkPresignedSignatureV2 error, request url invalid %v ", r.RequestURI) + return nil, errors.New("uri is invalid") + } + + vars := mux.Vars(r) + ai.accessKeyId = vars["accessKey"] + ai.signature = vars["signature"] + ai.expires = vars["expires"] + + return ai, nil +} + +// Authorization: AWS AWSAccessKeyId:Signature +func parseRequestAuthInfoV2(r *http.Request) (ra *requestAuthInfoV2, err error) { + ra = &requestAuthInfoV2{r: r} + + vars := mux.Vars(r) + ra.bucket = vars["bucket"] + + authStr := r.Header.Get(RequestHeaderV2Authorization) + if authStr == "" { + err = errors.New("header not found authentication") + return nil, err + } + + if !strings.HasPrefix(authStr, RequestHeaderV2AuthorizationScheme) { + return nil, errors.New("header has no prefix ") + } + + credentialStr := util.SubString(authStr, len(RequestHeaderV2AuthorizationScheme), len(authStr)) + credentialStr = strings.Trim(credentialStr, " ") + credentials := strings.Split(credentialStr, ":") + if len(credentials) < 2 { + err = errors.New("") + return nil, err + } + + ra.accessKeyId = credentials[0] + ra.signature = credentials[1] + + return +} + +func isSignaturedV2(r *http.Request) bool { + _, ok1 := r.Header[HeaderNameAuthorization] + _, ok2 := r.Header["X-Amz-Content-Sha256"] + if ok1 && !ok2 { + log.LogDebugf("[handleHttpRestAPI] invalid request, has no authorization info, request id [%s]", r.URL.EscapedPath()) + return true + } + + return false +} + +func isRequestQueryValid(queries url.Values, neededQueries []string) bool { + for _, q := range neededQueries { + k := strings.ToLower(q) + if _, ok := queries[k]; !ok { + return false + } + } + return true +} + +// http://127.0.0.1:33032/ltptest/b.txt +// ?AWSAccessKeyId=Yqnqp4v6q1fzNM2e +// &Expires=1573369185 +// &Signature=GJCqOY0ahf1BdzJDjNnFWB7vfSc%3D +// +func isPresignedSignaturedV2(r *http.Request) bool { + if u, err := url.Parse(strings.ToLower(r.URL.String())); err == nil { + return isRequestQueryValid(u.Query(), PresignedSignatureV2Queries) + } + return false +} + +// +func (o *ObjectNode) checkSignatureV2(r *http.Request) (bool, error) { + // parse v2 request header and query, and get reqSignature + authInfo, err := parseRequestAuthInfoV2(r) + if err != nil { + log.LogInfof("parseRequestAuthInfoV2 error: %v, %v", authInfo.r, err) + return false, err + } + + v, err := o.vm.Volume(authInfo.bucket) + if err != nil { + log.LogInfof("load Volume error: %v, %v", authInfo.r, err) + return false, err + } + volAccessKey, volSecret := v.OSSSecure() + + if authInfo.accessKeyId != volAccessKey { + log.LogInfof("load Volume error: %v, %v", authInfo.accessKeyId, volAccessKey) + return false, errors.New("") + } + + // 2. calculate new signature + newSignature, err1 := calculateSignatureV2(authInfo, volSecret, o.domains) + if err1 != nil { + log.LogInfof("calculute SignatureV2 error: %v, %v", authInfo.r, err) + return false, err1 + } + + // 3. compare newSignatrue and reqSignature + if authInfo.signature == newSignature { + return true, nil + } + log.LogInfof("newSignature: %v, reqSignature: %v, %v", newSignature, authInfo.signature, authInfo.r) + + return false, nil +} + +func calculateSignatureV2(authInfo *requestAuthInfoV2, secretKey string, domains []string) (signature string, err error) { + + //encodedResource := strings.Split(authInfo.r.RequestURI, "?")[0] + canonicalResource, err1 := getCanonicalizedResourceV2(authInfo.r, domains) + if err1 != nil { + return "", err1 + } + + canonicalResourceQuery := getCanonicalQueryV2(canonicalResource, authInfo.r.URL.Query().Encode()) + + date := authInfo.r.Header.Get("Date") + method := authInfo.r.Method + canonicalHeaders := canonicalizedAmzHeadersV2(authInfo.r.Header) + if len(canonicalHeaders) > 0 { + canonicalHeaders += "\n" + } + contentHash := authInfo.r.Header.Get(HeaderNameContentMD5) + contentEnc := authInfo.r.Header.Get(HeaderNameContentEnc) + stringToSign := strings.Join([]string{ + method, + contentHash, + contentEnc, + date, + canonicalHeaders, + }, "\n") + + stringToSign = stringToSign + canonicalResourceQuery + + hm := hmac.New(sha1.New, []byte(secretKey)) + hm.Write([]byte(stringToSign)) + + signature = base64.StdEncoding.EncodeToString(hm.Sum(nil)) + + return +} + +// +// +func (o *ObjectNode) checkPresignedSignatureV2(r *http.Request) (bool, error) { + // + uris := strings.SplitN(r.RequestURI, "?", 2) + if len(uris) < 2 { + log.LogInfof("checkPresignedSignatureV2 error, request url invalid %v ", r.RequestURI) + return false, nil + } + + params, _, _, vl, err := o.parseRequestParams(r) + if err != nil || vl == nil { + log.LogInfof("check PresignedSignatureV2 error: %v %v", err, vl) + return false, err + } + accessKey := params["accessKey"] + signature := params["signature"] + expires := params["expires"] + if accessKey == "" || signature == "" || expires == "" { + log.LogInfof("checkPresignedSignatureV2 params not valid: %v", params) + return false, nil + } + + log.LogDebugf("checkPresignedSignatureV2: parse signature info: requestID(%v) url(%v) accessKey(%v) signature(%v) expires(%v)", + RequestIDFromRequest(r), r.URL.String(), accessKey, signature, expires) + + //check access key + vlKey, secretKey := vl.OSSSecure() + if vlKey == "" || accessKey != vlKey { + return false, nil + } + + // check expires + if ok, _ := checkExpires(expires); !ok { + log.LogDebugf("checkPresignedSignatureV2: signature expired: requestID(%v) expires(%v)", RequestIDFromRequest(r), expires) + return false, nil + } + + //calculatePresignedSignature + uri := strings.Split(r.RequestURI, "?")[0] + canoncialResourceQuery := getCanonicalQueryV2(uri, r.URL.Query().Encode()) + calSignature := calPresignedSignatureV2(r.Method, canoncialResourceQuery, expires, secretKey, r.Header) + if calSignature != signature { + log.LogDebugf("checkPresignedSignatureV2: invalid signature: requestID(%v) client(%v) server(%v)", + RequestIDFromRequest(r), signature, calSignature) + return false, nil + } + + return true, nil +} + +func checkExpires(expires string) (ok bool, err error) { + expiresInt, err := strconv.ParseInt(expires, 10, 64) + if err != nil { + return false, err + } + now := time.Now().UTC().Unix() + if now < expiresInt { + log.LogInfof("checkPresignedSignatureV2 expired is out time %v, now: %v", expires, now) + return true, nil + } + + return false, nil +} + +func getCanonicalQueryV2(encodeResource string, encodeQuery string) string { + var canonicalQueries []string + items := strings.Split(encodeQuery, "&") + queries := make(map[string]string) + for _, item := range items { + k := item + v := "" + i := strings.Index(item, "=") + if i != -1 { + k = item[:i] + v = item[i+1:] + } + queries[k] = v + } + + for k, v := range queries { + if _, ok := SignatureV2WhiteQueries[k]; !ok { + continue + } + + query := k + if v != "" { + query = k + "=" + v + } + canonicalQueries = append(canonicalQueries, query) + } + + sort.Strings(canonicalQueries) + + canonicalQuery := strings.Join(canonicalQueries, "&") + if canonicalQuery != "" { + return encodeResource + "?" + canonicalQuery + } + + if encodeResource == "/" { + return "" + } + + return encodeResource +} + +// +func canonicalizedAmzHeadersV2(headers http.Header) string { + var keys []string + keyval := make(map[string]string) + for key := range headers { + lkey := strings.ToLower(key) + if !strings.HasPrefix(lkey, "x-amz-") { + continue + } + keys = append(keys, lkey) + keyval[lkey] = strings.Join(headers[key], ",") + } + sort.Strings(keys) + var canonicalHeaders []string + for _, key := range keys { + canonicalHeaders = append(canonicalHeaders, key+":"+keyval[key]) + } + return strings.Join(canonicalHeaders, "\n") +} + +// +func calPresignedSignatureV2(method, canonicalQuery, expires, secretKey string, header http.Header) string { + date := expires + if date == "" { + date = header.Get(HeaderNameDate) + } + canonicalHeaders := canonicalizedAmzHeadersV2(header) + contentHash := header.Get(HeaderNameContentMD5) + contentEnc := header.Get(HeaderNameContentEnc) + stringToSign := strings.Join([]string{ + method, + contentHash, + contentEnc, + date, + canonicalHeaders, + }, "\n") + canonicalQuery + + hm := hmac.New(sha1.New, []byte(secretKey)) + hm.Write([]byte(stringToSign)) + + return base64.StdEncoding.EncodeToString(hm.Sum(nil)) +} + +func getCanonicalizedResourceV2(r *http.Request, domains []string) (resource string, err error) { + path := strings.Split(r.RequestURI, "?")[0] + if len(domains) > 0 { + for _, d := range domains { + if !strings.HasSuffix(r.Host, "."+d) { + continue + } + vars := mux.Vars(r) + bucket := vars["bucket"] + resource = "/" + bucket + path + return + } + + } else { + resource = path + } + + return +} diff --git a/objectnode/auth_signature_v4.go b/objectnode/auth_signature_v4.go new file mode 100644 index 000000000..8ea3560d6 --- /dev/null +++ b/objectnode/auth_signature_v4.go @@ -0,0 +1,577 @@ +package objectnode + +import ( + "encoding/hex" + "errors" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/chubaofs/chubaofs/util" + "github.com/chubaofs/chubaofs/util/log" + "github.com/gorilla/mux" +) + +const ( + MaxPresignedExpires = 3 * 365 * 24 * 60 * 60 //10years + DateFormatISO8601 = "20060102T150405Z" //"yyyyMMddTHHmmssZ" + MaxSkewTime = 15 * time.Minute + + XAmzContentSha256 = "X-Amz-Content-Sha256" + XAmzCredential = "X-Amz-Credential" + XAmzSignature = "X-Amz-Signature" // + XAmzSignedHeaders = "X-Amz-SignedHeaders" + XAmzAlgorithm = "X-Amz-Algorithm" + XAmzDate = "X-Amz-Date" + XAmzExpires = "X-Amz-Expires" + + SignatureV4Algorithm = "AWS4-HMAC-SHA256" + SignatureV4Request = "aws4-request" + SignedHeaderHost = "host" + UnsignedPayload = "UNSIGNED-PAYLOAD" + PresignedV4QueryAuth = "Authorization" + + credentialFlag = "Credential=" + signatureFlag = "Signature=" + signedHeadersFlag = "SignedHeaders=" +) + +var ( + InvalidParamError = errors.New("invalid param") + MaxExpiresError = errors.New("max expires value") +) + +var PresignedSignatureV4Queries = []string{ + XAmzCredential, + XAmzSignature, +} + +var AuthSignatureV4Headers = []string{ + PresignedV4QueryAuth, + XAmzContentSha256, +} + +// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +// url: "127.0.0.1:9000/umptest/b.tinyExtents +// ?X-Amz-Algorithm=AWS4-HMAC-SHA256_CMD +// &X-Amz-Credential=/20130721/us-east-1/s3/aws4_request +// &X-Amz-Date=20191111T112557Z +// &X-Amz-Expires=432000 +// &X-Amz-SignedHeaders=host +// &X-Amz-Signature=55c588734f017b861c24cbd69c203c283aad566cfe4ee712b7a3c846e1de151a" +// +func isPresignedSignaturedV4(r *http.Request) bool { + if u, err := url.Parse(strings.ToLower(r.URL.String())); err == nil { + return isRequestQueryValid(u.Query(), PresignedSignatureV4Queries) + } + return false +} + +// is request signature V4 +func isSignaturedV4(r *http.Request) bool { + for _, h := range AuthSignatureV4Headers { + _, ok := r.Header[h] + if !ok { + return false + } + } + + return true +} + +// check request signature valid +func (o *ObjectNode) checkSignatureV4(r *http.Request) (bool, error) { + _, _, _, vl, _ := o.parseRequestParams(r) + if vl == nil { + log.LogInfof("[handleHttpRestAPI] invalid request, has no authorization info, request id [%o]", r.URL.EscapedPath()) + return false, nil + } + _, secretKey := vl.OSSSecure() + req, err := parseRequestV4(r) + if err != nil { + return false, err + } + newSignature := calculateSignatureV4(r, o.region, secretKey, req.SignedHeaders) + if req.Signature != newSignature { + log.LogDebugf("checkSignatureV4: invalid signature: requestID(%v) client(%v) server(%v)", + RequestIDFromRequest(r), req.Signature, newSignature) + return false, nil + } + + return true, nil +} + +// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +// url: "127.0.0.1:9000/umptest/b.tinyExtents +// ?X-Amz-Algorithm=AWS4-HMAC-SHA256_CMD +// &X-Amz-Credential=/20130721/us-east-1/s3/aws4_request +// &X-Amz-Date=20191111T112557Z +// &X-Amz-Expires=432000 +// &X-Amz-SignedHeaders=host +// &X-Amz-Signature=55c588734f017b861c24cbd69c203c283aad566cfe4ee712b7a3c846e1de151a" +// +func (o *ObjectNode) checkPresignedSignatureV4(r *http.Request) (pass bool, err error) { + var req *signatureRequestV4 + req, err = parseRequestV4(r) + if err != nil { + log.LogErrorf("checkPresignedSignatureV4: parse request fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + return + } + + //check req valid + var ok bool + if ok, err = req.isValid(); !ok { + log.LogErrorf("checkPresignedSignatureV4: request invalid: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + return + } + + // check accessKey valid + var v Volume + v, err = o.vm.Volume(req.bucket) + if err != nil { + log.LogErrorf("checkPresignedSignatureV4: get volume fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + return + } + //check accesskey + vaKey, secretKey := v.OSSSecure() + if req.Credential.AccessKey != vaKey { + log.LogInfof("checkPresignedSignatureV4: credential accessKey invalid: requestID(%v) requestAK(%v) volAK(%v)", + RequestIDFromRequest(r), req.Credential.AccessKey, vaKey) + err = errors.New("accesskey invalid") + return + } + // create canonicalRequest + var canonicalHeader http.Header + canonicalHeader, err = req.createCanonicalHeaderV4() + if err != nil { + log.LogErrorf("checkPresignedSignatureV4: create canonical header fail: requestID(%v) err(%v)", RequestIDFromRequest(r), err) + return + } + canonicalHeaderStr := buildCanonicalHeaderString(r.Host, canonicalHeader, req.SignedHeaders) + headerNames := getCanonicalHeaderNames(req.SignedHeaders) + payload := UnsignedPayload + canonicalQuery := createCanonicalQueryV4(req) + canonicalRequestString := createCanonicalRequestString(r.Method, req.URI, canonicalQuery, canonicalHeaderStr, headerNames, payload) + + log.LogDebugf("checkPresignedSignatureV4: middle data:\n"+ + " RequestID: %v\n"+ + " CanonicalRequest: %v", + RequestIDFromRequest(r), + canonicalRequestString) + + // build signingKey + signingKey := buildSigningKey(SCHEME, secretKey, req.Credential.Date, req.Credential.Region, req.Credential.Service, req.Credential.Request) + + // build stringToSign + scope := buildScope(req.Credential.Date, req.Credential.Region, req.Credential.Service, req.Credential.Request) + stringToSign := buildStringToSign(req.Algorithm, req.Timestamp, scope, canonicalRequestString) + + //sign stringToSign with signingKey + newSignature := hex.EncodeToString(sign(stringToSign, signingKey)) + + //compare newSignature with request signature + pass = newSignature == req.Signature + return +} + +type credential struct { + AccessKey string + Date string + Region string + Service string //s3 + Request string +} + +type signatureRequestV4 struct { + r *http.Request + bucket string + URI string + Algorithm string + Timestamp string + Expires string + Signature string + SignedHeaders []string + Credential credential +} + +func (c *credential) GetScopeString() string { + return strings.Join([]string{ + c.Date, + c.Region, + c.Service, + c.Request, + }, "/") +} + +//get presignedReq query +func (req *signatureRequestV4) Query() url.Values { + return req.r.URL.Query() +} + +// get Timestamp +func (req *signatureRequestV4) GetTimestamp() (time.Time, error) { + return time.Parse(DateFormatISO8601, req.Timestamp) +} + +// get +func (req *signatureRequestV4) GetExpires() (time.Duration, error) { + return time.ParseDuration(req.Expires + "s") +} + +// +func (req *signatureRequestV4) isValid() (bool, error) { + expires, err := req.GetExpires() + if err != nil { + return false, errors.New("expires is invalid ") + } + if expires < 0 { + return false, errors.New("expires < 0 ") + } + if expires.Seconds() > MaxPresignedExpires { + return false, errors.New("expires > MaxPresignedExpires ") + } + utcNow := time.Now().UTC() + ts, err1 := req.GetTimestamp() + if err1 != nil { + return false, errors.New("expires is invalid ") + } + if ts.After(utcNow.Add(MaxSkewTime)) { + return false, errors.New("req date invalid ") + } + if utcNow.Sub(ts) > expires { + return false, errors.New("expires time out ") + } + + return true, nil +} + +// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html +// +// Authorization: AWS4-HMAC-SHA256\nCredential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-date, +// Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024 +// +func (req *signatureRequestV4) parseRequestHeaderV4(r *http.Request) (err error) { + authorizationValue := r.Header.Get(HeaderNameAuthorization) + if authorizationValue != "" { + authorizationValue = strings.ReplaceAll(authorizationValue, " ", "") + credentialIndex := strings.Index(authorizationValue, credentialFlag) + algorithm := util.SubString(authorizationValue, len(HeaderNameAuthorization)+1, credentialIndex) + if algorithm != "" { + req.Algorithm = algorithm + } + credentialSignatureStr := util.SubString(authorizationValue, credentialIndex+len(credentialFlag), len(authorizationValue)) + authorizationValues := strings.Split(credentialSignatureStr, ",") + if len(authorizationValues) < 2 { + log.LogInfof("decode signature error: %v ", authorizationValue) + return errors.New("request header authorization parse error") + } + + credentialStr := authorizationValues[0] + if credentialStr != "" { + req.parseCredential(credentialStr) + } + signedHeadersStr := authorizationValues[1] + signatureStr := authorizationValues[2] + req.Signature = util.SubString(signatureStr, len(signatureFlag), len(signatureStr)) + + signedHeadersStr = util.SubString(signedHeadersStr, len(signedHeadersFlag), len(signedHeadersStr)) + req.SignedHeaders = strings.Split(signedHeadersStr, ";") + } + return +} + +func (req *signatureRequestV4) parseCredential(credentialStr string) error { + credentialStr = strings.TrimPrefix(credentialStr, credentialFlag) + item := strings.Split(credentialStr, "/") + if len(item) < 5 { + return errors.New("x-amz-credential params invalid") + } + req.Credential = credential{ + AccessKey: item[0], + Date: item[1], + Region: item[2], + Service: item[3], + Request: item[4], + } + return nil +} + +func (req *signatureRequestV4) parseRequestQueryV4(r *http.Request) (err error) { + q := r.URL.Query() + vars := mux.Vars(r) + bucket := vars["bucket"] + if bucket != "" { + req.bucket = bucket + } + algorithm := q.Get(XAmzAlgorithm) + if algorithm != "" { + req.Algorithm = algorithm + } + timestamp := q.Get(XAmzDate) + if timestamp != "" { + req.Timestamp = timestamp + if _, err = req.GetTimestamp(); err != nil { + return err + } + } + expires := q.Get(XAmzExpires) + if expires != "" { + req.Expires = expires + if _, err = req.GetExpires(); err != nil { + return err + } + } + credentialStr := q.Get(XAmzCredential) + if credentialStr != "" { + req.parseCredential(credentialStr) + } + signatrue := q.Get(XAmzSignature) + if signatrue != "" { + req.Signature = signatrue + } + signedHeaders := q.Get(XAmzSignedHeaders) + if signedHeaders != "" { + req.SignedHeaders = strings.Split(signedHeaders, ";") + } + return +} + +// getPresignedV4 encodeQuery +// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +// url: "127.0.0.1:9000/umptest/b.tinyExtents +// ?X-Amz-Algorithm=AWS4-HMAC-SHA256_CMD +// &X-Amz-Credential=/20130721/us-east-1/s3/aws4_request +// &X-Amz-Date=20191111T112557Z +// &X-Amz-Expires=432000 +// &X-Amz-SignedHeaders=host +// &X-Amz-Signature=55c588734f017b861c24cbd69c203c283aad566cfe4ee712b7a3c846e1de151a" +func parseRequestV4(r *http.Request) (req *signatureRequestV4, err error) { + req = &signatureRequestV4{} + req.r = r + + uri := strings.Split(r.RequestURI, "?")[0] + if uri != "" { + req.URI = uri + } else { + req.URI = "/" + } + + //parseHeader + err = req.parseRequestHeaderV4(r) + if err != nil { + return + } + + //parseQuery + err = req.parseRequestQueryV4(r) + if err != nil { + return + } + + return +} + +// create canonical query not contain X-Amz-Signature query +func createCanonicalQueryV4(req *signatureRequestV4) string { + newQuery := make(url.Values) + newQuery.Set(XAmzAlgorithm, req.Algorithm) + newQuery.Set(XAmzExpires, req.Expires) + newQuery.Set(XAmzDate, req.Timestamp) + newQuery.Set(XAmzSignedHeaders, req.Query().Get(XAmzSignedHeaders)) + newQuery.Set(XAmzCredential, req.Query().Get(XAmzCredential)) + hashContent := req.Query().Get(XAmzContentSha256) + if hashContent != "" { + newQuery.Set(XAmzContentSha256, hashContent) + } else { + } + + for k, v := range req.Query() { + key := strings.ToLower(k) + if strings.Contains(key, "x-amz-meta-") { + newQuery.Set(k, v[0]) + continue + } + if strings.Contains(key, "x-amz-server-side-") { + newQuery.Set(k, v[0]) + } + if strings.HasPrefix(key, "x-amz") { + continue + } + newQuery[k] = v + } + + return newQuery.Encode() +} + +func buildSigningKey(scheme, secret, date, region, service, terminator string) []byte { + secretKey := []byte(scheme + secret) + dateKey := sign(date, secretKey) + dateRegionKey := sign(region, dateKey) + dateRegionServiceKey := sign(service, dateRegionKey) + signingKey := sign(terminator, dateRegionServiceKey) + return signingKey +} + +func getContentHash(headers http.Header) (contentHash string) { + for headerName := range headers { + if strings.ToLower(headerName) == strings.ToLower(HeaderNameContentHash) { + contentHash = headers.Get(headerName) + break + } + } + return +} + +func getEncodeQuery(r *http.Request) string { + return r.URL.Query().Encode() +} + +// calculete signature v4 +func calculateSignatureV4(r *http.Request, region, secretKey string, signedHeaders []string) string { + headers := r.Header + + // get request start time in ISO8601 type + canonicalHeaderString := buildCanonicalHeaderString(r.Host, headers, signedHeaders) + headerNames := getCanonicalHeaderNames(signedHeaders) + contentHash := getContentHash(headers) + encodeQuery := getEncodeQuery(r) + canonicalURI := getCanonicalURI(r) + canonicalRequest := createCanonicalRequestString(r.Method, canonicalURI, encodeQuery, canonicalHeaderString, headerNames, contentHash) + + dateStamp := getCurrentDateStamp() + timestamp := getStartTime(headers) + signingKey := buildSigningKey(SCHEME, secretKey, dateStamp, region, SERVICE, TERMINATOR) + scope := buildScope(dateStamp, region, SERVICE, TERMINATOR) + stringToSign := buildStringToSign(SignatureV4Algorithm, timestamp, scope, canonicalRequest) + signature := sign(stringToSign, signingKey) + + log.LogDebugf("calculateSignatureV4: middle data:\n"+ + " RequestID: %v\n"+ + " CanonicalRequest: %v", + RequestIDFromRequest(r), + canonicalRequest) + return hex.EncodeToString(signature) +} + +func getCanonicalURI(r *http.Request) string { + return strings.Split(r.URL.RequestURI(), "?")[0] +} + +// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html +func createCanonicalRequestString(httpMethod, canonicalURI, encodeQuery, canonicalHeaders, headerName, hashedPayload string) string { + canonicalQueryString := strings.Replace(encodeQuery, "+", "%20", -1) + return strings.Join([]string{ + httpMethod, + canonicalURI, + canonicalQueryString, + canonicalHeaders, + headerName, + hashedPayload, + }, "\n") +} + +// get headerNames string from signedHeaders +// 1. sort headers +// 2. lower +// 3. join with ; +func getCanonicalHeaderNames(signedHeaders []string) (headerName string) { + sort.Strings(signedHeaders) + + lowerSignedHeaders := make([]string, 0) + for _, headerName := range signedHeaders { + lowerSignedHeaders = append(lowerSignedHeaders, headerName) + } + headerName = strings.Join(lowerSignedHeaders, ";") + return +} + +func contains(items []string, key string) bool { + for _, s := range items { + if s == key { + return true + } + } + return false +} + +// canonical header +func (req *signatureRequestV4) createCanonicalHeaderV4() (canonicalHeader http.Header, err error) { + if !contains(req.SignedHeaders, SignedHeaderHost) { + return nil, errors.New("signedHeaders not contain host") + } + canonicalHeader = make(http.Header) + reqHeader := req.r.Header + reqQuery := req.r.URL.Query() + for _, header := range req.SignedHeaders { + vals, ok := reqHeader[http.CanonicalHeaderKey(header)] // + if !ok { + vals, ok = reqQuery[header] // + } + if ok { + for _, val := range vals { + canonicalHeader.Add(header, val) + } + continue + } + switch strings.ToLower(header) { + case "content-length": + canonicalHeader.Set(header, strconv.FormatInt(req.r.ContentLength, 10)) + case "expect": + canonicalHeader.Set(header, "100-continue") + case "transfer-encoding": + canonicalHeader.Set(header, req.r.Host) + default: + return nil, nil + } + } + + return +} + +// filter request headers by signedHeaders then gen join to a string +func buildCanonicalHeaderString(host string, headers http.Header, signedHeaders []string) (headerString string) { + // copy a new http header from headers, because net/http package remove host header when parsing request + newHeaders := make(http.Header) + for n := range headers { + newHeaders.Add(n, headers.Get(n)) + } + newHeaders.Add(HeaderNameHost, host) + + canonicalHeaders := make([]string, 0) + sort.Strings(signedHeaders) + for _, signedHeaderName := range signedHeaders { + vals, ok := newHeaders[(http.CanonicalHeaderKey(signedHeaderName))] + if ok { + sb := strings.Builder{} + sb.WriteString(strings.ToLower(signedHeaderName)) + sb.WriteString(":") + sb.WriteString(strings.Join(vals, ",")) + canonicalHeaders = append(canonicalHeaders, sb.String()) + } + } + headerString = strings.Join(canonicalHeaders, "\n") + "\n" + return +} + +// build scope string to sign +func buildScope(date, region, service, request string) string { + return strings.Join([]string{ + date, + region, + service, + request, + }, "/") +} + +// build string to sign +func buildStringToSign(algorithm, timeStamp, scope, canonicalRequestString string) string { + return strings.Join([]string{ + algorithm, + timeStamp, + scope, + calcHash(canonicalRequestString), + }, "\n") +} diff --git a/objectnode/auth_signature_v4_test.go b/objectnode/auth_signature_v4_test.go new file mode 100644 index 000000000..f899da2fb --- /dev/null +++ b/objectnode/auth_signature_v4_test.go @@ -0,0 +1 @@ +package objectnode diff --git a/objectnode/chunk.go b/objectnode/chunk.go new file mode 100644 index 000000000..87a4b3bbd --- /dev/null +++ b/objectnode/chunk.go @@ -0,0 +1,188 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "errors" + "fmt" + "io" + "strconv" + "strings" +) + +var ( + ErrUnexpectedChunkEnd = errors.New("unexpected chunk end") + ErrUnexpectedChunkSize = errors.New("unexpected chunk size") +) + +type chunkedReader struct { + readCloser io.ReadCloser + chunkSize int + pos int + bof bool + eof bool +} + +func (r *chunkedReader) Close() error { + return r.readCloser.Close() +} + +func (r *chunkedReader) Read(p []byte) (n int, err error) { + + if r.eof { + n = 0 + err = io.EOF + return + } + + if r.pos >= r.chunkSize { + if err = r.nextChunk(); err != nil { + return + } + if r.eof { + n = 0 + err = io.EOF + return + } + } + length := len(p) + if r.chunkSize-r.pos < length { + length = r.chunkSize - r.pos + } + if n, err = r.readCloser.Read(p[:length]); err != nil { + return + } + r.pos += n + return +} + +func (r *chunkedReader) nextChunk() (err error) { + if !r.bof { + if err = r.readCRLF(); err != nil { + return + } + } + if r.chunkSize, err = r.readChunkSize(); err != nil { + return + } + r.bof = false + r.pos = 0 + if r.chunkSize == 0 { + r.eof = true + } + return +} + +func (r *chunkedReader) readCRLF() (err error) { + + var tmp = make([]byte, 2) + if _, err = r.readCloser.Read(tmp); err != nil { + return + } + if cr, lf := int(tmp[0]), int(tmp[1]); (cr != '\r') || (lf != '\n') { + err = fmt.Errorf("CRLF expected and end of chunk: %v/%v", cr, lf) + return + } + return +} + +func (r *chunkedReader) readByteToInt() (i int, err error) { + var tmp = make([]byte, 1) + if _, err = r.readCloser.Read(tmp); err != nil { + return + } + i = int(tmp[0]) + return +} + +func (r *chunkedReader) readChunkSize() (size int, err error) { + var buf = make([]byte, 0) + var state int + for state != -1 { + var b int + if b, err = r.readByteToInt(); err != nil { + return + } + if b == -1 { + err = ErrUnexpectedChunkEnd + return + } + switch state { + case 0: + switch b { + case '\r': + state = 1 + break + + case '"': + state = 2 + + default: + buf = append(buf, byte(b)) + } + break + + case 1: + if b == '\n' { + state = -1 + } else { + err = ErrUnexpectedChunkSize + return + } + break + + case 2: + switch b { + case '\\': + if b, err = r.readByteToInt(); err != nil { + return + } + buf = append(buf, byte(b)) + break + + case '"': + state = 0 + + default: + buf = append(buf, byte(b)) + } + break + + default: + panic("assertion fail") + } + } + var str = string(buf) + if sepIndex := strings.Index(str, ";"); sepIndex > 0 { + str = str[:sepIndex] + } + str = strings.TrimSpace(str) + + var result int64 + if result, err = strconv.ParseInt(str, 16, 64); err != nil { + return + } + size = int(result) + return +} + +func NewChunkedReader(reader io.ReadCloser) io.ReadCloser { + return &chunkedReader{ + readCloser: reader, + pos: 0, + bof: true, + eof: false, + } +} diff --git a/objectnode/chunk_test.go b/objectnode/chunk_test.go new file mode 100644 index 000000000..e482fe1ff --- /dev/null +++ b/objectnode/chunk_test.go @@ -0,0 +1,65 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "bytes" + "io" + "io/ioutil" + "net/http/httputil" + "testing" +) + +type readCloser struct { + io.Reader +} + +func (r *readCloser) Read(p []byte) (n int, err error) { + return r.Reader.Read(p) +} + +func (readCloser) Close() error { + return nil +} + +func wrapReader(reader io.Reader) io.ReadCloser { + return &readCloser{ + Reader: reader, + } +} + +func TestChunkedReader_ReadChunked(t *testing.T) { + var err error + plain := `Hello world.` + buffer := new(bytes.Buffer) + chunkedWriter := httputil.NewChunkedWriter(buffer) + if _, err = chunkedWriter.Write([]byte(plain)); err != nil { + t.Fatal(err) + } + _ = chunkedWriter.Close() + chunked := string(buffer.Bytes()) + t.Logf("plain:\n%v", plain) + t.Logf("chunked:\n%v", chunked) + + reader := NewChunkedReader(wrapReader(bytes.NewReader([]byte(chunked)))) + result, err := ioutil.ReadAll(reader) + if err != nil { + t.Fatalf("read from chunked readCloser fail: err(%v)", err) + } + if string(result) != plain { + t.Fatalf("result mismatch:\nexpect(%v)\nactual(%v)", plain, string(result)) + } + t.Logf("result: %v", string(result)) +} diff --git a/objectnode/const.go b/objectnode/const.go new file mode 100644 index 000000000..a5ac8fcea --- /dev/null +++ b/objectnode/const.go @@ -0,0 +1,98 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +type OSSOperation string + +const ( + HeaderNameServer = "server" + HeaderNameHost = "Host" + HeaderNameLastModified = "Last-Modified" + HeaderNameETag = "ETag" + HeaderNameDate = "Date" + HeaderNameContentMD5 = "content-md5" + HeaderNameContentEnc = "content-encoding" + HeaderNameContentType = "Content-Type" + HeaderNameContentLength = "Content-Length" + HeaderNameContentRange = "Content-Range" + HeaderNameAuthorization = "Authorization" + HeaderNameAcceptRange = "Accept-Ranges" + HeaderNameRange = "Range" + + HeaderNameStartDate = "x-amz-date" + HeaderNameRequestId = "x-amz-request-id" + HeaderNameContentHash = "X-Amz-Content-SHA256" + HeaderNameCopySource = "X-Amz-Copy-Source" + HeaderNameCopyMatch = "x-amz-copy-source-if-match" + HeaderNameCopyNoneMatch = "x-amz-copy-source-if-none-match" + HeaderNameCopyModified = "x-amz-copy-source-if-modified-since" + HeaderNameCopyUnModified = "x-amz-copy-source-if-unmodified-since" + HeaderNameDecodeContentLength = "X-Amz-Decoded-Content-Length" +) + +const ( + HeaderValueServer = "ChubaoFS" + HeaderValueAcceptRange = "bytes" + HeaderValueTypeStream = "application/octet-stream" + HeaderValueContentTypeXML = "application/xml" +) + +const ( + SubObjectDelete = "delete" + SubMultipartUpload = "uploads" +) + +const ( + ParamUploadId = "uploadId" + ParamPartNumber = "partNumber" + ParamKeyMarker = "key-marker" + ParamMarker = "marker" + ParamPrefix = "prefix" + ParamContToken = "continuation-token" + ParamFetchOwner = "fetch-owner" + ParamMaxKeys = "max-keys" + ParamStartAfter = "start-after" + + ParamMaxParts = "max-parts" + ParamUploadIdMarker = "upload-id-​marker" + ParamPartNoMarker = "part-number-marker" + ParamPartMaxUploads = "max-uploads" + ParamPartDelimiter = "delimiter" +) + +const ( + MaxKeys = 1000 + MaxParts = 1000 + MaxUploads = 1000 +) + +const ( + StorageClassStandard = "Standard" +) + +// XAttr keys for ObjectNode compatible feature +const ( + XAttrKeyOSSETag = "oss:etag" + XAttrKeyOSSTagging = "oss:tg" + XAttrKeyOSSPolicy = "oss:ply" +) + +const ( + AMZTimeFormat = "2006-01-02T15:04:05Z" +) + +const ( + EmptyContentMD5String = "d41d8cd98f00b204e9800998ecf8427e" +) diff --git a/objectnode/fs.go b/objectnode/fs.go new file mode 100644 index 000000000..c55ded2fb --- /dev/null +++ b/objectnode/fs.go @@ -0,0 +1,95 @@ +package objectnode + +import ( + "io" + "os" + "sort" + "time" + + "github.com/chubaofs/chubaofs/proto" +) + +type VolumeManager interface { + Volume(volName string) (Volume, error) + Release(volName string) + GetStore() (Store, error) + InitStore(s Store) + Close() +} + +type FSFileInfo struct { + Path string + Size int64 + Mode os.FileMode + ModifyTime time.Time + ETag string + Inode uint64 +} + +type Prefixes []string + +type PrefixMap map[string]struct{} + +func (m PrefixMap) AddPrefix(prefix string) { + m[prefix] = struct{}{} +} + +func (m PrefixMap) Prefixes() Prefixes { + s := make([]string, 0, len(m)) + for prefix := range m { + s = append(s, prefix) + } + sort.Strings(s) + return s +} + +type FSUpload struct { + Key string + UploadId string + StorageClass string + Initiated string +} + +type FSPart struct { + PartNumber int + LastModified string + ETag string + Size int +} + +type Volume interface { + OSSSecure() (accessKey, secretKey string) + OSSMeta() *OSSMeta + + // ListFiles return an FileInfo slice of specified volume, like read dir for hole volume. + // The result will be ordered by full path. + ListFilesV1(request *ListBucketRequestV1) ([]*FSFileInfo, string, bool, []string, error) + + ListFilesV2(request *ListBucketRequestV2) ([]*FSFileInfo, uint64, string, bool, []string, error) + + // PutObject create file in specified volume with specified path. + WriteFile(path string, reader io.Reader) (*FSFileInfo, error) + + // DeleteFile delete specified file from specified volume. If target is not exists then returns error. + DeleteFile(path string) error + + FileInfo(path string) (*FSFileInfo, error) + + // operation about multipart uploads + InitMultipart(path string) (multipartID string, err error) + WritePart(path, multipartID string, partId uint16, reader io.Reader) (*FSFileInfo, error) + ListParts(path, multipartID string, maxParts, partNumberMarker uint64) ([]*FSPart, uint64, bool, error) + CompleteMultipart(path, multipartID string) (*FSFileInfo, error) + AbortMultipart(path, multipartID string) error + ListMultipartUploads(prefix, delimiter, keyMarker, uploadIdMarker string, maxUploads uint64) ([]*FSUpload, string, string, bool, []string, error) + + ReadFile(path string, writer io.Writer, offset, size uint64) error + + CopyFile(path, sourcePath string) (*FSFileInfo, error) + + SetXAttr(path string, key string, data []byte) error + GetXAttr(path string, key string) (*proto.XAttrInfo, error) + DeleteXAttr(path string, key string) error + + Close() error +} diff --git a/objectnode/fs_manager.go b/objectnode/fs_manager.go new file mode 100644 index 000000000..1f5dd448e --- /dev/null +++ b/objectnode/fs_manager.go @@ -0,0 +1,89 @@ +package objectnode + +import ( + "errors" + "sync" + + "github.com/chubaofs/chubaofs/util/log" +) + +type volumeManager struct { + masters []string + volumes map[string]*volume // volume key -> vol + volMu sync.RWMutex + store Store + closeOnce sync.Once +} + +func (m *volumeManager) Release(volName string) { + panic("implement me") +} + +func (m *volumeManager) ReleaseAll() { + panic("implement me") +} + +func (m *volumeManager) Volume(volName string) (Volume, error) { + return m.loadVolume(volName) +} + +func (m *volumeManager) loadVolume(volName string) (*volume, error) { + var err error + var volume *volume + var exist bool + m.volMu.RLock() + volume, exist = m.volumes[volName] + m.volMu.RUnlock() + if !exist { + m.volMu.Lock() + volume, exist = m.volumes[volName] + if exist { + m.volMu.Unlock() + return volume, nil + } + if volume, err = newVolume(m.masters, volName); err != nil { + m.volMu.Unlock() + return nil, err + } + ak, sk := volume.OSSSecure() + log.LogDebugf("[loadVolume] load volume: Name[%v] AccessKey[%v] SecretKey[%v]", volName, ak, sk) + m.volumes[volName] = volume + volume.vm = m + m.volMu.Unlock() + + volume.loadOSSMeta() + } + + return volume, nil +} + +// Release all +func (m *volumeManager) Close() { + m.volMu.Lock() + defer m.volMu.Unlock() + for volKey, vol := range m.volumes { + _ = vol.Close() + log.LogDebugf("release volume %v", volKey) + } + m.volumes = make(map[string]*volume) +} + +func (m *volumeManager) InitStore(s Store) { + s.Init(m) + m.store = s +} + +func (m *volumeManager) GetStore() (Store, error) { + if m.store == nil { + return nil, errors.New("store not init") + } + return m.store, nil +} + +func NewVolumeManager(masters []string) VolumeManager { + vc := &volumeManager{ + volumes: make(map[string]*volume), + masters: masters, + } + return vc +} diff --git a/objectnode/fs_store.go b/objectnode/fs_store.go new file mode 100644 index 000000000..0eaa7e497 --- /dev/null +++ b/objectnode/fs_store.go @@ -0,0 +1,13 @@ +package objectnode + +type MetaStore interface { +} + +// MetaStore +type Store interface { + Init(vm *volumeManager) + Put(ns, obj, key string, data []byte) error + Get(ns, obj, key string) (data []byte, err error) + List(ns, obj string) (data [][]byte, err error) + Delete(ns, obj, key string) error +} diff --git a/objectnode/fs_store_authnode.go b/objectnode/fs_store_authnode.go new file mode 100644 index 000000000..f71a0ea12 --- /dev/null +++ b/objectnode/fs_store_authnode.go @@ -0,0 +1,33 @@ +package objectnode + +type authnodeStore struct { + vm *volumeManager //vol *volume +} + +func (s *authnodeStore) Init(vm *volumeManager) { + s.vm = vm + //TODO: init authnode store +} + +func (s *authnodeStore) Get(vol, path, key string) (val []byte, err error) { + + return +} + +func (s *authnodeStore) Put(vol, path, key string, data []byte) (err error) { + //TODO: implement authonode store put method + + return nil +} + +func (s *authnodeStore) Delete(vol, path, key string) (err error) { + //TODO: implement authonode store put method + + return +} + +func (s *authnodeStore) List(vol, path string) (data [][]byte, err error) { + //TODO: implement authonode store list method + + return +} diff --git a/objectnode/fs_store_object.go b/objectnode/fs_store_object.go new file mode 100644 index 000000000..fecf13990 --- /dev/null +++ b/objectnode/fs_store_object.go @@ -0,0 +1,36 @@ +package objectnode + +import ( + "github.com/chubaofs/chubaofs/util/log" +) + +const ( + META_OSS_VOLUME = ".oss_meta" +) + +type objectStore struct { + vm *volumeManager +} + +func (s *objectStore) Init(vm *volumeManager) { + s.vm = vm + //TODO: init meta dir +} + +func (s *objectStore) Put(vol, obj, key string, data []byte) (err error) { + log.LogInfo("put object store") + + return +} + +func (s *objectStore) Get(vol, obj, key string) (data []byte, err error) { + return +} + +func (s *objectStore) Delete(vol, obj, key string) (err error) { + return +} + +func (s *objectStore) List(vol, obj string) (data [][]byte, err error) { + return +} diff --git a/objectnode/fs_store_xattr.go b/objectnode/fs_store_xattr.go new file mode 100644 index 000000000..f28239306 --- /dev/null +++ b/objectnode/fs_store_xattr.go @@ -0,0 +1,89 @@ +package objectnode + +import ( + "strings" + + "github.com/chubaofs/chubaofs/proto" + + "github.com/chubaofs/chubaofs/util/log" +) + +const ( + volumeRootInode = uint64(1) +) + +type xattrStore struct { + vm *volumeManager //vol *volume +} + +func (s *xattrStore) Init(vm *volumeManager) { + s.vm = vm +} + +func (s *xattrStore) getInode(vol, path string) (*volume, uint64, error) { + v, err := s.vm.loadVolume(vol) + if err != nil { + return nil, 0, err + } + inode := volumeRootInode + if path != "" && path != "/" { + items := strings.Split(path, "/") + for _, item := range items { + if item == "" { + continue + } + inode, _, err = v.mw.Lookup_ll(inode, item) + if err != nil { + return v, inode, err + } + } + } + return v, inode, nil +} + +func (s *xattrStore) Put(vol, path, key string, data []byte) (err error) { + v, err1 := s.vm.loadVolume(vol) + if err1 != nil { + err = err1 + return + } + err = v.SetXAttr(path, key, data) + if err != nil { + log.LogErrorf("policy: %v, %v", key, data) + } + return +} + +func (s *xattrStore) Get(vol, path, key string) (val []byte, err error) { + var v *volume + v, err = s.vm.loadVolume(vol) + if err != nil { + return + } + + var xattrInfo *proto.XAttrInfo + if xattrInfo, err = v.GetXAttr(path, key); err != nil { + return + } + if xattrInfo == nil { + return + } + + var strVal string + strVal = xattrInfo.XAttrs[key] + if len(strVal) > 0 { + val = []byte(strVal) + return + } + return +} + +func (s *xattrStore) Delete(vol, obj, key string) (err error) { + + return +} + +func (s *xattrStore) List(vol, obj string) (data [][]byte, err error) { + + return +} diff --git a/objectnode/fs_stream.go b/objectnode/fs_stream.go new file mode 100644 index 000000000..bc3f9ea94 --- /dev/null +++ b/objectnode/fs_stream.go @@ -0,0 +1,81 @@ +package objectnode + +import ( + "context" + "io" + "sync" +) + +type Stream struct { + mu *sync.Mutex + cond *sync.Cond + buffer []byte + closed bool + ctx context.Context +} + +func (s *Stream) Write(p []byte) (n int, err error) { + s.mu.Lock() + defer s.mu.Unlock() + select { + case <-s.ctx.Done(): + return 0, s.ctx.Err() + default: + } + if s.closed { + return 0, io.EOF + } + s.buffer = append(s.buffer, p...) + s.cond.L.Lock() + s.cond.Broadcast() + s.cond.L.Unlock() + return len(p), nil +} + +func (s *Stream) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + s.cond.L.Lock() + s.cond.Broadcast() + s.cond.L.Unlock() + return nil +} + +func (s *Stream) Read(p []byte) (n int, err error) { + s.mu.Lock() + defer s.mu.Unlock() + select { + case <-s.ctx.Done(): + return 0, s.ctx.Err() + default: + } + if len(s.buffer) == 0 && s.closed { + return 0, io.EOF + } + if len(s.buffer) == 0 { + s.mu.Unlock() + s.cond.L.Lock() + s.cond.Wait() + s.cond.L.Unlock() + s.mu.Lock() + if len(s.buffer) == 0 && s.closed { + return 0, io.EOF + } + } + var readN = len(p) + if len(s.buffer) < readN { + readN = len(s.buffer) + } + copy(p, s.buffer[:readN]) + s.buffer = s.buffer[readN:] + return readN, nil +} + +func NewStream(ctx context.Context) *Stream { + stream := &Stream{} + stream.mu = new(sync.Mutex) + stream.cond = sync.NewCond(new(sync.Mutex)) + stream.ctx = ctx + return stream +} diff --git a/objectnode/fs_stream_test.go b/objectnode/fs_stream_test.go new file mode 100644 index 000000000..f7cf0c994 --- /dev/null +++ b/objectnode/fs_stream_test.go @@ -0,0 +1,88 @@ +package objectnode + +import ( + "context" + "io" + "reflect" + "sync" + "testing" + "time" +) + +func TestNewStream_modify(t *testing.T) { + ctx, _ := context.WithCancel(context.Background()) + stream := NewStream(ctx) + var src = []byte("1234567890") + + // write to stream + var err error + if _, err = stream.Write(src); err != nil { + t.Fatalf("write fail casue: %v", err) + } + _ = stream.Close() + // modify source + for i := 0; i < len(src); i++ { + src[i] = 0 + } + // read from stream + var actual = make([]byte, len(src)) + if _, err = stream.Read(actual); err != nil && err != io.EOF { + t.Fatalf("read fail cause: %v", err) + } + if string(actual) != "1234567890" { + t.Fatalf("result mismatch") + } +} + +func TestStream_rw(t *testing.T) { + ctx, _ := context.WithCancel(context.Background()) + stream := NewStream(ctx) + var src = []byte("1234567890") + + asyncWG := new(sync.WaitGroup) + + // async write + asyncWG.Add(1) + go func() { + defer asyncWG.Done() + var err error + var n int + for _, b := range src { + n, err = stream.Write([]byte{b}) + if err != nil { + t.Fatalf("write fail cause: %v", err) + } + if n != 1 { + t.Fatalf("write bytes count mismatch: expect 10 actual %v", n) + } + time.Sleep(100 * time.Millisecond) + } + _ = stream.Close() + }() + + // async read + asyncWG.Add(1) + go func() { + defer asyncWG.Done() + var err error + var n int + tmp := make([]byte, 3) + result := make([]byte, 0) + for { + n, err = stream.Read(tmp) + if err != nil && err != io.EOF { + t.Fatalf("read fail cause: %v", err) + } + if err == io.EOF { + break + } + t.Logf("read: %v", tmp[:n]) + result = append(result, tmp[:n]...) + } + if !reflect.DeepEqual(result, src) { + t.Fatalf("result mismatch: expect %v actual %v", src, result) + } + }() + + asyncWG.Wait() +} diff --git a/objectnode/fs_volume.go b/objectnode/fs_volume.go new file mode 100644 index 000000000..e89befc21 --- /dev/null +++ b/objectnode/fs_volume.go @@ -0,0 +1,1395 @@ +package objectnode + +import ( + "encoding/hex" + "encoding/json" + "encoding/xml" + "hash" + "io" + "os" + "sort" + "strings" + "sync" + "syscall" + + "crypto/md5" + "time" + + "github.com/chubaofs/chubaofs/proto" + "github.com/chubaofs/chubaofs/sdk/data/stream" + "github.com/chubaofs/chubaofs/sdk/meta" + "github.com/chubaofs/chubaofs/util/log" + + "github.com/chubaofs/chubaofs/util" + "github.com/tiglabs/raft/logger" +) + +const ( + rootIno = proto.RootIno + OSSMetaUpdateDuration = time.Duration(time.Second * 30) +) + +// Used to validate interface implementation +var _ Volume = &volume{} + +type OSSMeta struct { + policy *Policy + acl *AccessControlPolicy + policyLock sync.RWMutex + aclLock sync.RWMutex +} + +func (v *volume) loadPolicy() (p *Policy) { + v.om.policyLock.RLock() + p = v.om.policy + v.om.policyLock.RUnlock() + return +} + +func (v *volume) storePolicy(p *Policy) { + v.om.policyLock.Lock() + v.om.policy = p + v.om.policyLock.Unlock() + return +} + +func (v *volume) loadACL() (p *AccessControlPolicy) { + v.om.aclLock.RLock() + p = v.om.acl + v.om.aclLock.RUnlock() + return +} + +func (v *volume) storeACL(p *AccessControlPolicy) { + v.om.aclLock.Lock() + v.om.acl = p + v.om.aclLock.Unlock() + return +} + +// This struct is implementation of Volume interface +type volume struct { + mw *meta.MetaWrapper + ec *stream.ExtentClient + vm *volumeManager + name string + om *OSSMeta + ticker *time.Ticker + //ms Store + + closeOnce sync.Once + closingCh chan struct{} + closedCh chan struct{} +} + +func (v *volume) syncOSSMeta() { + defer v.ticker.Stop() + v.ticker = time.NewTicker(OSSMetaUpdateDuration) + for { + select { + case <-v.ticker.C: + v.loadOSSMeta() + case <-v.closingCh: + v.closedCh <- struct{}{} + return + } + } +} + +func (v *volume) stopOSSMetaSync() { + v.closingCh <- struct{}{} + + select { + case <-v.closedCh: + break + } +} + +// update volume meta info +func (v *volume) loadOSSMeta() { + policy, _ := v.loadBucketPolicy() + if policy != nil { + v.storePolicy(policy) + } + + acl, _ := v.loadBucketACL() + if acl != nil { + v.storeACL(acl) + } +} + +// load bucket policy from vm +func (v *volume) loadBucketPolicy() (policy *Policy, err error) { + var store Store + store, err = v.vm.GetStore() + if err != nil { + return + } + var data []byte + data, err = store.Get(v.name, bucketRootPath, XAttrKeyOSSPolicy) + if err != nil { + log.LogErrorf("loadBucketPolicy: load bucket policy fail: volume(%v) err(%v)", v.name, err) + return + } + policy = &Policy{} + if err = json.Unmarshal(data, policy); err != nil { + return + } + return +} + +func (v *volume) loadBucketACL() (*AccessControlPolicy, error) { + store, err1 := v.vm.GetStore() + if err1 != nil { + return nil, err1 + } + data, err2 := store.Get(v.name, bucketRootPath, OSS_ACL_KEY) + if err2 != nil { + return nil, err2 + } + acl := &AccessControlPolicy{} + err3 := xml.Unmarshal(data, acl) + if err3 != nil { + return nil, err3 + } + return acl, nil +} + +func (v *volume) OSSMeta() *OSSMeta { + return v.om +} + +func (v *volume) getInodeFromPath(path string) (inode uint64, err error) { + // path == "/" + if path == "/" { + return volumeRootInode, nil + } + + dirs, filename := splitPath(path) + log.LogInfof("dirs %v %v %v", path, len(dirs), filename) + + if len(dirs) == 0 && filename == "" { + return volumeRootInode, nil + } else { + // process path + var parentId uint64 + if parentId, err = v.lookupDirectories(dirs, false); err != nil { + return 0, err + } + log.LogDebugf("GetXAttr: lookup directories: path(%v) parentId(%v)", path, parentId) + // check file + var lookupMode uint32 + inode, lookupMode, err = v.mw.Lookup_ll(parentId, filename) + if err != nil { + return 0, err + } + if os.FileMode(lookupMode).IsDir() { + err = syscall.ENOENT + return 0, err + } + } + + return +} + +func (v *volume) SetXAttr(path string, key string, data []byte) error { + inode, err1 := v.getInodeFromPath(path) + if err1 != nil { + return err1 + } + return v.mw.XAttrSet_ll(inode, []byte(key), data) +} + +func (v *volume) GetXAttr(path string, key string) (info *proto.XAttrInfo, err error) { + var inode uint64 + inode, err = v.getInodeFromPath(path) + if err != nil { + return + } + if info, err = v.mw.XAttrGet_ll(inode, key); err != nil { + log.LogErrorf("GetXAttr: meta get xattr fail: path(%v) inode(%v) err(%v)", path, inode, err) + return + } + return +} + +func (v *volume) DeleteXAttr(path string, key string) (err error) { + inode, err1 := v.getInodeFromPath(path) + if err1 != nil { + err = err1 + return + } + if err = v.mw.XAttrDel_ll(inode, key); err != nil { + log.LogErrorf("SetXAttr: meta set xattr fail: path(%v) inode(%v) err(%v)", path, inode, err) + return + } + return +} + +func (v *volume) OSSSecure() (accessKey, secretKey string) { + return v.mw.OSSSecure() +} + +func (v *volume) ListFilesV1(request *ListBucketRequestV1) ([]*FSFileInfo, string, bool, []string, error) { + //prefix, delimiter, marker string, maxKeys uint64 + + marker := request.marker + prefix := request.prefix + maxKeys := request.maxKeys + delimiter := request.delimiter + + var err error + var infos []*FSFileInfo + var prefixes Prefixes + + infos, prefixes, err = v.listFilesV1(prefix, marker, delimiter, maxKeys) + if err != nil { + return nil, "", false, nil, err + } + if len(infos) <= 0 { + return nil, "", false, nil, nil + } + + var nextMarker string + var isTruncated bool + + if len(infos) > int(maxKeys) { + nextMarker = infos[maxKeys].Path + infos = infos[:maxKeys] + isTruncated = true + } + + return infos, nextMarker, isTruncated, prefixes, nil +} + +func (v *volume) ListFilesV2(request *ListBucketRequestV2) ([]*FSFileInfo, uint64, string, bool, []string, error) { + + delimiter := request.delimiter + maxKeys := request.maxKeys + prefix := request.prefix + contToken := request.contToken + startAfter := request.startAfter + + var err error + var infos []*FSFileInfo + var prefixes Prefixes + + infos, prefixes, err = v.listFilesV2(prefix, startAfter, contToken, delimiter, maxKeys) + if err != nil { + return nil, 0, "", false, nil, err + } + if len(infos) <= 0 { + return nil, 0, "", false, nil, err + } + + var keyCount uint64 + var nextToken string + var isTruncated bool + + keyCount = uint64(len(infos)) + if len(infos) > int(maxKeys) { + nextToken = infos[maxKeys].Path + infos = infos[:maxKeys] + isTruncated = true + keyCount = maxKeys + } + + return infos, keyCount, nextToken, isTruncated, prefixes, nil +} + +func (v *volume) WriteFile(path string, reader io.Reader) (*FSFileInfo, error) { + var err error + var fInfo *FSFileInfo + + dirs, filename := splitPath(path) + + // process path + var parentId uint64 + if parentId, err = v.lookupDirectories(dirs, true); err != nil { + return nil, err + } + log.LogDebugf("WriteFile: lookup directories, path(%v) parentId(%v)", path, parentId) + // check file + var lookupMode uint32 + _, lookupMode, err = v.mw.Lookup_ll(parentId, filename) + if err != nil && err != syscall.ENOENT { + return nil, err + } + if err == nil && os.FileMode(lookupMode).IsDir() { + return nil, syscall.EEXIST + } + // create temp file + var tempFilename = tempFileName(filename) + var tempInodeInfo *proto.InodeInfo + if tempInodeInfo, err = v.mw.Create_ll(parentId, tempFilename, 0600, 0, 0, nil); err != nil { + log.LogErrorf("WriteFile: create temp file fail, parent(%v) name(%v) err(%v)", parentId, tempFilename, err) + return nil, err + } + log.LogDebugf("WriteFile: create temp file, parent(%v) name(%v) inode(%v)", parentId, tempFilename, tempInodeInfo.Inode) + + // write data + var buf = make([]byte, 128*1024) + var readN, writeN, offset int + + var fileMd5 string + var md5Buf = make([]byte, 128*1024) + var md5Hash = md5.New() + + if err = v.ec.OpenStream(tempInodeInfo.Inode); err != nil { + log.LogErrorf("WriteFile: open stream fail, inode(%v) err(%v)", tempInodeInfo.Inode, err) + return nil, err + } + defer func() { + if closeErr := v.ec.CloseStream(tempInodeInfo.Inode); closeErr != nil { + log.LogErrorf("WriteFile: close stream fail, inode(%v) err(%v)", tempInodeInfo.Inode, closeErr) + } + if evictErr := v.ec.EvictStream(tempInodeInfo.Inode); evictErr != nil { + log.LogErrorf("WriteFile: evict stream fail, inode(%v) err(%v)", tempInodeInfo.Inode, evictErr) + } + }() + + for { + readN, err = reader.Read(buf) + if err != nil && err != io.EOF { + return nil, err + } + if readN > 0 { + if writeN, err = v.ec.Write(tempInodeInfo.Inode, offset, buf[:readN], false); err != nil { + log.LogErrorf("WriteFile: write data fail, inode(%v) offset(%v) err(%v)", tempInodeInfo.Inode, offset, err) + return nil, err + } + log.LogDebugf("WriteFile: write data, inode(%v) offset(%v) bytes(%v)", tempInodeInfo.Inode, offset, readN) + offset += writeN + // copy to md5 buffer, and then write to md5 + copy(md5Buf, buf[:readN]) + md5Hash.Write(md5Buf[:readN]) + } + if err == io.EOF { + break + } + } + + // compute file md5 + fileMd5 = hex.EncodeToString(md5Hash.Sum(nil)) + + // flush + if err = v.ec.Flush(tempInodeInfo.Inode); err != nil { + log.LogErrorf("WriteFile: flush data fail, inode(%v) err(%v)", tempInodeInfo.Inode, err) + return nil, err + } + + // rename temp file to origin + if err = v.mw.Rename_ll(parentId, tempFilename, parentId, filename); err != nil { + log.LogErrorf("WriteFile: rename temp file fail, (%v_%v) to (%v_%v) err(%v)", parentId, tempFilename, parentId, filename, err) + return nil, err + } + log.LogDebugf("WriteFile: rename temp file, (%v_%v) to (%v_%v)", + parentId, tempFilename, parentId, filename) + + // set file md5 to meta data + if err = v.mw.XAttrSet_ll(tempInodeInfo.Inode, []byte(XAttrKeyOSSETag), []byte(fileMd5)); err != nil { + log.LogErrorf("WriteFile: set xattr fail, inode(%v) key(%v) val(%v) err(%v)", tempInodeInfo.Inode, XAttrKeyOSSETag, fileMd5, err) + return nil, err + } + + // create file info + fInfo = &FSFileInfo{ + Path: path, + Size: int64(writeN), + Mode: os.FileMode(lookupMode), + ModifyTime: time.Now(), + ETag: fileMd5, + Inode: tempInodeInfo.Inode, + } + + return fInfo, nil +} + +func (v *volume) DeleteFile(path string) error { + var err error + dirs, filename := splitPath(path) + + // process path + var parentId uint64 + if parentId, err = v.lookupDirectories(dirs, false); err != nil { + return err + } + // lookup target inode + var inode uint64 + var mode uint32 + if inode, mode, err = v.mw.Lookup_ll(parentId, filename); err != nil { + return err + } + if os.FileMode(mode).IsDir() { + return syscall.ENOENT + } + + // remove file + if _, err = v.mw.Delete_ll(parentId, filename, false); err != nil { + return err + } + + // evict inode + if err = v.ec.EvictStream(inode); err != nil { + return err + } + if err = v.mw.Evict(inode); err != nil { + return err + } + + return nil +} + +func (v *volume) InitMultipart(path string) (multipartID string, err error) { + // Invoke meta service to get a session id + // Create parent path + + dirs, _ := splitPath(path) + // process path + var parentId uint64 + if parentId, err = v.lookupDirectories(dirs, true); err != nil { + log.LogErrorf("InitMultipart: lookup directories fail: path(%v) err(%v)", path, err) + return "", err + } + + // save parent id to meta + multipartID, err = v.mw.InitMultipart_ll(path, parentId) + if err != nil { + log.LogErrorf("InitMultipart: meta init multipart fail: path(%v) err(%v)", path, err) + return "", err + } + return multipartID, nil +} + +func (v *volume) WritePart(path string, multipartId string, partId uint16, reader io.Reader) (*FSFileInfo, error) { + var parentId uint64 + var err error + var fInfo *FSFileInfo + + dirs, fileName := splitPath(path) + // process path + if parentId, err = v.lookupDirectories(dirs, true); err != nil { + log.LogErrorf("WritePart: lookup directories fail, path(%v) err(%v)", path, err) + return nil, err + } + + // create temp file (inode only, invisible for user) + var tempInodeInfo *proto.InodeInfo + if tempInodeInfo, err = v.mw.InodeCreate_ll(0600, 0, 0, nil); err != nil { + log.LogErrorf("WritePart: meta create inode fail: multipartID(%v) partID(%v) err(%v)", + multipartId, parentId, err) + return nil, err + } + log.LogDebugf("WritePart: meta create temp file inode: multipartID(%v) partID(%v) inode(%v)", + multipartId, parentId, tempInodeInfo.Inode) + + // write data + var buf = make([]byte, 128*1024) + var readN, writeN, offset int + + var fileMd5 string + var md5Buf = make([]byte, 128*1024) + var md5Hash = md5.New() + var size uint64 + + if err = v.ec.OpenStream(tempInodeInfo.Inode); err != nil { + log.LogErrorf("WritePart: data open stream fail, inode(%v) err(%v)", tempInodeInfo.Inode, err) + return nil, err + } + defer func() { + if closeErr := v.ec.CloseStream(tempInodeInfo.Inode); closeErr != nil { + log.LogErrorf("WritePart: data close stream fail, inode(%v) err(%v)", tempInodeInfo.Inode, closeErr) + } + }() + + for { + readN, err = reader.Read(buf) + if err != nil && err != io.EOF { + return nil, err + } + if readN > 0 { + if writeN, err = v.ec.Write(tempInodeInfo.Inode, offset, buf[:readN], false); err != nil { + log.LogErrorf("WritePart: data write tmp file fail, inode(%v) offset(%v) err(%v)", tempInodeInfo.Inode, offset, err) + return nil, err + } + offset += writeN + // copy to md5 buffer, and then write to md5 + size += uint64(readN) + copy(md5Buf, buf[:readN]) + md5Hash.Write(md5Buf[:readN]) + } + if err == io.EOF { + break + } + } + + // compute file md5 + fileMd5 = hex.EncodeToString(md5Hash.Sum(nil)) + + // flush + if err = v.ec.Flush(tempInodeInfo.Inode); err != nil { + log.LogErrorf("WritePart: data flush inode fail, inode(%v) err(%v)", tempInodeInfo.Inode, err) + return nil, err + } + + // save temp file MD5 + if err = v.mw.XAttrSet_ll(tempInodeInfo.Inode, []byte(XAttrKeyOSSETag), []byte(fileMd5)); err != nil { + log.LogErrorf("WritePart: meta set xattr fail, inode(%v) key(%v) val(%v) err(%v)", + tempInodeInfo, XAttrKeyOSSETag, fileMd5, err) + return nil, err + } + + // update temp file inode to meta with session + if err = v.mw.AddMultipartPart_ll(multipartId, parentId, partId, size, fileMd5, tempInodeInfo.Inode); err != nil { + log.LogErrorf("WritePart: meta add multipart part fail: multipartID(%v) parentID(%v) partID(%v) inode(%v) size(%v) MD5(%v) err(%v)", + multipartId, parentId, parentId, tempInodeInfo.Inode, size, fileMd5, err) + } + log.LogDebugf("WritePart: meta add multipart part: multipartID(%v) parentID(%v) partID(%v) inode(%v) size(%v) MD5(%v)", + multipartId, parentId, parentId, tempInodeInfo.Inode, size, fileMd5) + + // create file info + fInfo = &FSFileInfo{ + Path: fileName, + Size: int64(writeN), + Mode: os.FileMode(0600), + ModifyTime: time.Now(), + ETag: fileMd5, + Inode: tempInodeInfo.Inode, + } + return fInfo, nil +} + +func (v *volume) AbortMultipart(path string, multipartID string) (err error) { + // TODO: cleanup data while abort multipart + var parentId uint64 + + dirs, _ := splitPath(path) + // process path + if parentId, err = v.lookupDirectories(dirs, true); err != nil { + log.LogErrorf("AbortMultipart: lookup directories fail, multipartID(%v) path(%v) dirs(%v) err(%v)", + multipartID, path, dirs, err) + return err + } + // get multipart info + var multipartInfo *proto.MultipartInfo + if multipartInfo, err = v.mw.GetMultipart_ll(multipartID, parentId); err != nil { + log.LogErrorf("AbortMultipart: meta get multipart fail, multipartID(%v) parentID(%v) path(%v) err(%v)", + multipartID, parentId, path, err) + return + } + // release part data + for _, part := range multipartInfo.Parts { + if _, err = v.mw.InodeUnlink_ll(part.Inode); err != nil { + log.LogErrorf("AbortMultipart: meta inode unlink fail: multipartID(%v) partID(%v) inode(%v) err(%v)", + multipartID, part.ID, part.Inode, err) + return + } + if err = v.mw.Evict(part.Inode); err != nil { + log.LogErrorf("AbortMultipart: meta inode evict fail: multipartID(%v) partID(%v) inode(%v) err(%v)", + multipartID, part.ID, part.Inode, err) + } + log.LogDebugf("AbortMultipart: multipart part data released: multipartID(%v) partID(%v) inode(%v)", + multipartID, part.ID, part.Inode) + } + + if err = v.mw.RemoveMultipart_ll(multipartID, parentId); err != nil { + log.LogErrorf("AbortMultipart: meta abort multipart fail, multipartID(%v) parentID(%v) err(%v)", + multipartID, parentId, err) + return err + } + log.LogDebugf("AbortMultipart: meta abort multipart, multipartID(%v) path(%v)", + multipartID, path) + return nil +} + +func (v *volume) CompleteMultipart(path string, multipartID string) (fsFileInfo *FSFileInfo, err error) { + + const mode = 0600 + + var parentId uint64 + + dirs, filename := splitPath(path) + // process path + if parentId, err = v.lookupDirectories(dirs, true); err != nil { + log.LogErrorf("CompleteMultipart: lookup directories fail, multipartID(%v) path(%v) dirs(%v) err(%v)", + multipartID, path, dirs, err) + return + } + + // get multipart info + var multipartInfo *proto.MultipartInfo + if multipartInfo, err = v.mw.GetMultipart_ll(multipartID, parentId); err != nil { + log.LogErrorf("CompleteMultipart: meta get multipart fail, multipartID(%v) parentID(%v) path(%v) err(%v)", + multipartID, parentId, path, err) + return + } + + // sort multipart parts + parts := multipartInfo.Parts + sort.SliceStable(parts, func(i, j int) bool { return parts[i].ID < parts[j].ID }) + + // create inode for complete data + var completeInodeInfo *proto.InodeInfo + if completeInodeInfo, err = v.mw.InodeCreate_ll(mode, 0, 0, nil); err != nil { + log.LogErrorf("CompleteMultipart: meta inode create fail: err(%v)", err) + return + } + log.LogDebugf("CompleteMultipart: meta inode create: inode(%v)", completeInodeInfo.Inode) + defer func() { + if err != nil { + if deleteErr := v.mw.InodeDelete_ll(completeInodeInfo.Inode); deleteErr != nil { + log.LogErrorf("CompleteMultipart: meta delete complete inode fail: inode(%v) err(%v)", + completeInodeInfo.Inode, err) + } + } + }() + + // merge complete extent keys + var size uint64 + var completeExtentKeys = make([]proto.ExtentKey, 0) + var fileOffset uint64 + for _, part := range parts { + var eks []proto.ExtentKey + if _, _, eks, err = v.mw.GetExtents(part.Inode); err != nil { + log.LogErrorf("CompleteMultipart: meta get extents fail: inode(%v) err(%v)", part.Inode, err) + return + } + // recompute offsets of extent keys + for _, ek := range eks { + ek.FileOffset = fileOffset + fileOffset += uint64(ek.Size) + completeExtentKeys = append(completeExtentKeys, ek) + } + size += part.Size + } + + // compute md5 hash + var md5Val string + if len(parts) == 1 { + md5Val = parts[0].MD5 + } else { + var md5Hash = md5.New() + var reuseBuf = make([]byte, util.BlockSize) + for _, part := range parts { + if err = v.appendInodeHash(md5Hash, part.Inode, part.Size, reuseBuf); err != nil { + log.LogErrorf("CompleteMultipart: append part hash fail: partID(%v) inode(%v) err(%v)", + part.ID, part.Inode, err) + return + } + } + md5Val = hex.EncodeToString(md5Hash.Sum(nil)) + } + log.LogDebugf("CompleteMultipart: merge parts: numParts(%v) MD5(%v)", len(parts), md5Val) + + if err = v.mw.AppendExtentKeys(completeInodeInfo.Inode, completeExtentKeys); err != nil { + log.LogErrorf("CompleteMultipart: meta append extent keys fail: inode(%v) err(%v)", completeInodeInfo.Inode, err) + return + } + + var ( + existMode uint32 + ) + _, existMode, err = v.mw.Lookup_ll(parentId, filename) + if err != nil && err != syscall.ENOENT { + log.LogErrorf("CompleteMultipart: meta lookup fail: parentID(%v) name(%v) err(%v)", parentId, filename, err) + return + } + + if err == syscall.ENOENT { + if err = v.applyInodeToNewDentry(parentId, filename, completeInodeInfo.Inode); err != nil { + log.LogErrorf("CompleteMultipart: apply inode to new dentry fail: parentID(%v) name(%v) inode(%v) err(%v)", + parentId, filename, completeInodeInfo.Inode, err) + return + } + log.LogDebugf("CompleteMultipart: apply inode to new dentry: parentID(%v) name(%v) inode(%v)", + parentId, filename, completeInodeInfo.Inode) + } else { + if os.FileMode(existMode).IsDir() { + log.LogErrorf("CompleteMultipart: target mode conflict: parentID(%v) name(%v) mode(%v)", + parentId, filename, os.FileMode(existMode).String()) + err = syscall.EEXIST + return + } + if err = v.applyInodeToExistDentry(parentId, filename, completeInodeInfo.Inode); err != nil { + log.LogErrorf("CompleteMultipart: apply inode to exist dentry fail: parentID(%v) name(%v) inode(%v) err(%v)", + parentId, filename, completeInodeInfo.Inode, err) + return + } + } + + if err = v.mw.XAttrSet_ll(completeInodeInfo.Inode, []byte(XAttrKeyOSSETag), []byte(md5Val)); err != nil { + log.LogErrorf("CompleteMultipart: save MD5 fail: inode(%v) err(%v)", completeInodeInfo, err) + return + } + + // remove multipart + err = v.mw.RemoveMultipart_ll(multipartID, parentId) + if err != nil { + log.LogErrorf("CompleteMultipart: meta complete multipart fail, multipartID(%v) path(%v) parentID(%v) err(%v)", + multipartID, path, parentId, err) + return nil, err + } + // delete part inodes + for _, part := range parts { + if err = v.mw.InodeDelete_ll(part.Inode); err != nil { + log.LogErrorf("CompleteMultipart: ") + } + } + + log.LogDebugf("CompleteMultipart: meta complete multipart, multipartID(%v) path(%v) parentID(%v) inode(%v) md5(%v)", + multipartID, path, parentId, completeInodeInfo.Inode, md5Val) + + // create file info + fInfo := &FSFileInfo{ + Path: path, + Size: int64(size), + Mode: os.FileMode(0600), + ModifyTime: time.Now(), + ETag: md5Val, + Inode: completeInodeInfo.Inode, + } + return fInfo, nil +} + +func (v *volume) appendInodeHash(h hash.Hash, inode uint64, total uint64, preAllocatedBuf []byte) (err error) { + if err = v.ec.OpenStream(inode); err != nil { + log.LogErrorf("appendInodeHash: data open stream fail: inode(%v) err(%v)", + inode, err) + return + } + defer func() { + if closeErr := v.ec.CloseStream(inode); closeErr != nil { + log.LogErrorf("appendInodeHash: data close stream fail: inode(%v) err(%v)", + inode, err) + } + if evictErr := v.ec.EvictStream(inode); evictErr != nil { + log.LogErrorf("appendInodeHash: data evict stream: inode(%v) err(%v)", + inode, err) + } + }() + + var buf = preAllocatedBuf + if len(buf) == 0 { + buf = make([]byte, 1024*64) + } + + var n, offset, size int + for { + size = len(buf) + rest := total - uint64(offset) + if uint64(size) > rest { + size = int(rest) + } + n, err = v.ec.Read(inode, buf, offset, size) + if err != nil && err != io.EOF { + log.LogErrorf("appendInodeHash: data read fail, inode(%v) offset(%v) size(%v) err(%v)", inode, offset, size, err) + return + } + log.LogDebugf("appendInodeHash: data read, inode(%v) offset(%v) n(%v)", inode, offset, n) + if n > 0 { + if _, err = h.Write(buf[:n]); err != nil { + return + } + offset += n + } + if n == 0 || err == io.EOF { + break + } + } + log.LogDebugf("appendInodeHash: append to hash function: inode(%v)", inode) + return +} + +func (v *volume) applyInodeToNewDentry(parentID uint64, name string, inode uint64) (err error) { + const mode = 0600 + if err = v.mw.DentryCreate_ll(parentID, name, inode, mode); err != nil { + log.LogErrorf("applyInodeToNewDentry: meta dentry create fail: parentID(%v) name(%v) inode(%v) mode(%v) err(%v)", + parentID, name, inode, mode, err) + return err + } + return +} + +func (v *volume) applyInodeToExistDentry(parentID uint64, name string, inode uint64) (err error) { + var oldInode uint64 + oldInode, err = v.mw.DentryUpdate_ll(parentID, name, inode) + if err != nil { + log.LogErrorf("applyInodeToExistDentry: meta update dentry fail: parentID(%v) name(%v) inode(%v) err(%v)", + parentID, name, inode, err) + return + } + + defer func() { + if err != nil { + // rollback dentry update + if _, updateErr := v.mw.DentryUpdate_ll(parentID, name, oldInode); updateErr != nil { + log.LogErrorf("CompleteMultipart: meta rollback dentry update fail: parentID(%v) name(%v) inode(%v) err(%v)", + parentID, name, oldInode, updateErr) + } + } + }() + + // unlink and evict old inode + if _, err = v.mw.InodeUnlink_ll(oldInode); err != nil { + log.LogErrorf("CompleteMultipart: meta unlink old inode fail: inode(%v) err(%v)", + oldInode, err) + return + } + + defer func() { + if err != nil { + // rollback unlink + if _, linkErr := v.mw.InodeLink_ll(oldInode); linkErr != nil { + log.LogErrorf("CompleteMultipart: meta rollback inode unlink fail: inode(%v) err(%v)", + oldInode, linkErr) + } + } + }() + + if err = v.mw.Evict(oldInode); err != nil { + log.LogErrorf("CompleteMultipart: meta evict old inode fail: inode(%v) err(%v)", + oldInode, err) + return + } + return +} + +func (v *volume) ReadFile(path string, writer io.Writer, offset, size uint64) error { + var err error + dirs, filename := splitPath(path) + + // process path + var parentId uint64 + if parentId, err = v.lookupDirectories(dirs, false); err != nil { + return err + } + log.LogDebugf("ReadFile: lookup directories, path(%v) parentId(%v)", path, parentId) + // check file + var fileInode uint64 + var lookupMode uint32 + fileInode, lookupMode, err = v.mw.Lookup_ll(parentId, filename) + if err != nil { + return err + } + if os.FileMode(lookupMode).IsDir() { + return syscall.ENOENT + } + // read file data + var fileInodeInfo *proto.InodeInfo + fileInodeInfo, err = v.mw.InodeGet_ll(fileInode) + if err != nil { + return err + } + + if err = v.ec.OpenStream(fileInode); err != nil { + log.LogErrorf("ReadFile: data open stream fail, Inode(%v) err(%v)", fileInode, err) + return err + } + defer func() { + if closeErr := v.ec.CloseStream(fileInode); closeErr != nil { + log.LogErrorf("ReadFile: data close stream fail: inode(%v) err(%v)", fileInode, closeErr) + } + }() + + var upper = size + offset + if upper > fileInodeInfo.Size { + upper = fileInodeInfo.Size - offset + } + + var n int + var tmp = make([]byte, util.BlockSize) + for { + var rest = upper - uint64(offset) + if rest == 0 { + break + } + var readSize = len(tmp) + if uint64(readSize) > rest { + readSize = int(rest) + } + n, err = v.ec.Read(fileInode, tmp, int(offset), readSize) + if err != nil && err != io.EOF { + log.LogErrorf("ReadFile: data read fail, inode(%v) offset(%v) size(%v) err(%v)", fileInode, offset, size, err) + return err + } + if n > 0 { + if _, err = writer.Write(tmp[:n]); err != nil { + return err + } + offset += uint64(n) + } + if n == 0 || err == io.EOF { + break + } + } + return nil +} + +func (v *volume) FileInfo(path string) (info *FSFileInfo, err error) { + dirs, filename := splitPath(path) + + // process path + var parentId uint64 + if parentId, err = v.lookupDirectories(dirs, false); err != nil { + return + } + // check file + var fileInode uint64 + var lookupMode uint32 + if fileInode, lookupMode, err = v.mw.Lookup_ll(parentId, filename); err != nil { + return + } + if os.FileMode(lookupMode).IsDir() { + return nil, syscall.ENOENT + } + // read file data + var fileInodeInfo *proto.InodeInfo + if fileInodeInfo, err = v.mw.InodeGet_ll(fileInode); err != nil { + return + } + + var xAttrInfo *proto.XAttrInfo + if xAttrInfo, err = v.mw.XAttrGet_ll(fileInode, XAttrKeyOSSETag); err != nil { + logger.Error("FileInfo: meta get xattr fail, inode(%v) path(%v) err(%v)", fileInode, path, err) + return + } + md5Val := xAttrInfo.XAttrs[XAttrKeyOSSETag] + + info = &FSFileInfo{ + Path: path, + Size: int64(fileInodeInfo.Size), + Mode: os.FileMode(fileInodeInfo.Mode), + ModifyTime: fileInodeInfo.ModifyTime, + ETag: md5Val, + Inode: fileInodeInfo.Inode, + } + return +} + +func (v *volume) Close() error { + v.closeOnce.Do(func() { + v.mw.Close() + v.stopOSSMetaSync() + }) + return nil +} + +func (v *volume) lookupDirectories(dirs []string, autoCreate bool) (inode uint64, err error) { + var parentId = rootIno + // check and create dirs + for _, dir := range dirs { + curIno, curMode, lookupErr := v.mw.Lookup_ll(parentId, dir) + if lookupErr != nil && lookupErr != syscall.ENOENT { + log.LogErrorf("lookupDirectories: meta lokkup fail, parentID(%v) name(%v) fail err(%v)", parentId, dir, lookupErr) + return 0, lookupErr + } + if lookupErr == syscall.ENOENT && !autoCreate { + return 0, syscall.ENOENT + } + // this item is not exist + if lookupErr == syscall.ENOENT { + var inodeInfo *proto.InodeInfo + var createErr error + inodeInfo, createErr = v.mw.Create_ll(parentId, dir, uint32(os.ModeDir), 0, 0, nil) + if createErr != nil && createErr != syscall.EEXIST { + log.LogErrorf("lookupDirectories: meta create fail, parentID(%v) name(%v) mode(%v) err(%v)", parentId, dir, os.ModeDir, createErr) + return 0, createErr + } + // retry lookup if it exists. + if createErr == syscall.EEXIST { + curIno, curMode, lookupErr = v.mw.Lookup_ll(parentId, dir) + if lookupErr != nil { + return 0, lookupErr + } + if !os.FileMode(curMode).IsDir() { + return 0, syscall.EEXIST + } + parentId = curIno + continue + } + if inodeInfo == nil { + panic("illegal internal pointer found") + } + parentId = inodeInfo.Inode + continue + } + + // check mode + if !os.FileMode(curMode).IsDir() { + return 0, syscall.EEXIST + } + parentId = curIno + } + inode = parentId + return +} + +func (v *volume) listFilesV1(prefix, marker, delimiter string, maxKeys uint64) (infos []*FSFileInfo, prefixes Prefixes, err error) { + var prefixMap = PrefixMap(make(map[string]struct{})) + + parentId, dirs, err := v.findParentId(prefix) + if err != nil { + return nil, nil, err + } + log.LogDebugf("listFilesV1: find parent ID, prefix(%v) marker(%v) delimiter(%v) parentId(%v) dirs(%v)", prefix, marker, delimiter, parentId, len(dirs)) + + // recursion call listDir method + infos, prefixMap, err = v.listDir(infos, prefixMap, parentId, maxKeys, dirs, prefix, marker, delimiter) + if err != nil { + log.LogErrorf("listFilesV1: volume list dir fail: volume(%v) err(%v)", v.name, err) + return + } + + // supply size and MD5 + if err = v.supplyListFileInfo(infos); err != nil { + log.LogDebugf("listFilesV1: supply list file info fail, err(%v)", err) + return + } + + prefixes = prefixMap.Prefixes() + + log.LogDebugf("listFilesV1: volume list dir: volume(%v) prefix(%v) marker(%v) delimiter(%v) maxKeys(%v) infos(%v) prefixes(%v)", + v.name, prefix, marker, delimiter, maxKeys, len(infos), len(prefixes)) + + return +} + +func (v *volume) listFilesV2(prefix, startAfter, contToken, delimiter string, maxKeys uint64) (infos []*FSFileInfo, prefixes Prefixes, err error) { + var prefixMap = PrefixMap(make(map[string]struct{})) + + var marker string + if startAfter != "" { + marker = startAfter + } + if contToken != "" { + marker = contToken + } + parentId, dirs, err := v.findParentId(prefix) + if err != nil { + log.LogErrorf("listFilesV2: find parent ID fail, prefix(%v) marker(%v) err(%v)", prefix, marker, err) + return + } + log.LogDebugf("listFilesV2: find parent ID, prefix(%v) marker(%v) delimiter(%v) parentId(%v) dirs(%v)", prefix, marker, delimiter, parentId, len(dirs)) + + // recursion call listDir method + infos, prefixMap, err = v.listDir(infos, prefixMap, parentId, maxKeys, dirs, prefix, marker, delimiter) + if err != nil { + log.LogErrorf("listFilesV2: volume list dir fail, volume(%v) err(%v)", v.name, err) + return + } + + // supply size and MD5 + err = v.supplyListFileInfo(infos) + if err != nil { + log.LogDebugf("listFilesV2: supply list file info fail, err(%v)", err) + return + } + + prefixes = prefixMap.Prefixes() + + log.LogDebugf("listFilesV2: volume list dir: volume(%v) prefix(%v) marker(%v) delimiter(%v) maxKeys(%v) infos(%v) prefixes(%v)", + v.name, prefix, marker, delimiter, maxKeys, len(infos), len(prefixes)) + + return +} + +func (v *volume) findParentId(prefix string) (inode uint64, prefixDirs []string, err error) { + prefixDirs = make([]string, 0) + + // if prefix and marker are both not empty, use marker + var dirs []string + if prefix != "" { + dirs = strings.Split(prefix, "/") + } + if len(dirs) <= 1 { + return proto.RootIno, prefixDirs, nil + } + + var parentId = proto.RootIno + for index, dir := range dirs { + curIno, curMode, err := v.mw.Lookup_ll(parentId, dir) + if index == len(dirs) && err == syscall.ENOENT { + return parentId, prefixDirs, nil + } + + if err != nil && err != syscall.ENOENT { + log.LogErrorf("findParentId: find directories fail: prefix(%v) err(%v)", prefix, err) + return 0, nil, err + } + + if !os.FileMode(curMode).IsDir() { + return 0, nil, err + } + + prefixDirs = append(prefixDirs, dir) + parentId = curIno + } + inode = parentId + return +} + +func (v *volume) listDir(fileInfos []*FSFileInfo, prefixMap PrefixMap, parentId, maxKeys uint64, dirs []string, prefix, marker, delimiter string) ([]*FSFileInfo, PrefixMap, error) { + + children, err := v.mw.ReadDir_ll(parentId) + if err != nil { + return fileInfos, prefixMap, err + } + + for _, child := range children { + log.LogDebugf("listDir: process child, inode(%v) name(%v) parentId(%v) isDir(%v) isRegular(%v)", + child.Inode, child.Name, parentId, os.FileMode(child.Type).IsDir(), os.FileMode(child.Type).IsRegular()) + if os.FileMode(child.Type).IsDir() { + fileInfos, prefixMap, err = v.listDir(fileInfos, prefixMap, child.Inode, maxKeys, append(dirs, child.Name), prefix, marker, delimiter) + if err != nil { + return fileInfos, prefixMap, err + } + } else { + path := strings.Join(dirs, "/") + if len(dirs) == 0 { + path += child.Name + } else { + path += "/" + child.Name + } + if marker != "" && path < marker { + continue + } + if prefix != "" && !strings.HasPrefix(path, prefix) { + continue + } + if delimiter != "" && strings.Contains(child.Name, delimiter) { + idx := strings.Index(path, delimiter) + prefix := util.SubString(path, 0, idx) + prefixMap.AddPrefix(prefix) + continue + } + fileInfo := &FSFileInfo{ + Inode: child.Inode, + Path: path, + } + fileInfos = append(fileInfos, fileInfo) + + // if file numbers is enough, end list dir + if len(fileInfos) >= int(maxKeys+1) { + return fileInfos, prefixMap, nil + } + } + } + return fileInfos, prefixMap, nil +} + +func (v *volume) supplyListFileInfo(fileInfos []*FSFileInfo) (err error) { + // get all file info indoes + inoInfoMap := make(map[uint64]*proto.InodeInfo) + var inodes []uint64 + for _, fileInfo := range fileInfos { + inodes = append(inodes, fileInfo.Inode) + } + + // Get size information in batches, then update to fileInfos + batchInodeInfos := v.mw.BatchInodeGet(inodes) + for _, inodeInfo := range batchInodeInfos { + inoInfoMap[inodeInfo.Inode] = inodeInfo + } + for _, fileInfo := range fileInfos { + inoInfo := inoInfoMap[fileInfo.Inode] + if inoInfo != nil { + fileInfo.Size = int64(inoInfo.Size) + fileInfo.ModifyTime = inoInfo.ModifyTime + fileInfo.Mode = os.FileMode(inoInfo.Mode) + } + } + + // Get MD5 information in batches, then update to fileInfos + md5Map := make(map[uint64]string) + keys := []string{XAttrKeyOSSETag} + batchXAttrInfos, err := v.mw.BatchGetXAttr(inodes, keys) + if err != nil { + logger.Error("supplyListFileInfo: batch get xattr fail, inodes(%v), err(%v)", inodes, err) + return + } + for _, xAttrInfo := range batchXAttrInfos { + md5Map[xAttrInfo.Inode] = xAttrInfo.XAttrs[XAttrKeyOSSETag] + } + for _, fileInfo := range fileInfos { + fileInfo.ETag = md5Map[fileInfo.Inode] + } + return +} + +func (v *volume) ListMultipartUploads(prefix, delimiter, keyMarker string, multipartIdMarker string, maxUploads uint64) ([]*FSUpload, string, string, bool, []string, error) { + sessions, err := v.mw.ListMultipart_ll(prefix, delimiter, keyMarker, multipartIdMarker, maxUploads) + if err != nil { + return nil, "", "", false, nil, err + } + + uploads := make([]*FSUpload, 0) + prefixes := make([]string, 0) + prefixMap := make(map[string]string) + + var nextUpload *proto.MultipartInfo + var NextMarker string + var NextSessionIdMarker string + var IsTruncated bool + //var sessionResult []*proto.MultipartInfo + + if len(sessions) == 0 { + return nil, "", "", false, nil, nil + } + + // get maxUploads number sessions from combined sessions + if len(sessions) > int(maxUploads) { + sessions = sessions[:maxUploads] + nextUpload = sessions[maxUploads] + NextMarker = nextUpload.Path + NextSessionIdMarker = nextUpload.ID + IsTruncated = true + } + + for _, session := range sessions { + if delimiter != "" && strings.Contains(session.Path, delimiter) { + idx := strings.Index(session.Path, delimiter) + prefix := util.SubString(session.Path, 0, idx) + prefixes = append(prefixes, prefix) + continue + } + fsUpload := &FSUpload{ + Key: session.Path, + UploadId: session.ID, + Initiated: formatTimeISO(session.InitTime), + StorageClass: StorageClassStandard, + } + uploads = append(uploads, fsUpload) + } + if len(prefixMap) > 0 { + for prefix := range prefixMap { + prefixes = append(prefixes, prefix) + } + } + + return uploads, NextMarker, NextSessionIdMarker, IsTruncated, prefixes, nil +} + +func (v *volume) ListParts(path, sessionId string, maxParts, partNumberMarker uint64) (parts []*FSPart, nextMarker uint64, IsTruncated bool, err error) { + var parentId uint64 + dirs, _ := splitPath(path) + // process path + if parentId, err = v.lookupDirectories(dirs, true); err != nil { + return nil, 0, false, err + } + + multipartInfo, err := v.mw.GetMultipart_ll(sessionId, parentId) + if err != nil { + log.LogErrorf("Get session(%s) parts failed cause: %v", sessionId, err) + } + + sessionParts := multipartInfo.Parts + resLength := maxParts + IsTruncated = true + if (uint64(len(sessionParts)) - partNumberMarker) < maxParts { + resLength = uint64(len(sessionParts)) - partNumberMarker + IsTruncated = false + nextMarker = 0 + } else { + nextMarker = partNumberMarker + resLength + } + + sessionPartsTemp := sessionParts[partNumberMarker:resLength] + for _, sessionPart := range sessionPartsTemp { + fsPart := &FSPart{ + PartNumber: int(sessionPart.ID), + LastModified: formatTimeISO(sessionPart.UploadTime), + ETag: sessionPart.MD5, + Size: int(sessionPart.Size), + } + parts = append(parts, fsPart) + } + + return parts, nextMarker, IsTruncated, nil +} + +func (v *volume) CopyFile(targetPath, sourcePath string) (info *FSFileInfo, err error) { + + sourceDirs, sourceFilename := splitPath(sourcePath) + // process source targetPath + var sourceParentId uint64 + if sourceParentId, err = v.lookupDirectories(sourceDirs, false); err != nil { + return nil, err + } + + // find source file inode + var sourceFileInode uint64 + var lookupMode uint32 + sourceFileInode, lookupMode, err = v.mw.Lookup_ll(sourceParentId, sourceFilename) + if err != nil { + return nil, err + } + if os.FileMode(lookupMode).IsDir() { + return nil, syscall.ENOENT + } + + // get new file parent id + newDirs, newFilename := splitPath(targetPath) + // process source targetPath + var newParentId uint64 + if newParentId, err = v.lookupDirectories(newDirs, false); err != nil { + return nil, err + } + + inodeInfo, err := v.copyFile(newParentId, newFilename, sourceFileInode, 0600) + if err != nil { + log.LogErrorf("CopyFile: meta copy file fail: newParentID(%v) newName(%v) sourceIno(%v) err(%v)", + newParentId, newFilename, sourceFileInode, err) + return nil, err + } + + xAttrInfo, err := v.mw.XAttrGet_ll(sourceFileInode, XAttrKeyOSSETag) + if err != nil { + log.LogErrorf("CopyFile: meta set xattr fail: inode(%v) err(%v)", sourceFileInode, err) + return nil, err + } + md5Val := xAttrInfo.XAttrs[XAttrKeyOSSETag] + + info = &FSFileInfo{ + Path: targetPath, + Size: int64(inodeInfo.Size), + Mode: os.FileMode(inodeInfo.Size), + ModifyTime: inodeInfo.ModifyTime, + ETag: md5Val, + Inode: inodeInfo.Inode, + } + return +} + +func (v *volume) copyFile(parentID uint64, newFileName string, sourceFileInode uint64, mode uint32) (info *proto.InodeInfo, err error) { + + if err = v.mw.DentryCreate_ll(parentID, newFileName, sourceFileInode, mode); err != nil { + return + } + if info, err = v.mw.InodeLink_ll(sourceFileInode); err != nil { + return + } + return +} + +func newVolume(masters []string, vol string) (*volume, error) { + var err error + var masterHosts = strings.Join(masters, ",") + + var mw *meta.MetaWrapper + if mw, err = meta.NewMetaWrapper(vol, "", masterHosts, false, false, nil); err != nil { + return nil, err + } + defer func() { + if err != nil { + mw.Close() + } + }() + var ec *stream.ExtentClient + if ec, err = stream.NewExtentClient(vol, masterHosts, -1, -1, mw.AppendExtentKey, mw.GetExtents, mw.Truncate); err != nil { + return nil, err + } + + v := &volume{mw: mw, ec: ec, name: vol, om: new(OSSMeta)} + go v.syncOSSMeta() + return v, nil +} diff --git a/objectnode/fs_volume_test.go b/objectnode/fs_volume_test.go new file mode 100644 index 000000000..f899da2fb --- /dev/null +++ b/objectnode/fs_volume_test.go @@ -0,0 +1 @@ +package objectnode diff --git a/objectnode/policy.go b/objectnode/policy.go new file mode 100644 index 000000000..7f2ae3a0b --- /dev/null +++ b/objectnode/policy.go @@ -0,0 +1,236 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strings" + + "github.com/chubaofs/chubaofs/util/log" +) + +// https://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html +const ( + PolicyDefaultVersion = "2012-10-17" + BucketPolicyLimitSize = 20 * 1024 //Bucket policies are limited to 20KB + ArnSplitToken = ":" +) + +//https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/example-bucket-policies.html + +type Policy struct { + Version string `json:"Version"` + Id string `json:"Id,omnistring"` + Statements []Statement `json:"Statement,omitempty"` +} + +// +// arn:partition:service:region:account-id:resource-id +// arn:partition:service:region:account-id:resource-type/resource-id +// arn:partition:service:region:account-id:resource-type:resource-id +type Arn struct { + arn Resource + partition string //aws + service string //s3/iam + region string // + accountId string // + resourceType string // + resourceId string // +} + +func parseArn(str string) (*Arn, error) { + items := strings.Split(str, ArnSplitToken) + if len(items) < 4 { + log.LogErrorf("Arn is invalid: %v", str) + return nil, errors.New("invalid arn") + } + arn := &Arn{ + partition: items[1], + service: items[2], + region: items[3], + accountId: items[4], + } + + if len(items) > 6 { + arn.resourceType = items[5] + arn.resourceId = items[6] + } else { + arn.resourceId = items[5] + } + + return arn, nil +} + +// write bucket policy into store and update vol policy meta +func storeBucketPolicy(bytes []byte, vol *volume) (*Policy, error) { + store, err1 := vol.vm.GetStore() + if err1 != nil { + return nil, err1 + } + + policy := &Policy{} + err2 := json.Unmarshal(bytes, policy) + if err2 != nil { + log.LogErrorf("policy unmarshal err: %v", err2) + return nil, err2 + } + + // validate policy + ok, err3 := policy.Validate(vol.name) + if err3 != nil { + log.LogErrorf("policy validate err: %v", err2) + return nil, err3 + } + if !ok { + return nil, errors.New("policy is invalid") + } + + // put policy bytes into store + err4 := store.Put(vol.name, bucketRootPath, XAttrKeyOSSPolicy, bytes) + if err4 != nil { + return nil, err4 + } + + vol.storePolicy(policy) + + return policy, nil +} + +// +func ParsePolicy(r io.Reader, bucket string) (*Policy, error) { + var policy Policy + d := json.NewDecoder(r) + d.DisallowUnknownFields() + if err := d.Decode(&policy); err != nil { + return nil, err + } + + if ok, err := policy.Validate(bucket); !ok { + return nil, err + } + + return &policy, nil +} + +func (p Policy) isValid() (bool, error) { + if p.Version == "" { + return false, errors.New("policy version cannot be empty") + } + + return true, nil +} + +func (p Policy) Validate(bucket string) (bool, error) { + if ok, err1 := p.isValid(); !ok { + return false, err1 + } + + for _, s := range p.Statements { + if ok, err := s.Validate(bucket); !ok { + return false, err + } + } + + return true, nil +} + +// check policy is allowed for request +// https://docs.aws.amazon.com/zh_cn/IAM/latest/UserGuide/reference_policies_evaluation-logic.html +// 如果适用策略包含 Deny 语句，则请求会导致显式拒绝。 +// 如果应用于请求的策略包含一个 Allow 语句和一个 Deny 语句，Deny 语句优先于 Allow 语句。将显式拒绝请求。 +// 当没有适用的 Deny 语句但也没有适用的 Allow 语句时，会发生隐式拒绝。 +func (p *Policy) IsAllowed(params *RequestParam) bool { + for _, s := range p.Statements { + if s.Effect == Deny { + if !s.IsAllowed(params) { + return false + } + } + } + + if params.isOwner { + return true + } + + for _, s := range p.Statements { + if s.Effect == Allow { + if s.IsAllowed(params) { + return true + } + } + } + + return false +} + +func (o *ObjectNode) policyCheck(f http.HandlerFunc, actions []Action) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + err error + ec *ErrorCode + ) + allowed := false + defer func() { + if allowed { + f(w, r) + } else { + if ec == nil { + ec = &AccessDenied + } + o.errorResponse(w, r, err, ec) + } + }() + + param, err1 := o.parseRequestParam(r) + if err1 != nil { + err = err1 + log.LogInfof("parse Request Param err %v", err) + return + } + if param.vol == nil { + log.LogInfof("vol is null") + allowed = true + return + } + + param.actions = actions + + //check policy and acl + acl := param.vol.loadACL() + policy := param.vol.loadPolicy() + //check ip policy + if policy != nil { + allowed = policy.IsAllowed(param) + if !allowed { + log.LogWarnf("policy not allowed %v", param) + return + } + } + + if acl != nil { + allowed = acl.IsAllowed(param) + if !allowed { + log.LogWarnf("acl not allowed %v", param) + return + } + } + + } +} diff --git a/objectnode/policy_action.go b/objectnode/policy_action.go new file mode 100644 index 000000000..71c0b513e --- /dev/null +++ b/objectnode/policy_action.go @@ -0,0 +1,94 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html + +// https://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html +// https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/amazon-s3-policy-keys.html +type Action string + +const ( + GetObjectAction Action = "s3:GetObject" + PutObjectAction = "s3:PutObject" + DeleteObjectAction = "s3:DeleteObject" // + HeadObjectAction = "s3:HeadObject" + CreateBucketAction = "s3:CreateBucket" // + ListBucketAction = "s3:ListBucket" + ListBucketVersionsAction = "s3:ListBucketVersions" // List + ListBucketMultipartUploadsAction = "s3:ListBucketMultipartUploads" + GetBucketPolicyAction = "s3:GetBucketPolicy" + PutBucketPolicyAction = "s3:PutBucketPolicy" + GetBucketAclAction = "s3:GetBucketAcl" + PutBucketAclAction = "s3:PutBucketAcl" + GetObjectAclAction = "s3:GetObjectAcl" + GetObjectVersionAction = "s3:GetObjectVersion" + PutObjectVersionAction = "s3:PutObjectVersion" + GetObjectTorrentAction = "s3:GetObjectTorrent" + PutObjectTorrentAction = "s3:PutObjectTorrent" + PutObjectAclAction = "s3:PutObjectAcl" + GetObjectVersionAclAction = "s3:GetObjectVersionAcl" + PutObjectVersionAclAction = "s3:PutObjectVersionAcl" + DeleteBucketPolicyAction = "s3:DeleteBucketPolicy" + ListMultipartUploadPartsAction = "s3:ListMultipartUploadParts" + AbortMultipartUploadAction = "s3:AbortMultipartUpload" + GetBucketLocationAction = "s3:GetBucketLocation" +) + +func (s Statement) checkActions(p *RequestParam) bool { + if s.Actions.Empty() { + return true + } + for _, pa := range p.actions { + if s.Actions.ContainsWithAny(string(pa)) { + return true + } + } + + return false +} + +func (s Statement) checkNotActions(p *RequestParam) bool { + if s.NotActions.Empty() { + return true + } + for _, pa := range p.actions { + if s.NotActions.ContainsWithAny(string(pa)) { + return false + } + } + + return true +} + +// +func IsIntersectionActions(actions1, actions2 []Action) bool { + if len(actions1) == 0 && len(actions2) == 0 { + return true + } + as1, as2 := actions1, actions2 + if len(actions1) > len(actions2) { + as1, as2 = actions2, actions1 + } + for _, action1 := range as1 { + for _, action2 := range as2 { + if action1 == action2 { + return true + } + } + } + + return false +} diff --git a/objectnode/policy_condition.go b/objectnode/policy_condition.go new file mode 100644 index 000000000..5844ce0e4 --- /dev/null +++ b/objectnode/policy_condition.go @@ -0,0 +1,432 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html + +import ( + "fmt" + "net/http" + "strconv" + "strings" + "time" +) + +//https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/example-bucket-policies.html +type ConditionValues map[string]StringSet +type Condition map[ConditionType]ConditionValues +type ConditionType string +type ConditionKey string + +// https://docs.aws.amazon.com/zh_cn/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html +const ( + IpAddress ConditionType = "IpAddress" + NotIpAddress = "NotIpAddress" + StringLike = "StringLike" + StringNotLike = "StringNotLike" + StringEquals = "StringEquals" + StringNotEquals = "StringNotEquals" + Bool = "Bool" + DateEquals = "DateEquals" + DateNotEquals = "DateNotEquals" + DateLessThan = "DateLessThan" + DateLessThanEquals = "DateLessThanEquals" + DateGreaterThan = "DateGreaterThan" + DateGreaterThanEquals = "DateGreaterThanEquals" + NumericEquals = "NumericEquals" + NumericNotEquals = "NumericNotEquals" + NumericLessThan = "NumericLessThan" + NumericLessThanEquals = "NumericLessThanEquals" + NumericGreaterThan = "NumericGreaterThan" + NumericGreaterThanEquals = "NumericGreaterThanEquals" + ArnEquals = "ArnEquals" + ArnLike = "ArnLike" // + ArnNotEquals = "ArnNotEquals" + ArnNotLike = "ArnNotLike" // +) + +var ( + StringFuncs = []ConditionFunc{StringEqualsFunc, StringNotEqualsFunc, StringLikeFunc, StringNotLikeFunc} + DateFuncs = []ConditionFunc{DateEqualsFunc, DateNotEqualsFunc, DateLessThanFunc, DateGreaterThanFunc, DateLessThanEqualsFunc, DateGreaterThanEqualsFunc} + IpAddressFuncs = []ConditionFunc{IpAddressFunc, NotIpAddressFunc} + BoolFuncs = []ConditionFunc{BoolFunc} + NumericFuncs = []ConditionFunc{NumericEqualsFunc, NumericNotEqualsFunc, NumericLessThanFunc, NumericLessThanEqualsFunc, NumericGreaterThanFunc, NumericGreaterThanEqualsFunc} + ArnFuncs = []ConditionFunc{ArnEqualsFunc, ArnNotEqualsFunc, ArnLikeFunc, ArnNotLikeFunc} +) + +type ConditionTypeSet map[ConditionType]null + +var ( + StringType = ConditionTypeSet{StringEquals: void, StringNotEquals: void, StringLike: void, StringNotLike: void} + DateType = ConditionTypeSet{DateEquals: void, DateNotEquals: void, DateLessThan: void, DateGreaterThan: void, DateLessThanEquals: void, DateGreaterThanEquals: void} + IpAddressType = ConditionTypeSet{IpAddress: void, NotIpAddress: void} + BoolType = ConditionTypeSet{Bool: void} + NumericType = ConditionTypeSet{NumericEquals: void, NumericNotEquals: void, NumericLessThan: void, NumericLessThanEquals: void, NumericGreaterThan: void, NumericGreaterThanEquals: void} + ArnType = ConditionTypeSet{ArnEquals: void, ArnNotEquals: void, ArnLike: void, ArnNotLike: void} +) + +// https://docs.aws.amazon.com/zh_cn/IAM/latest/UserGuide/reference_policies_condition-keys.html +const ( + AwsCurrentTime ConditionKey = "aws:CurrentTime" + AwsEpochTime = "aws:EpochTime" + AwsMultiFactorAuthPresent = "aws:MultiFactorAuthPresent" + AwsPrincipalAccount = "aws:PrincipalAccount" + AwsPrincipalArn = "aws:PrincipalArn" + AwsPrincipalOrgID = "aws:PrincipalOrgID" + AwsPrincipalTag = "aws:PrincipalTag" + AwsPrincipalType = "aws:PrincipalType" + AwsReferer = "aws:Referer" + AwsRequestRegion = "aws:RequestRegion" + AwsRequestTagKey = "aws:RequestTag/tag-key" + AwsResourceTagKey = "aws:ResourceTag/tag-key" + AwsSecureTransport = "aws:SecureTransport" + AwsSourceAccout = "aws:AwsSourceAccout" + AwsSourceArn = "aws:SourceArn" + AwsSourceIp = "aws:SourceIp" + AwsSourceVpc = "aws:SourceVpc" + AwsSourceVpce = "aws:SourceVpce" + AwsTagKeys = "aws:TagKeys" + AwsTokenIssueTime = "aws:TokenIssueTime" + AwsUserAgent = "aws:UserAgent" + AwsUserId = "aws:userid" + AwsUserName = "aws:username" + AwsVpcSourceIp = "aws:VpcSourceIp" +) + +var ConditionKeyType = map[ConditionKey]ConditionTypeSet{ + AwsCurrentTime: DateType, + AwsEpochTime: DateType, + AwsMultiFactorAuthPresent: BoolType, + AwsPrincipalAccount: StringType, + AwsPrincipalArn: ArnType, + AwsPrincipalOrgID: StringType, + AwsPrincipalTag: StringType, + AwsPrincipalType: StringType, + AwsReferer: StringType, + AwsRequestRegion: StringType, + AwsRequestTagKey: StringType, + AwsResourceTagKey: StringType, + AwsSecureTransport: BoolType, + AwsSourceAccout: StringType, + AwsSourceArn: ArnType, + AwsSourceIp: IpAddressType, + AwsSourceVpc: StringType, + AwsSourceVpce: StringType, + AwsTagKeys: StringType, + AwsTokenIssueTime: DateType, + AwsUserAgent: StringType, + AwsUserId: StringType, + AwsUserName: StringType, + AwsVpcSourceIp: IpAddressType, +} + +var ConditionFuncMap = map[ConditionType]ConditionFunc{ + IpAddress: IpAddressFunc, + NotIpAddress: NotIpAddressFunc, + StringLike: StringLikeFunc, + StringNotLike: StringNotLikeFunc, + StringEquals: StringEqualsFunc, + StringNotEquals: StringNotEqualsFunc, + Bool: BoolFunc, + DateEquals: DateEqualsFunc, + DateNotEquals: DateNotEqualsFunc, + DateLessThan: DateLessThanFunc, + DateLessThanEquals: DateLessThanEqualsFunc, + DateGreaterThan: DateGreaterThanFunc, + DateGreaterThanEquals: DateGreaterThanEqualsFunc, +} + +type ConditionFunc func(p *RequestParam, values ConditionValues) bool + +var ( + awsTrimedPrefix = []string{"aws:", "jwt:", "s3:"} +) + +func TrimAwsPrefixKey(key string) string { + for _, prefix := range awsTrimedPrefix { + if strings.HasPrefix(key, prefix) { + return strings.TrimPrefix(key, prefix) + } + } + + return key +} + +func getCondtionValues(r *http.Request) map[string][]string { + currentTime := time.Now().UTC() + authInfo := parseRequestAuthInfo(r) + accessKey := authInfo.accessKey + principalType := "User" + if accessKey == "" { + principalType = "Anonymous" + } + values := map[string][]string{ + "SourceIp": {getRequestIP(r)}, + "UserAgent": {r.UserAgent()}, + "Referer": {r.Referer()}, + "CurrentTime": {currentTime.Format(AMZTimeFormat)}, + "EpochTime": {fmt.Sprintf("%d", currentTime.Unix())}, + "userid": {accessKey}, + "username": {accessKey}, + "PrincipalType": {principalType}, + } + + for k, v := range r.Header { + if existsV, ok := values[k]; ok { + values[k] = append(existsV, v...) + } else { + values[k] = v + } + } + + for k, v := range r.URL.Query() { + if existsV, ok := values[k]; ok { + values[k] = append(existsV, v...) + } else { + values[k] = v + } + } + + return values +} + +func IpAddressFunc(p *RequestParam, value ConditionValues) bool { + key := TrimAwsPrefixKey(AwsSourceIp) + canonicalKey := http.CanonicalHeaderKey(key) + sourceIP, ok := value[canonicalKey] + if !ok { + return false + } + for ipnet, _ := range sourceIP.values { + if ok, _ := isIPNetContainsIP(p.sourceIP, ipnet); ok { + return true + } + } + + return true +} + +func NotIpAddressFunc(p *RequestParam, values ConditionValues) bool { + return !IpAddressFunc(p, values) +} + +func StringLikeFunc(reqParam *RequestParam, storeCondVals ConditionValues) bool { + for k, storeVals := range storeCondVals { + key := TrimAwsPrefixKey(k) + canonicalKey := http.CanonicalHeaderKey(key) + if reqVals, ok := reqParam.condVals[canonicalKey]; ok { + for _, rv := range reqVals { + for sv, _ := range storeVals.values { + if match := patternMatch(rv, sv); match { + return true + } + } + } + } + } + + return false +} + +func StringNotLikeFunc(p *RequestParam, values ConditionValues) bool { + return !StringLikeFunc(p, values) +} + +func StringEqualsFunc(reqParam *RequestParam, storeCondVals ConditionValues) bool { + for k, storeVals := range storeCondVals { + key := TrimAwsPrefixKey(k) + canonicalKey := http.CanonicalHeaderKey(key) + if reqVals, ok := reqParam.condVals[canonicalKey]; ok { + for _, rv := range reqVals { + if storeVals.Contains(rv) { + return true + } + } + } else if reqVals, ok := reqParam.condVals[key]; ok { + for _, rv := range reqVals { + if storeVals.Contains(rv) { + return true + } + } + } + } + + return false +} + +func StringNotEqualsFunc(p *RequestParam, values ConditionValues) bool { + return !StringNotEqualsFunc(p, values) +} + +// check statement conditions +func (s Statement) checkConditions(param *RequestParam) bool { + if len(s.Condition) == 0 { + return true + } + for k, v := range s.Condition { + f, ok := ConditionFuncMap[k] + if !ok { + continue + } + if !f(param, v) { + return false + } + } + + return true +} + +func BoolFunc(p *RequestParam, policyCondtion ConditionValues) bool { + if len(policyCondtion) == 0 { + return true + } + for condKey, condVal := range policyCondtion { + for vals, _ := range condVal.values { + val1, _ := strconv.ParseBool(vals) + if cond, ok := p.condVals[condKey]; ok { + for _, c := range cond { + val2, _ := strconv.ParseBool(c) + return val1 == val2 + } + } + } + } + + return false +} + +func DateEqualsFunc(p *RequestParam, policyVals ConditionValues) bool { + for k, pVals := range policyVals { + for pVal, _ := range pVals.values { + pDate, err := time.Parse(AMZTimeFormat, pVal) + if err != nil { + return false + } + if reqVals, ok := p.condVals[k]; ok { + for _, reqVal := range reqVals { + reqDate, err := time.Parse(AMZTimeFormat, reqVal) + if err != nil { + return false + } + return pDate.Equal(reqDate) + } + } + } + } + + return false +} + +func DateNotEqualsFunc(p *RequestParam, value ConditionValues) bool { + return !DateEqualsFunc(p, value) +} + +func DateLessThanFunc(p *RequestParam, value ConditionValues) bool { + for k, pVals := range value { + for pVal, _ := range pVals.values { + pDate, err := time.Parse(AMZTimeFormat, pVal) + if err != nil { + return false + } + if reqVals, ok := p.condVals[k]; ok { + for _, reqVal := range reqVals { + reqDate, err := time.Parse(AMZTimeFormat, reqVal) + if err != nil { + return false + } + return reqDate.Before(pDate) + } + } + } + } + return false +} + +func DateLessThanEqualsFunc(p *RequestParam, value ConditionValues) bool { + for k, pVals := range value { + for pVal, _ := range pVals.values { + pDate, err := time.Parse(AMZTimeFormat, pVal) + if err != nil { + return false + } + if reqVals, ok := p.condVals[k]; ok { + for _, reqVal := range reqVals { + reqDate, err := time.Parse(AMZTimeFormat, reqVal) + if err != nil { + return false + } + return reqDate.Before(pDate) || reqDate.Equal(pDate) + } + } + } + } + return true +} + +func DateGreaterThanFunc(p *RequestParam, value ConditionValues) bool { + return !DateLessThanEqualsFunc(p, value) +} + +func DateGreaterThanEqualsFunc(p *RequestParam, value ConditionValues) bool { + return !DateLessThanFunc(p, value) +} + +func NumericEqualsFunc(p *RequestParam, value ConditionValues) bool { + //TODO: numeric equal implements + + return true +} + +func NumericNotEqualsFunc(p *RequestParam, value ConditionValues) bool { + return !NumericEqualsFunc(p, value) +} + +func NumericLessThanFunc(p *RequestParam, value ConditionValues) bool { + //TODO: numeric less than implements + + return true +} + +func NumericLessThanEqualsFunc(p *RequestParam, value ConditionValues) bool { + return NumericLessThanFunc(p, value) || NumericEqualsFunc(p, value) +} + +func NumericGreaterThanFunc(p *RequestParam, value ConditionValues) bool { + return !NumericLessThanEqualsFunc(p, value) +} + +func NumericGreaterThanEqualsFunc(p *RequestParam, value ConditionValues) bool { + return !NumericLessThanFunc(p, value) +} + +func ArnEqualsFunc(p *RequestParam, value ConditionValues) bool { + //TODO: numeric equal implements + + return true +} + +func ArnNotEqualsFunc(p *RequestParam, value ConditionValues) bool { + return !ArnEqualsFunc(p, value) +} + +func ArnLikeFunc(p *RequestParam, value ConditionValues) bool { + //TODO: numeric equal implements + + return true +} + +func ArnNotLikeFunc(p *RequestParam, value ConditionValues) bool { + return !ArnLikeFunc(p, value) +} diff --git a/objectnode/policy_handler.go b/objectnode/policy_handler.go new file mode 100644 index 000000000..6cba64fbb --- /dev/null +++ b/objectnode/policy_handler.go @@ -0,0 +1,114 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + + "github.com/chubaofs/chubaofs/util/log" + "github.com/gorilla/mux" +) + +func (o *ObjectNode) getBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("Get bucket acl") + var ( + err error + ec *ErrorCode + ) + defer o.errorResponse(w, r, err, ec) + + _, bucket, _, vol, err := o.parseRequestParams(r) + if bucket == "" { + ec = &NoSuchBucket + return + } + + ossMeta := vol.OSSMeta() + if ossMeta == nil { + ec = &InternalError + return + } + + policyData, err2 := json.Marshal(ossMeta.policy) + if err2 != nil { + err = err2 + ec = &InternalError + return + } + + w.Write(policyData) + + return +} + +func (o *ObjectNode) putBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + var ( + err error + ec *ErrorCode + ) + defer o.errorResponse(w, r, err, ec) + + _, bucket, _, vol, err := o.parseRequestParams(r) + if bucket == "" { + ec = &NoSuchBucket + return + } + + if r.ContentLength > BucketPolicyLimitSize { + ec = &MaxContentLength + return + } + + bytes, err2 := ioutil.ReadAll(r.Body) + if err2 != nil && err2 != io.EOF { + err = err2 + log.LogInfof("read body err, %v", err) + return + } + + policy, err3 := storeBucketPolicy(bytes, vol) + if err3 != nil { + err = err3 + log.LogErrorf("store policy err, %v", err) + return + } + + log.LogInfof("put bucket policy %v %v", bucket, policy) + + return +} + +func (o *ObjectNode) deleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + log.LogInfof("delete bucket policy...") + var ( + err error + ec *ErrorCode + ) + defer o.errorResponse(w, r, err, ec) + + vars := mux.Vars(r) + bucket := vars["bucket"] + if bucket == "" { + err = errors.New("") + ec = &NoSuchBucket + return + } + + return +} diff --git a/objectnode/policy_handler_test.go b/objectnode/policy_handler_test.go new file mode 100644 index 000000000..5206a1e41 --- /dev/null +++ b/objectnode/policy_handler_test.go @@ -0,0 +1,2 @@ +// Package s3 provides ... +package objectnode diff --git a/objectnode/policy_statement.go b/objectnode/policy_statement.go new file mode 100644 index 000000000..a369c7427 --- /dev/null +++ b/objectnode/policy_statement.go @@ -0,0 +1,114 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html + +// https://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html +//https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/example-bucket-policies.html + +type Effect string +type Principal map[string]StringSet +type Resource string + +const ( + Allow Effect = "Allow" + Deny = "Deny" +) + +type Statement struct { + Sid string `json:"Sid,omitempty"` + Effect Effect `json:"Effect"` + Principal Principal `json:"Principal"` + Actions StringSet `json:"Action,omitempty"` + NotActions StringSet `json:"NotAction,omitempty"` + Resources StringSet `json:"Resource,omitempty"` + NotResources StringSet `json:"NotResource,omitempty"` + Condition Condition `json:"Condition,omitempty"` +} + +func (s *Statement) Validate(bucket string) (bool, error) { + return s.isValid(bucket) +} + +func (s *Statement) isValid(bucket string) (bool, error) { + //TODO: + + return true, nil +} + +func (s Statement) IsAllowed(p *RequestParam) bool { + checked := s.check(p) + + if s.Effect != Allow { + return !checked + } + + return checked +} + +type CheckFuncs func(p *RequestParam) bool + +func (s Statement) check(p *RequestParam) bool { + checkFuncs := []CheckFuncs{ + s.checkPrincipal, + s.checkNotActions, + s.checkActions, + s.checkNotResources, + s.checkResources, + s.checkConditions, + } + for _, f := range checkFuncs { + checked := f(p) + if !checked { + return false + } + } + + return true +} + +func (s Statement) checkPrincipal(p *RequestParam) bool { + if len(s.Principal) == 0 { + return true + } + for _, principal := range s.Principal { + if principal.ContainsWild(p.account) { + return true + } + } + + return false +} + +func (s Statement) checkResources(p *RequestParam) bool { + if s.Resources.Empty() { + return true + } + if s.Resources.ContainsRegex(p.resource) { + return true + } + return false +} + +func (s Statement) checkNotResources(p *RequestParam) bool { + if s.NotResources.Empty() { + return true + } + if s.NotResources.ContainsRegex(p.resource) { + return false + } + return true +} diff --git a/objectnode/policy_test.go b/objectnode/policy_test.go new file mode 100644 index 000000000..a2dff9c96 --- /dev/null +++ b/objectnode/policy_test.go @@ -0,0 +1,55 @@ +// Package s3 provides ... +package objectnode + +/* + +https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/example-bucket-policies.html +https://docs.aws.amazon.com/zh_cn/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_IPAddress + +{ + "Version": "2012-10-17", + "Id": "S3PolicyId1", + "Statement": [ + { + "Sid": "IPAllow", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:*", + "Resource": "arn:aws:s3:::examplebucket/*", + "Condition": { + "IpAddress": {"aws:SourceIp": "54.240.143.0/24"}, + "NotIpAddress": {"aws:SourceIp": "54.240.143.188/32"} + } + } + ] +} + +{ + "Id":"PolicyId2", + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"AllowIPmix", + "Effect":"Allow", + "Principal":"*", + "Action":"s3:*", + "Resource":"arn:aws:s3:::examplebucket/*", + "Condition": { + "IpAddress": { + "aws:SourceIp": [ + "54.240.143.0/24", + "2001:DB8:1234:5678::/64" + ] + }, + "NotIpAddress": { + "aws:SourceIp": [ + "54.240.143.128/30", + "2001:DB8:1234:5678:ABCD::/80" + ] + } + } + } + ] +} + +*/ diff --git a/objectnode/result.go b/objectnode/result.go new file mode 100644 index 000000000..72dacae77 --- /dev/null +++ b/objectnode/result.go @@ -0,0 +1,266 @@ +package objectnode + +import ( + "encoding/xml" +) + +func MarshalXMLEntity(entity interface{}) ([]byte, error) { + var err error + var result []byte + if result, err = xml.Marshal(entity); err != nil { + return nil, err + } + return append([]byte(xml.Header), result...), nil +} + +func UnmarshalXMLEntity(bytes []byte, data interface{}) error { + err := xml.Unmarshal(bytes, data) + if err != nil { + return err + } + return nil +} + +type CopyObjectResult struct { + XMLName xml.Name `xml:"CopyObjectResult"` + ETag string `xml:"ETag"` + LastModified string `xml:"LastModified"` +} + +type Deleted struct { + XMLName xml.Name `xml:"Deleted"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId,omitempty"` + DeleteMarker string `xml:"DeleteMarker,omitempty"` + DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId,omitempty"` +} + +type Error struct { + XMLName xml.Name `xml:"Error"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId,omitempty"` + Code string `xml:"Code,omitempty"` + Message string `xml:"Message,omitempty"` +} + +type DeleteResult struct { + XMLName xml.Name `json:"DeleteResult"` + Deleted []Deleted `xml:"Deleted,omitempty"` + Error []Error `xml:"Error,omitempty"` +} + +type InitMultipartResult struct { + XMLName xml.Name `xml:"InitiateMultipartUploadResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` +} + +type CompleteMultipartResult struct { + XMLName xml.Name `xml:"CompleteMultipartUploadResult"` + Location string `xml:"Location"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` +} + +type BucketOwner struct { + XMLName xml.Name `xml:"Owner"` + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` +} + +type Upload struct { + XMLName xml.Name `xml:"Upload"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + StorageClass string `xml:"StorageClass"` + Initiated string `xml:"Initiated"` + Owner *BucketOwner `xml:"Owner"` +} + +type Part struct { + XMLName xml.Name `xml:"Part"` + PartNumber int `xml:"PartNumber"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int `xml:"Size"` +} + +type ListPartsResult struct { + XMLName xml.Name `xml:"ListPartsResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + Owner *BucketOwner `xml:"Owner"` + StorageClass string `xml:"StorageClass"` + PartNumberMarker int `xml:"PartNumberMarker"` + NextMarker int `xml:"NextPartNumberMarker"` + MaxParts int `xml:"MaxParts"` + IsTruncated bool `xml:"IsTruncated"` + Parts []*Part `xml:"Parts"` +} + +type CommonPrefix struct { + XMLName xml.Name `xml:"CommonPrefixes"` + Prefix string +} + +type ListUploadsResult struct { + XMLName xml.Name `xml:"ListMultipartUploadsResult"` + Bucket string `xml:"Bucket"` + KeyMarker string `xml:"KeyMarker"` + UploadIdMarker string `xml:"UploadIdMarker"` + NextKeyMarker string `xml:"NextKeyMarker"` + NextUploadIdMarker string `xml:"NextUploadIdMarker"` + Delimiter string `xml:"Delimiter"` + Prefix string `xml:"Prefix"` + MaxUploads int `xml:"MaxUploads"` + IsTruncated bool `xml:"IsTruncated"` + Uploads []*Upload `xml:"Uploads"` + CommonPrefixes []*CommonPrefix `xml:"CommonPrefixes"` +} + +type Content struct { + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int `xml:"Size"` + StorageClass string `xml:"StorageClass"` + Owner *BucketOwner `xml:"Owner,omitempty"` +} + +type ListBucketResult struct { + XMLName xml.Name `xml:"ListBucketResults"` + Bucket string `xml:"Name"` + Prefix string `xml:"Prefix"` + Marker string `xml:"Marker"` + MaxKeys int `xml:"MaxKeys"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + NextMarker string `xml:"NextMarker"` + Contents []*Content `xml:"Contents"` + CommonPrefixes []*CommonPrefix `xml:"CommonPrefixes"` +} + +func NewParts(fsParts []*FSPart) []*Part { + parts := make([]*Part, 0) + for _, fsPart := range fsParts { + part := &Part{ + PartNumber: fsPart.PartNumber, + LastModified: fsPart.LastModified, + ETag: fsPart.ETag, + Size: fsPart.Size, + } + parts = append(parts, part) + } + return parts +} + +func NewUploads(fsUploads []*FSUpload, accessKey string) []*Upload { + owner := &BucketOwner{ + ID: accessKey, + DisplayName: accessKey, + } + uploads := make([]*Upload, 0) + for _, fsUpload := range fsUploads { + upload := &Upload{ + Key: fsUpload.Key, + UploadId: fsUpload.UploadId, + StorageClass: StorageClassStandard, + Initiated: fsUpload.Initiated, + Owner: owner, + } + uploads = append(uploads, upload) + } + return uploads +} + +func NewBucketOwner(accessKey string) *BucketOwner { + return &BucketOwner{ + ID: accessKey, + DisplayName: accessKey, + } +} + +type Object struct { + Key string `xml:"Key"` + VersionId string `xml:"VersionId,omitempty"` +} + +type DeleteRequest struct { + XMLName xml.Name `xml:"Delete"` + Objects []Object `xml:"Object"` +} + +type DeletesResult struct { + XMLName xml.Name `xml:"DeleteObjectsOutput"` + DeletedObjects []Deleted `xml:"Deleted,omitempty"` + DeletedErrors []Error `xml:"Error,omitempty"` +} + +type CopyResult struct { + XMLName xml.Name `xml:"CopyObjectResult"` + LastModified string `xml:"LastModified,omitempty"` + ETag string `xml:"ETag,omitempty"` +} + +type ListBucketRequestV1 struct { + prefix string + delimiter string + marker string + maxKeys uint64 +} + +type ListBucketRequestV2 struct { + delimiter string + maxKeys uint64 + prefix string + contToken string + fetchOwner bool + startAfter string +} + +type ListBucketResultV2 struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix,omitempty"` + Token string `xml:"ContinuationToken,omitempty"` + NextToken string `xml:"NextContinuationToken,omitempty"` + KeyCount uint64 `xml:"KeyCount"` + MaxKeys uint64 `xml:"MaxKeys"` + Delimiter string `xml:"Delimiter,omitempty"` + IsTruncated bool `xml:"IsTruncated,omitempty"` + Contents []*Content `xml:"Contents"` + CommonPrefixes []*CommonPrefix `xml:"CommonPrefixes"` +} + +type Tag struct { + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +type Tagging struct { + XMLName xml.Name `xml:"Tagging"` + TagSet []*Tag `xml:"TagSet>Tag"` +} + +type GetBucketLocationOutput struct { + XMLName xml.Name `xml:"GetBucketLocationOutput"` + LocationConstraint string `xml:"LocationConstraint"` +} + +type XAttr struct { + Key string `xml:"key"` + Value string `xml:"Value"` +} + +type PutXAttrRequest struct { + XMLName xml.Name `xml:"PutXAttrRequest"` + XAttr *XAttr `xml:"XAttr"` +} + +type ListXAttrsResult struct { + XMLName xml.Name `xml:"ListXAttrsResult"` + XAttrs []*XAttr `xml:"XAttrs>XAttr"` +} diff --git a/objectnode/result_error.go b/objectnode/result_error.go new file mode 100644 index 000000000..2158173d7 --- /dev/null +++ b/objectnode/result_error.go @@ -0,0 +1,93 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "encoding/xml" + "net/http" + "strings" + + "github.com/chubaofs/chubaofs/util/log" +) + +type ErrorCode struct { + ErrorCode string + ErrorMessage string + StatusCode int +} + +func (code ErrorCode) ServeResponse(w http.ResponseWriter, r *http.Request) error { + var err error + var marshaled []byte + var xmlError = struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + Resource string `xml:"Resource"` + RequestId string `xml:"RequestId"` + }{ + Code: code.ErrorCode, + Message: code.ErrorMessage, + Resource: r.URL.String(), + RequestId: RequestIDFromRequest(r), + } + if marshaled, err = xml.Marshal(&xmlError); err != nil { + return err + } + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.WriteHeader(code.StatusCode) + log.LogInfof("Error info : %s", string(marshaled)) + if _, err = w.Write(marshaled); err != nil { + return err + } + return nil +} + +func ServeInternalStaticErrorResponse(w http.ResponseWriter, r *http.Request) { + w.Header().Set(HeaderNameContentType, HeaderValueContentTypeXML) + w.WriteHeader(http.StatusInternalServerError) + sb := strings.Builder{} + sb.WriteString(xml.Header) + sb.WriteString("InternalErrorWe encountered an internal error. Please try again.") + sb.WriteString(r.URL.String()) + sb.WriteString("") + sb.WriteString(RequestIDFromRequest(r)) + sb.WriteString("") + _, _ = w.Write([]byte(sb.String())) +} + +var ( + UnsupportedOperation = ErrorCode{ErrorCode: "UnsupportedOperation", ErrorMessage: "Operation is not supported", StatusCode: http.StatusBadRequest} + AccessDenied = ErrorCode{ErrorCode: "AccessDenied", ErrorMessage: "Access Denied", StatusCode: http.StatusForbidden} + BadDigest = ErrorCode{ErrorCode: "BadDigest", ErrorMessage: "The Content-MD5 you specified did not match what we received.", StatusCode: http.StatusBadRequest} + BucketNotExisted = ErrorCode{ErrorCode: "BucketNotExisted", ErrorMessage: "The requested bucket name is not existed.", StatusCode: http.StatusNotFound} + BucketNotExistedForHead = ErrorCode{ErrorCode: "BucketNotExisted", ErrorMessage: "The requested bucket name is not existed.", StatusCode: http.StatusConflict} + BucketNotEmpty = ErrorCode{ErrorCode: "BucketNotEmpty", ErrorMessage: "The bucket you tried to delete is not empty.", StatusCode: http.StatusConflict} + BucketNotOwnedByYou = ErrorCode{ErrorCode: "BucketNotOwnedByYou", ErrorMessage: "The bucket is not owned by you.", StatusCode: http.StatusConflict} + KeyTooLongError = ErrorCode{ErrorCode: "KeyTooLongError", ErrorMessage: "", StatusCode: http.StatusBadRequest} + InvalidKey = ErrorCode{ErrorCode: "InvalidKey", ErrorMessage: "Object key is Illegal", StatusCode: http.StatusBadRequest} + EntityTooSmall = ErrorCode{ErrorCode: "EntityTooSmall", ErrorMessage: "Your proposed upload is smaller than the minimum allowed object size.", StatusCode: http.StatusBadRequest} + EntityTooLarge = ErrorCode{ErrorCode: "EntityTooLarge", ErrorMessage: "Your proposed upload exceeds the maximum allowed object size.", StatusCode: http.StatusBadRequest} + IncorrectNumberOfFilesInPostRequest = ErrorCode{ErrorCode: "IncorrectNumberOfFilesInPostRequest", ErrorMessage: "POST requires exactly one file upload per request.", StatusCode: http.StatusBadRequest} + InternalError = ErrorCode{ErrorCode: "InternalError", ErrorMessage: "We encountered an internal error. Please try again.", StatusCode: http.StatusInternalServerError} + InvalidArgument = ErrorCode{ErrorCode: "InvalidArgument", ErrorMessage: "Invalid Argument", StatusCode: http.StatusBadRequest} + InvalidBucketName = ErrorCode{ErrorCode: "InvalidBucketName", ErrorMessage: "The specified bucket is not valid.", StatusCode: http.StatusBadRequest} + InvalidRange = ErrorCode{ErrorCode: "InvalidRange", ErrorMessage: "The requested range cannot be satisfied.", StatusCode: http.StatusRequestedRangeNotSatisfiable} + MissingContentLength = ErrorCode{ErrorCode: "MissingContentLength", ErrorMessage: "You must provide the Content-Length HTTP header.", StatusCode: http.StatusLengthRequired} + NoSuchBucket = ErrorCode{ErrorCode: "NoSuchBucket", ErrorMessage: "The specified bucket does not exist.", StatusCode: http.StatusNotFound} + NoSuchKey = ErrorCode{ErrorCode: "NoLoggingStatusForKey", ErrorMessage: "The specified key does not exist.", StatusCode: http.StatusNotFound} + PreconditionFailed = ErrorCode{ErrorCode: "PreconditionFailed", ErrorMessage: "At least one of the preconditions you specified did not hold.", StatusCode: http.StatusPreconditionFailed} + MaxContentLength = ErrorCode{ErrorCode: "MaxContentLength", ErrorMessage: "Content-Length is bigger than 20KB.", StatusCode: http.StatusLengthRequired} +) diff --git a/objectnode/result_test.go b/objectnode/result_test.go new file mode 100644 index 000000000..825e3a15d --- /dev/null +++ b/objectnode/result_test.go @@ -0,0 +1,199 @@ +package objectnode + +import ( + "fmt" + "testing" + "time" +) + +func TestXmlMarshal_CopyObjectResult(t *testing.T) { + result := &CopyObjectResult{ + ETag: "etag_value", + LastModified: time.Now().Format("2006-01-02 15:04:05"), + } + if marshaled, marshalErr := MarshalXMLEntity(result); marshalErr != nil { + t.Fatalf("marshal fail cause: %v", marshalErr) + } else { + t.Logf("marshal result: %v", string(marshaled)) + } +} + +func TestXmlMarshal_DeleteResult(t *testing.T) { + result := DeleteResult{ + Deleted: []Deleted{ + {Key: "sample1.txt"}, + }, + Error: []Error{ + {Key: "sample2.txt", Code: "AccessDenied", Message: "Access Denied"}, + }, + } + if marshaled, marshalErr := MarshalXMLEntity(result); marshalErr != nil { + t.Fatalf("marshal fail cause: %v", marshalErr) + } else { + t.Logf("marshal result: %v", string(marshaled)) + } +} + +func TestXmlMarshal_InitMultipartUpload(t *testing.T) { + initResult := InitMultipartResult{ + Bucket: "ngwCloud1oss", + Key: "4989/txt/5678.txt", + UploadId: "UZSEAFV367N8BZP5LQXVHFDACLTX9HXX", + } + + if bytes, err := MarshalXMLEntity(initResult); err != nil { + t.Fatalf("marshal fail cause: %v", err) + } else { + t.Logf("marshal result: %v", string(bytes)) + } +} + +func TestXMLMarshal_ListPartsResult(t *testing.T) { + owner := &BucketOwner{ + ID: "YLWBsakx5hJK4cO4NcwyE72hA9KTGQQ3", + DisplayName: "YLWBsakx5hJK4cO4NcwyE72hA9KTGQQ3", + } + parts := []*Part{ + { + PartNumber: 1, + LastModified: "Tue, 20 Aug 2019 07:29:33 GMT", + ETag: "d8e2155e77cebd8fa1c3ab77da7c2ca8", + Size: 12582912, + }, + { + PartNumber: 2, + LastModified: "Tue, 20 Aug 2019 16:09:15 GMT", + ETag: "9a7909810df6cde3dedaa06966db5a56", + Size: 12582912, + }, + } + + listPartsResult := ListPartsResult{ + Bucket: "ngwCloud1oss", + Key: "4989/txt/5678.txt", + UploadId: "UZSEAFV367N8BZP5LQXVHFDACLTX9HXX", + StorageClass: "Standard", + PartNumberMarker: 1, + NextMarker: 3, + MaxParts: 2, + IsTruncated: true, + Parts: parts, + Owner: owner, + } + + if bytes, err := MarshalXMLEntity(listPartsResult); err != nil { + t.Fatalf("marshal fail cause: %v", err) + } else { + t.Logf("marshal result: %v", string(bytes)) + } +} + +func TestXMLMarshal_ListUploadsResult(t *testing.T) { + accessKey := "YLWBsakx5hJK4cO4NcwyE72hA9KTGQQ3" + + fsUploads := []*FSUpload{ + { + Key: "mybatis-11.pdf", + UploadId: "8378DFB508AE393AAAXGXTYU28", + StorageClass: "Standard", + Initiated: "2018-09-30T19:08:42.000Z", + }, + { + Key: "books/mybatis/mybatis-11.pdf", + UploadId: "B86949F2376C6B8BKOINTQ123E", + StorageClass: "Standard", + Initiated: "2018-09-30T19:08:42.000Z", + }, + } + uploads := NewUploads(fsUploads, accessKey) + + listUploadsResult := ListUploadsResult{ + Bucket: "ngwCloud1oss", + KeyMarker: "", + UploadIdMarker: "", + NextKeyMarker: "4989/txt/5678.txt", + NextUploadIdMarker: "UZSEAFV367N8BZP5LQXVHFDACLTX9HXX", + Delimiter: "", + Prefix: "4789", + MaxUploads: 1000, + IsTruncated: false, + Uploads: uploads, + CommonPrefixes: nil, + } + + if bytes, err := MarshalXMLEntity(listUploadsResult); err != nil { + t.Fatalf("marshal fail cause: %v", err) + } else { + t.Logf("marshal result: %v", string(bytes)) + } +} + +func TestNewDeleteRequest(t *testing.T) { + objectRequest := []Object{ + { + Key: "jvsTest001_1", + //VersionId:"v0001", + }, + { + Key: "jvsTest001_2", + //VersionId:"v0001", + }, + { + Key: "jvsTest001_3", + //VersionId:"v0001", + }, + } + + deleteRequest := DeleteRequest{ + Objects: objectRequest, + } + + bytes, err := MarshalXMLEntity(deleteRequest) + if err != nil { + t.Fatalf("marshal fail cause: %v", err) + } else { + t.Logf("marshal result: %v", string(bytes)) + } +} + +func TestUnmarshalDeleteRequest(t *testing.T) { + source := ` + + + jvsTest001_1 + + + jvsTest001_2 + + + jvsTest001_3 + + + ` + deleteReq := DeleteRequest{} + err := UnmarshalXMLEntity([]byte(source), &deleteReq) + if err != nil { + fmt.Println(err) + } + fmt.Println("----", deleteReq.Objects) +} + +func TestMarshalTagging(t *testing.T) { + tagging := &Tagging{ + TagSet: []*Tag{ + { + Key: "tag1", + Value: "val1", + }, + { + Key: "tag2", + Value: "val2", + }, + }, + } + marshaled, err := MarshalXMLEntity(tagging) + if err != nil { + t.Fatalf("marshal tagging fail: err(%v)", err) + } + t.Logf("marshal tagging:\n%v", string(marshaled)) +} diff --git a/objectnode/router.go b/objectnode/router.go new file mode 100644 index 000000000..898e50d19 --- /dev/null +++ b/objectnode/router.go @@ -0,0 +1,261 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "net/http" + + "github.com/gorilla/mux" +) + +// register api routers +func (o *ObjectNode) registerApiRouters(router *mux.Router) { + + var routers []*mux.Router + bRouter := router.PathPrefix("/").Subrouter() + for _, d := range o.domains { + routers = append(routers, bRouter.Host("{bucket:.+}."+d).Subrouter()) + routers = append(routers, bRouter.Host("{bucket:.+}."+d+":{port:[0-9]+}").Subrouter()) + } + routers = append(routers, bRouter.PathPrefix("/{bucket}").Subrouter()) + + for _, r := range routers { + + // List parts + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html + r.Methods(http.MethodGet). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.listPartsHandler, []Action{ListMultipartUploadPartsAction})). + Queries("uploadId", "{uploadId:.*}") + + // Get object with presgined auth signature v2 + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html + r.Methods(http.MethodGet). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.getObjectHandler, []Action{ListBucketAction})). + Queries("AWSAccessKeyId", "{accessKey:.+}", + "Expires", "{expires:[0-9]+}", "Signature", "{signature:.+}") + + // Get object with presigned auth signature v4 + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html + r.Methods(http.MethodGet). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.getObjectHandler, []Action{ListBucketAction})). + Queries("X-Amz-Credential", "{creadential:.+}", + "X-Amz-Algorithm", "{algorithm:.+}", "X-Amz-Signature", "{signature:.+}", + "X-Amz-Date", "{date:.+}", "X-Amz-SignedHeaders", "{signedHeaders:.+}", + "X-Amz-Expires", "{expires:[0-9]+}") + + // Get object + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html + r.Methods(http.MethodGet). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.getObjectHandler, []Action{GetObjectAction})) + + // Create multipart upload + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html + r.Methods(http.MethodPost). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.createMultipleUploadHandler, []Action{PutObjectAction})). + Queries("uploads", "") + + // Complete multipart + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html + r.Methods(http.MethodPost). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.completeMultipartUploadHandler, []Action{PutObjectAction})). + Queries("uploadId", "{uploadId:.*}") + + // Upload part + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html . + r.Methods(http.MethodPut). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.uploadPartHandler, []Action{PutObjectAction})). + Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + + // Copy object + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html . + r.Methods(http.MethodPut). + Path("/{object:.+}"). + HeadersRegexp(HeaderNameCopySource, ".*?(\\/|%2F).*?"). + HandlerFunc(o.policyCheck(o.copyObjectHandler, []Action{PutObjectAction})) + + // Put object + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + r.Methods(http.MethodPut). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.putObjectHandler, []Action{PutObjectAction})) + + // Delete object + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html . + r.Methods(http.MethodDelete). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.deleteObjectHandler, []Action{DeleteObjectAction})) + + // Abort multipart + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html . + r.Methods(http.MethodDelete). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.abortMultipartUploadHandler, []Action{AbortMultipartUploadAction})). + Queries("uploadId", "{uploadId:.*}") + + // Head object + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html + r.Methods(http.MethodHead). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.headObjectHandler, []Action{GetObjectAction})) + + // Get object tagging + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html + r.Methods(http.MethodGet). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.getObjectTagging, []Action{GetBucketPolicyAction})). + Queries("tagging", "") + + // Put object tagging + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html + r.Methods(http.MethodPut). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.putObjectTagging, []Action{PutBucketPolicyAction})). + Queries("tagging", "") + + // Delete object tagging + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjectTagging.html + r.Methods(http.MethodDelete). + Path("/{object:.+"). + HandlerFunc(o.policyCheck(o.deleteObjectTagging, []Action{PutBucketPolicyAction})). + Queries("tagging", "") + + // Put object xattrs + // Notes: ChubaoFS owned API for XAttr operation + r.Methods(http.MethodPut). + Path("/{object:.+}"). + HandlerFunc(o.putObjectXAttr). + Queries("xattr", "") + + // Delete object xattrs + // Notes: ChubaoFS owned API for XAttr operation + r.Methods(http.MethodDelete). + Path("/{object:.+}"). + HandlerFunc(o.deleteObjectXAttr). + Queries("xattr", "key", "{key:.+}}") + + // Get object XAttr + // Notes: ChubaoFS owned API for XAttr operation + r.Methods(http.MethodGet). + Path("/{object:.+}"). + HandlerFunc(o.getObjectXAttr). + Queries("xattr", "", "key", "{key:.+}") + + // List object XAttrs + r.Methods(http.MethodGet). + Path("/{object:.+}"). + HandlerFunc(o.listObjectXAttrs). + Queries("xattr", "") + + // List objects version 2 + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + r.Methods(http.MethodGet). + HandlerFunc(o.policyCheck(o.getBucketV2Handler, []Action{ListBucketAction})). + Queries("list-type", "2") + + // List multipart uploads + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html + r.Methods(http.MethodGet). + HandlerFunc(o.policyCheck(o.listMultipartUploadsHandler, []Action{ListMultipartUploadPartsAction})). + Queries("uploads", "") + + // List parts + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html + r.Methods(http.MethodGet). + HandlerFunc(o.policyCheck(o.listPartsHandler, []Action{ListMultipartUploadPartsAction})). + Queries("uploadId", "{uploadId:.*}") + + // Delete objects (multiple objects) + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html + r.Methods(http.MethodPost). + HandlerFunc(o.policyCheck(o.deleteObjectsHandler, []Action{DeleteObjectAction})). + Queries("delete", "") + + // List objects version 1 + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html + r.Methods(http.MethodGet). + HandlerFunc(o.policyCheck(o.getBucketV1Handler, []Action{ListBucketAction})) + + // Head bucket + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html + r.Methods(http.MethodHead). + HandlerFunc(o.policyCheck(o.headBucketHandler, []Action{ListBucketAction})) + + // Get bucket location + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html + r.Methods(http.MethodGet). + HandlerFunc(o.policyCheck(o.getBucketLocation, []Action{GetBucketLocationAction})). + Queries("location", "") + + // Get bucket policy + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketPolicy.html + r.Methods(http.MethodGet). + HandlerFunc(o.policyCheck(o.getBucketPolicyHandler, []Action{GetBucketPolicyAction})). + Queries("policy", "") + + // Put bucket policy + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketPolicy.html + r.Methods(http.MethodPut). + HandlerFunc(o.policyCheck(o.putBucketPolicyHandler, []Action{PutBucketPolicyAction})). + Queries("policy", "") + + // Delete bucket policy + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketPolicy.html + r.Methods(http.MethodDelete). + HandlerFunc(o.policyCheck(o.deleteBucketPolicyHandler, []Action{DeleteBucketPolicyAction})). + Queries("policy", "") + + // Get bucket acl + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html + r.Methods(http.MethodGet). + HandlerFunc(o.policyCheck(o.getBucketACLHandler, []Action{GetBucketAclAction})). + Queries("acl", "") + + // Put bucket acl + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html + r.Methods(http.MethodPut). + HandlerFunc(o.policyCheck(o.putBucketACLHandler, []Action{PutBucketAclAction})). + Queries("acl", "") + + // Get object acl + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html + r.Methods(http.MethodGet). + Path("/{objject:.+}"). + HandlerFunc(o.policyCheck(o.getObjectACLHandler, []Action{GetObjectAclAction})). + Queries("acl", "") + + // Put object acl + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html + r.Methods(http.MethodPut). + Path("/{object:.+}"). + HandlerFunc(o.policyCheck(o.putObjectACLHandler, []Action{PutObjectAclAction})). + Queries("acl", "") + + } + + // List buckets + // API reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html + router.Methods(http.MethodGet). + HandlerFunc(o.policyCheck(o.listBucketsHandler, []Action{ListBucketAction})) + + // Unsupported operation + router.NotFoundHandler = http.HandlerFunc(o.unsupportedOperationHandler) +} diff --git a/objectnode/server.go b/objectnode/server.go new file mode 100644 index 000000000..10e7579e0 --- /dev/null +++ b/objectnode/server.go @@ -0,0 +1,188 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "context" + "net/http" + "regexp" + "sync" + "sync/atomic" + + "github.com/chubaofs/chubaofs/util/config" + "github.com/chubaofs/chubaofs/util/errors" + "github.com/chubaofs/chubaofs/util/log" + "github.com/gorilla/mux" +) + +// The status of the s3 server +const ( + Standby uint32 = iota + Start + Running + Shutdown + Stopped +) + +// Configuration keys +const ( + configListen = "listen" + configDomains = "domains" + configMasters = "masters" + configAuthnodes = "authNodes" + configRegion = "region" +) + +// Default of configuration value +const ( + defaultListen = ":80" + defaultRegion = "cfs_default" +) + +var ( + regexpListen = regexp.MustCompile("^(([0-9]{1,3}.){3}([0-9]{1,3}))?:(\\d)+$") +) + +type ObjectNode struct { + domains []string + listen string + region string + httpServer *http.Server + vm VolumeManager + state uint32 + wg sync.WaitGroup +} + +func (o *ObjectNode) Start(cfg *config.Config) (err error) { + if atomic.CompareAndSwapUint32(&o.state, Standby, Start) { + defer func() { + if err != nil { + atomic.StoreUint32(&o.state, Standby) + } else { + atomic.StoreUint32(&o.state, Running) + } + }() + if err = o.handleStart(cfg); err != nil { + return + } + o.wg.Add(1) + } + return +} + +func (o *ObjectNode) Shutdown() { + if atomic.CompareAndSwapUint32(&o.state, Running, Shutdown) { + o.handleShutdown() + o.wg.Done() + atomic.StoreUint32(&o.state, Stopped) + } +} + +func (o *ObjectNode) Sync() { + if atomic.LoadUint32(&o.state) == Running { + o.wg.Wait() + } +} + +func (o *ObjectNode) parseConfig(cfg *config.Config) (err error) { + // parse listen + listen := cfg.GetString(configListen) + if len(listen) == 0 { + listen = defaultListen + } + if match := regexpListen.MatchString(listen); !match { + err = errors.New("invalid listen configuration") + return + } + o.listen = listen + + // parse domain + domainCfgs := cfg.GetArray(configDomains) + domains := make([]string, len(domainCfgs)) + for i, domainCfg := range domainCfgs { + domains[i] = domainCfg.(string) + } + o.domains = domains + + // parse master config + masterCfgs := cfg.GetArray(configMasters) + masters := make([]string, len(masterCfgs)) + for i, masterCfg := range masterCfgs { + masters[i] = masterCfg.(string) + } + o.vm = NewVolumeManager(masters) + o.vm.InitStore(new(xattrStore)) + + // parse region + region := cfg.GetString(configRegion) + if len(region) == 0 { + region = defaultRegion + } + o.region = region + return +} + +func (o *ObjectNode) handleStart(cfg *config.Config) (err error) { + // parse config + if err = o.parseConfig(cfg); err != nil { + return + } + // start rest api + if err = o.startMuxRestAPI(); err != nil { + log.LogInfof("handleStart: start mux rest api fail, err(%v)", err) + return + } + log.LogInfo("s3node start success") + return +} + +func (o *ObjectNode) handleShutdown() { + o.shutdownRestAPI() +} + +func (o *ObjectNode) startMuxRestAPI() (err error) { + router := mux.NewRouter().SkipClean(true) + o.registerApiRouters(router) + router.Use( + o.traceMiddleware, + o.authMiddleware, + o.contentMiddleware, + ) + + var server = &http.Server{ + Addr: o.listen, + Handler: router, + } + + go func() { + if err = server.ListenAndServe(); err != nil { + log.LogErrorf("startMuxRestAPI: start http server fail, err(%o)", err) + return + } + }() + o.httpServer = server + return +} + +func (o *ObjectNode) shutdownRestAPI() { + if o.httpServer != nil { + _ = o.httpServer.Shutdown(context.Background()) + o.httpServer = nil + } +} + +func NewServer() *ObjectNode { + return &ObjectNode{} +} diff --git a/objectnode/server_test.go b/objectnode/server_test.go new file mode 100644 index 000000000..352b89271 --- /dev/null +++ b/objectnode/server_test.go @@ -0,0 +1,54 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "fmt" + "os" + "testing" + + "github.com/chubaofs/chubaofs/util/config" + "github.com/chubaofs/chubaofs/util/log" +) + +func TestObjectNode_Lifecycle(t *testing.T) { + var err error + cfgStr := ` +{ + "listen": ":10004", + "logDir": "/tmp/Logs/chubaofs", + "module": "object" +} +` + // test log + cfg := config.LoadConfigString(cfgStr) + if _, err := log.InitLog(cfg.GetString("logDir"), "node", log.DebugLevel, nil); err != nil { + fmt.Println("Fatal: failed to init log - ", err) + os.Exit(1) + return + } + + node := NewServer() + if err = node.Start(cfg); err != nil { + t.Fatalf("start node server fail cause: %v", err) + } + //go func() { + // fmt.Printf("node server will be shutdown after 3 seconds.\n") + // time.Sleep(3 * time.Second) + // fmt.Printf("node server will be shutdown.\n") + // node.Shutdown() + //}() + node.Sync() +} diff --git a/objectnode/signer.go b/objectnode/signer.go new file mode 100644 index 000000000..476ad5784 --- /dev/null +++ b/objectnode/signer.go @@ -0,0 +1,72 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/url" + "strings" + "time" +) + +var SERVICE = "s3" +var SCHEME = "AWS4" +var ALGORITHM = "HMAC-SHA256" +var TERMINATOR = "aws4_request" + +func getStartTime(headers http.Header) string { + for headerName := range headers { + if strings.ToLower(headerName) == strings.ToLower(HeaderNameStartDate) { + return headers.Get(headerName) + } + } + return "" +} + +func getURIPath(u *url.URL) string { + var uri string + + if len(u.Opaque) > 0 { + uri = "/" + strings.Join(strings.Split(u.Opaque, "/")[3:], "/") + } else { + uri = u.EscapedPath() + } + + if len(uri) == 0 { + uri = "/" + } + + return uri +} + +func getCurrentDateStamp() string { + // get current date in UTC time zone + n := time.Now() + return n.UTC().Format("20060102") +} + +func calcHash(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[0:]) +} + +func sign(stringData string, key []byte) []byte { + mac := hmac.New(sha256.New, key) + mac.Write([]byte(stringData)) + return mac.Sum(nil) +} diff --git a/objectnode/stringset.go b/objectnode/stringset.go new file mode 100644 index 000000000..8854ea054 --- /dev/null +++ b/objectnode/stringset.go @@ -0,0 +1,135 @@ +// Copyright 2018 The ChubaoFS Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package objectnode + +// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html + +import ( + "encoding/json" + "strings" +) + +//https://docs.aws.amazon.com/zh_cn/AmazonS3/latest/dev/example-bucket-policies.html + +type null struct{} + +var ( + void null = struct{}{} +) + +type StringSet struct { + values map[string]null +} + +func (ss StringSet) String() string { + s := make([]string, 0, len(ss.values)) + for k, _ := range ss.values { + s = append(s, k) + } + return "[" + strings.Join(s, ",") + "]" +} + +func (ss StringSet) MarshalJSON() ([]byte, error) { + if len(ss.values) == 1 { + for v, _ := range ss.values { + return json.Marshal(v) + } + } + + return json.Marshal(ss.values) +} + +func (ss *StringSet) UnmarshalJSON(b []byte) error { + ss.values = make(map[string]null) + var s string + if err := json.Unmarshal(b, &s); err == nil { + ss.values[s] = void + return nil + } + var sl []string + if err := json.Unmarshal(b, &sl); err == nil { + for _, s := range sl { + ss.values[s] = void + } + return nil + } + + var il []interface{} + if err := json.Unmarshal(b, &il); err == nil { + for _, i := range il { + ss.values[i.(string)] = void + } + return nil + } + + return nil +} + +func (ss *StringSet) Empty() bool { + return len(ss.values) == 0 +} + +func (ss *StringSet) ContainsWithAny(val string) bool { + strs := strings.Split(val, ":") + if len(strs) > 1 { + prefix := strs[0] + any := prefix + ":*" + if _, ok1 := ss.values[any]; ok1 { + return true + } + } + return ss.Contains(val) +} + +func (ss *StringSet) Contains(val string) bool { + _, ok := ss.values[val] + return ok +} + +func (ss *StringSet) ContainsRegex(val string) bool { + if ss.Contains("*") { + return true + } + if ss.Contains(val) { + return true + } + for k, _ := range ss.values { + if patternMatch(k, val) { + return true + } + } + return false +} + +func (ss *StringSet) ContainsWild(val string) bool { + if ss.Contains("*") { + return true + } + if ss.Contains(val) { + return true + } + + return false +} + +func (ss *StringSet) Intersection(set *StringSet) bool { + for s, _ := range ss.values { + if set.ContainsWild(s) { + return true + } + } + + return false +} diff --git a/objectnode/util.go b/objectnode/util.go new file mode 100644 index 000000000..d98b4e9bc --- /dev/null +++ b/objectnode/util.go @@ -0,0 +1,117 @@ +package objectnode + +import ( + "regexp" + "strings" + + "net" + "net/http" + "time" + + "github.com/chubaofs/chubaofs/util" + "github.com/chubaofs/chubaofs/util/log" +) + +const ( + pathSep = "/" + tempFileNameSep = "_" +) + +func splitPath(path string) (dirs []string, filename string) { + pathParts := strings.Split(path, pathSep) + if len(pathParts) > 1 { + dirs = pathParts[:len(pathParts)-1] + } + filename = pathParts[len(pathParts)-1] + return +} + +func tempFileName(origin string) string { + return "." + origin + tempFileNameSep + util.RandomString(16, util.LowerLetter|util.UpperLetter) +} + +func formatSimpleTime(time time.Time) string { + return time.UTC().Format("2006-01-02T15:04:05") +} + +func formatTimeISO(time time.Time) string { + return time.UTC().Format("2006-01-02T15:04:05.000Z") +} + +func formatTimeRFC1123(time time.Time) string { + return time.UTC().Format(http.TimeFormat) +} + +func parseTimeRFC1123(timeStr string) (time.Time, error) { + t, err := time.Parse(http.TimeFormat, timeStr) + if err != nil { + return t, err + } + return t, err +} + +func transferError(key string, err error) Error { + // TODO: complete sys error transfer + ossError := Error{ + Key: key, + Message: err.Error(), + } + return ossError +} + +// get request remote IP +func getRequestIP(r *http.Request) string { + IPAddress := r.Header.Get("X-Real-Ip") + if IPAddress == "" { + IPAddress = r.Header.Get("X-Forwarded-For") + } + if IPAddress == "" { + IPAddress = r.RemoteAddr + } + if ok := strings.Contains(IPAddress, ":"); ok { + IPAddress = strings.Split(IPAddress, ":")[0] + } + + return IPAddress +} + +// check ipnet contains ip +// ip: 172.17.0.2 +// ipnet: 172.17.0.0/16 +func isIPNetContainsIP(ipStr, ipnetStr string) (bool, error) { + if !strings.Contains(ipnetStr, "/") { + if ipStr == ipnetStr { + return true, nil + } else { + return false, nil + } + } + _, ipnet, err := net.ParseCIDR(ipnetStr) + if err != nil { + log.LogInfof("parse ipnet error ipnet %v", ipnetStr) + return false, err + } + + ip := net.ParseIP(ipStr) + if ipnet.Contains(ip) { + return true, nil + } + + return false, nil +} + +func patternMatch(pattern, key string) bool { + if pattern == "" { + return key == pattern + } + if pattern == "*" { + return true + } + matched, err := regexp.MatchString(pattern, key) + if err != nil { + log.LogErrorf("patternMatch error %v", err) + return false + } + + return matched +} diff --git a/proto/admin_proto.go b/proto/admin_proto.go index 0ff8bb805..a041e7bf8 100644 --- a/proto/admin_proto.go +++ b/proto/admin_proto.go @@ -30,7 +30,7 @@ const ( AdminGetVol = "/admin/getVol" AdminClusterFreeze = "/cluster/freeze" AdminGetIP = "/admin/getIp" - AdminCreateMP = "/metaPartition/create" + AdminCreateMetaPartition = "/metaPartition/create" AdminSetMetaNodeThreshold = "/threshold/set" // Client APIs @@ -62,6 +62,9 @@ const ( GetDataNodeTaskResponse = "/dataNode/response" // Method: 'POST', ContentType: 'application/json' GetTopologyView = "/topo/get" + + // Header keys + SkipOwnerValidation = "Skip-Owner-Validation" ) // HTTPReply uniform response structure @@ -354,13 +357,28 @@ type MetaPartitionView struct { Status int8 } +type OSSSecure struct { + AccessKey string + SecretKey string +} + // VolView defines the view of a volume type VolView struct { Name string + Owner string Status uint8 FollowerRead bool MetaPartitions []*MetaPartitionView DataPartitions []*DataPartitionResponse + OSSSecure *OSSSecure +} + +func (v *VolView) SetOwner(owner string) { + v.Owner = owner +} + +func (v *VolView) SetOSSSecure(accessKey, secretKey string) { + v.OSSSecure = &OSSSecure{AccessKey: accessKey, SecretKey: secretKey} } func NewVolView(name string, status uint8, followerRead bool) (view *VolView) { @@ -400,8 +418,8 @@ type SimpleVolView struct { Authenticate bool } -// GetVolResponse defines the response for getting meta partition -type GetVolResponse struct { - VolViewCache []byte `json:"vol_view_cache"` - CheckMsg string `json:"check_message"` +// MasterAPIAccessResp defines the response for getting meta partition +type MasterAPIAccessResp struct { + APIResp APIAccessResp `json:"api_resp"` + Data []byte `json:"data"` } diff --git a/proto/auth_proto.go b/proto/auth_proto.go index 4e207d8f7..0f1061509 100644 --- a/proto/auth_proto.go +++ b/proto/auth_proto.go @@ -40,13 +40,14 @@ type MsgType uint32 type Nonce uint64 const ( - APIRsc = "API" - APIAccess = "access" - capSeparator = ":" - reqLiveLength = 10 - ClientMessage = "Token" - VOLRsc = "VOL" - VOLAccess = "*" + APIRsc = "API" + APIAccess = "access" + capSeparator = ":" + reqLiveLength = 10 + ClientMessage = "Token" + OwnerVOLRsc = "VOL" + NoneOwnerVOLRsc = "NoneOwnerVOL" + VOLAccess = "*" ) // api @@ -65,6 +66,11 @@ const ( //raft node APIs AdminAddRaftNode = "/admin/addraftnode" AdminRemoveRaftNode = "/admin/removeraftnode" + + // Object node APIs + OSAddCaps = "/os/addcaps" + OSDeleteCaps = "/os/deletecaps" + OSGetCaps = "/os/getcaps" ) const ( @@ -163,6 +169,24 @@ const ( // MsgAuthRemoveRaftNodeResp response type for authnode remove node MsgAuthRemoveRaftNodeResp MsgType = MsgAuthBase + 0x58001 + // MsgAuthOSAddCapsReq request type from ObjectNode to add caps + MsgAuthOSAddCapsReq MsgType = MsgAuthBase + 0x61000 + + // MsgAuthOSAddCapsResp request type from ObjectNode to add caps + MsgAuthOSAddCapsResp MsgType = MsgAuthBase + 0x61001 + + // MsgAuthOSDeleteCapsReq request type from ObjectNode to delete caps + MsgAuthOSDeleteCapsReq MsgType = MsgAuthBase + 0x62000 + + // MsgAuthOSDeleteCapsResp request type from ObjectNode to delete caps + MsgAuthOSDeleteCapsResp MsgType = MsgAuthBase + 0x62001 + + // MsgAuthOSGetCapsReq request type from ObjectNode to get caps + MsgAuthOSGetCapsReq MsgType = MsgAuthBase + 0x63000 + + // MsgAuthOSGetCapsResp response type from ObjectNode to get caps + MsgAuthOSGetCapsResp MsgType = MsgAuthBase + 0x63001 + // MsgMasterAPIAccessReq request type for master api access MsgMasterAPIAccessReq MsgType = 0x60000 @@ -190,6 +214,9 @@ var MsgType2ResourceMap = map[MsgType]string{ MsgAuthGetCapsReq: "auth:getcaps", MsgAuthAddRaftNodeReq: "auth:addnode", MsgAuthRemoveRaftNodeReq: "auth:removenode", + MsgAuthOSAddCapsReq: "auth:osaddcaps", + MsgAuthOSDeleteCapsReq: "auth:osdeletecaps", + MsgAuthOSGetCapsReq: "auth:osgetcaps", MsgMasterFetchVolViewReq: "master:getvol", } @@ -225,7 +252,7 @@ type APIAccessReq struct { Ticket string `json:"ticket"` } -// APIAccessResp defines the respose for access restful api +// APIAccessResp defines the response for access restful api // use Timestamp as verifier for MITM mitigation // verifier is also used to verify the server identity type APIAccessResp struct { @@ -241,7 +268,7 @@ type AuthAPIAccessReq struct { KeyInfo keystore.KeyInfo `json:"key_info"` } -// AuthAPIAccessResp defines the respose for creating an key in authnode +// AuthAPIAccessResp defines the response for creating an key in authnode type AuthAPIAccessResp struct { APIResp APIAccessResp `json:"api_resp"` KeyInfo keystore.KeyInfo `json:"key_info"` @@ -253,18 +280,30 @@ type AuthRaftNodeInfo struct { Addr string `json:"addr"` } -// AuthRaftNodeReq defines Auth API request for add/remove a raft ndoe +// AuthRaftNodeReq defines Auth API request for add/remove a raft node type AuthRaftNodeReq struct { APIReq APIAccessReq `json:"api_req"` RaftNodeInfo AuthRaftNodeInfo `json:"node_info"` } -// AuthRaftNodeResp defines Auth API respose for add/remove a raft ndoe +// AuthRaftNodeResp defines Auth API response for add/remove a raft node type AuthRaftNodeResp struct { APIResp APIAccessResp `json:"api_resp"` Msg string `json:"msg"` } +// AuthAPIAccessKeystoreReq defines Auth API for put/delete Access Keystore vols +type AuthAPIAccessKeystoreReq struct { + APIReq APIAccessReq `json:"api_req"` + AccessKeyInfo keystore.AccessKeyInfo `json:"access_key_info"` +} + +// AuthAPIAccessKeystoreResp defines the response for put/delete Access Keystore vols +type AuthAPIAccessKeystoreResp struct { + APIResp APIAccessResp `json:"api_resp"` + AccessKeyInfo keystore.AccessKeyInfo `json:"access_key_info"` +} + // IsValidServiceID determine the validity of a serviceID func IsValidServiceID(serviceID string) (err error) { if serviceID != AuthServiceID && serviceID != MasterServiceID && serviceID != MetaServiceID && serviceID != DataServiceID { diff --git a/proto/errors.go b/proto/errors.go index acb79423f..943242cfe 100644 --- a/proto/errors.go +++ b/proto/errors.go @@ -59,6 +59,7 @@ var ( ErrAuthAPIAccessGenRespError = errors.New("auth API access response error") ErrKeyNotExists = errors.New("key not exists") ErrDuplicateKey = errors.New("duplicate key") + ErrAccessKeyNotExists = errors.New("access key not exists") ErrInvalidTicket = errors.New("invalid ticket") ErrExpiredTicket = errors.New("expired ticket") ErrMasterAPIGenRespError = errors.New("master API generate response error") @@ -103,6 +104,7 @@ const ( ErrCodeAuthAPIAccessGenRespError ErrCodeAuthRaftNodeGenRespError ErrCodeAuthReqRedirectError + ErrCodeAccessKeyNotExists ErrCodeInvalidTicket ErrCodeExpiredTicket ErrCodeMasterAPIGenRespError @@ -145,6 +147,7 @@ var Err2CodeMap = map[error]int32{ ErrVolAuthKeyNotMatch: ErrCodeVolAuthKeyNotMatch, ErrAuthKeyStoreError: ErrCodeAuthKeyStoreError, ErrAuthAPIAccessGenRespError: ErrCodeAuthAPIAccessGenRespError, + ErrAccessKeyNotExists: ErrCodeAccessKeyNotExists, ErrInvalidTicket: ErrCodeInvalidTicket, ErrExpiredTicket: ErrCodeExpiredTicket, ErrMasterAPIGenRespError: ErrCodeMasterAPIGenRespError, diff --git a/proto/fs_proto.go b/proto/fs_proto.go index f423b2c8e..f59fbf9cc 100644 --- a/proto/fs_proto.go +++ b/proto/fs_proto.go @@ -17,6 +17,7 @@ package proto import ( "fmt" "os" + "strings" "time" ) @@ -69,6 +70,22 @@ func (info *InodeInfo) String() string { return fmt.Sprintf("Inode(%v) Mode(%v) OsMode(%v) Nlink(%v) Size(%v) Uid(%v) Gid(%v) Gen(%v)", info.Inode, info.Mode, OsMode(info.Mode), info.Nlink, info.Size, info.Uid, info.Gid, info.Generation) } +type XAttrInfo struct { + Inode uint64 + XAttrs map[string]string +} + +func (info XAttrInfo) String() string { + builder := strings.Builder{} + for k, v := range info.XAttrs { + if builder.Len() != 0 { + builder.WriteString(",") + } + builder.WriteString(fmt.Sprintf("%s:%s", k, v)) + } + return fmt.Sprintf("XAttrInfo{Inode(%v), XAttrs(%v)}", info.Inode, builder.String()) +} + // Dentry defines the dentry struct. type Dentry struct { Name string `json:"name"` @@ -214,7 +231,7 @@ type ReadDirResponse struct { Children []Dentry `json:"children"` } -// AppendExtentKeyRequest defines the request to append an extent key. +// BatchAppendExtentKeyRequest defines the request to append an extent key. type AppendExtentKeyRequest struct { VolName string `json:"vol"` PartitionID uint64 `json:"pid"` @@ -260,3 +277,136 @@ const ( AttrUid AttrGid ) + +// DeleteInodeRequest defines the request to delete an inode. +type DeleteInodeRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Inode uint64 `json:"ino"` +} + +// AppendExtentKeysRequest defines the request to append an extent key. +type AppendExtentKeysRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Inode uint64 `json:"ino"` + Extents []ExtentKey `json:"eks"` +} + +type SetXAttrRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Inode uint64 `json:"ino"` + Key string `json:"key"` + Value string `json:"val"` +} + +type GetXAttrRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Inode uint64 `json:"ino"` + Key string `json:"key"` +} + +type GetXAttrResponse struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Inode uint64 `json:"ino"` + Key string `json:"key"` + Value string `json:"val"` +} + +type RemoveXAttrRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Inode uint64 `json:"ino"` + Key string `json:"key"` +} + +type ListXAttrRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Inode uint64 `json:"ino"` +} + +type ListXAttrResponse struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Inode uint64 `json:"ino"` + XAttr map[string]string `json:"xattr"` +} + +type BatchGetXAttrRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Inodes []uint64 `json:"inos"` + Keys []string `json:"keys"` +} + +type BatchGetXAttrResponse struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + XAttrs []*XAttrInfo +} + +type MultipartInfo struct { + ID string `json:"id"` + Path string `json:"path"` + InitTime time.Time `json:"itime"` + Parts []*MultipartPartInfo `json:"parts"` +} + +type MultipartPartInfo struct { + ID uint16 `json:"id"` + Inode uint64 `json:"ino"` + MD5 string `json:"md5"` + Size uint64 `json:"sz"` + UploadTime time.Time `json:"ut"` +} + +type CreateMultipartRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Path string `json:"path"` +} + +type CreateMultipartResponse struct { + Info *MultipartInfo `json:"info"` +} + +type GetMultipartRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + MultipartId string `json:"mid"` +} + +type GetMultipartResponse struct { + Info *MultipartInfo `json:"info"` +} + +type AddMultipartPartRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + MultipartId string `json:"mid"` + Part *MultipartPartInfo `json:"part"` +} + +type RemoveMultipartRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + MultipartId string `json:"mid"` +} + +type ListMultipartRequest struct { + VolName string `json:"vol"` + PartitionId uint64 `json:"pid"` + Marker string `json:"mk"` + MultipartIdMarker string `json:"mmk"` + Max uint64 `json:"max"` + Delimiter string `json:"dm"` + Prefix string `json:"pf"` +} + +type ListMultipartResponse struct { + Multiparts []*MultipartInfo `json:"mps"` +} diff --git a/proto/packet.go b/proto/packet.go index bc4f565ef..b806cc3e0 100644 --- a/proto/packet.go +++ b/proto/packet.go @@ -19,13 +19,14 @@ import ( "encoding/json" "errors" "fmt" - "github.com/chubaofs/chubaofs/util" - "github.com/chubaofs/chubaofs/util/buf" "io" "net" "strconv" "sync/atomic" "time" + + "github.com/chubaofs/chubaofs/util" + "github.com/chubaofs/chubaofs/util/buf" ) var ( @@ -89,6 +90,14 @@ const ( //Operations: MetaNode Leader -> MetaNode Follower OpMetaFreeInodesOnRaftFollower uint8 = 0x32 + OpMetaDeleteInode uint8 = 0x33 // delete specified inode immediately and do not remove data. + OpMetaBatchExtentsAdd uint8 = 0x34 // for extents batch attachment + OpMetaSetXAttr uint8 = 0x35 + OpMetaGetXAttr uint8 = 0x36 + OpMetaRemoveXAttr uint8 = 0x37 + OpMetaListXAttr uint8 = 0x38 + OpMetaBatchGetXAttr uint8 = 0x39 + // Operations: Master -> MetaNode OpCreateMetaPartition uint8 = 0x40 OpMetaNodeHeartbeat uint8 = 0x41 @@ -112,6 +121,13 @@ const ( OpRemoveDataPartitionRaftMember uint8 = 0x68 OpDataPartitionTryToLeader uint8 = 0x69 + // Operations: MultipartInfo + OpCreateMultipart uint8 = 0x70 + OpGetMultipart uint8 = 0x71 + OpAddMultipartPart uint8 = 0x72 + OpRemoveMultipart uint8 = 0x73 + OpListMultiparts uint8 = 0x74 + // Commons OpIntraGroupNetErr uint8 = 0xF3 OpArgMismatchErr uint8 = 0xF4 @@ -320,6 +336,30 @@ func (p *Packet) GetOpMsg() (m string) { m = "OpMetaPartitionTryToLeader" case OpDataPartitionTryToLeader: m = "OpDataPartitionTryToLeader" + case OpMetaDeleteInode: + m = "OpMetaDeleteInode" + case OpMetaBatchExtentsAdd: + m = "OpMetaBatchExtentsAdd" + case OpMetaSetXAttr: + m = "OpMetaSetXAttr" + case OpMetaGetXAttr: + m = "OpMetaGetXAttr" + case OpMetaRemoveXAttr: + m = "OpMetaRemoveXAttr" + case OpMetaListXAttr: + m = "OpMetaListXAttr" + case OpMetaBatchGetXAttr: + m = "OpMetaBatchGetXAttr" + case OpCreateMultipart: + m = "OpCreateMultipart" + case OpGetMultipart: + m = "OpGetMultipart" + case OpAddMultipartPart: + m = "OpAddMultipartPart" + case OpRemoveMultipart: + m = "OpRemoveMultipart" + case OpListMultiparts: + m = "OpListMultiparts" } return } diff --git a/raftstore/partition.go b/raftstore/partition.go index 6dc677d0f..8cec6cda1 100644 --- a/raftstore/partition.go +++ b/raftstore/partition.go @@ -63,9 +63,6 @@ type Partition interface { // CommittedIndex returns the current index of the applied raft log in the raft store partition. CommittedIndex() uint64 - // FirstCommittedIndex returns the first committed index of raft log in the raft store partition. - FirstCommittedIndex() uint64 - // Truncate raft log Truncate(index uint64) @@ -159,11 +156,6 @@ func (p *partition) CommittedIndex() (applied uint64) { return } -// FirstCommittedIndex returns the first committed index of raft log in the raft store partition. -func (p *partition) FirstCommittedIndex() uint64 { - return p.raft.FirstCommittedIndex(p.id) -} - // Submit submits command data to raft log. func (p *partition) Submit(cmd []byte) (resp interface{}, err error) { if !p.IsRaftLeader() { diff --git a/raftstore/store_rocksdb.go b/raftstore/store_rocksdb.go index 914a0a855..02f654ff3 100644 --- a/raftstore/store_rocksdb.go +++ b/raftstore/store_rocksdb.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/tecbot/gorocksdb" + "os" ) // RocksDBStore is a wrapper of the gorocksdb.DB @@ -27,13 +28,15 @@ type RocksDBStore struct { } // NewRocksDBStore returns a new RocksDB instance. -func NewRocksDBStore(dir string, lruCacheSize, writeBufferSize int) (store *RocksDBStore) { - store = &RocksDBStore{dir: dir} - err := store.Open(lruCacheSize, writeBufferSize) - if err != nil { - panic(fmt.Sprintf("Failed to Open rocksDB! err:%v", err.Error())) +func NewRocksDBStore(dir string, lruCacheSize, writeBufferSize int) (store *RocksDBStore, err error) { + if err = os.MkdirAll(dir, os.ModePerm); err != nil { + return } - return store + store = &RocksDBStore{dir: dir} + if err = store.Open(lruCacheSize, writeBufferSize); err != nil { + return + } + return } // Open opens the RocksDB instance. diff --git a/sdk/data/stream/extent_client.go b/sdk/data/stream/extent_client.go index 6f75f96d3..6b34b619d 100644 --- a/sdk/data/stream/extent_client.go +++ b/sdk/data/stream/extent_client.go @@ -44,7 +44,6 @@ const ( ) var ( - gDataWrapper *wrapper.Wrapper openRequestPool *sync.Pool writeRequestPool *sync.Pool flushRequestPool *sync.Pool @@ -61,6 +60,7 @@ type ExtentClient struct { readLimiter *rate.Limiter writeLimiter *rate.Limiter + dataWrapper *wrapper.Wrapper appendExtentKey AppendExtentKeyFunc getExtents GetExtentsFunc truncate TruncateFunc @@ -74,7 +74,7 @@ func NewExtentClient(volname, master string, readRate, writeRate int64, appendEx limit := MaxMountRetryLimit retry: - gDataWrapper, err = wrapper.NewDataPartitionWrapper(volname, master) + client.dataWrapper, err = wrapper.NewDataPartitionWrapper(volname, master) if err != nil { if limit <= 0 { return nil, errors.Trace(err, "Init data wrapper failed!") @@ -89,7 +89,7 @@ retry: client.appendExtentKey = appendExtentKey client.getExtents = getExtents client.truncate = truncate - client.followerRead = gDataWrapper.FollowerRead() + client.followerRead = client.dataWrapper.FollowerRead() // Init request pools openRequestPool = &sync.Pool{New: func() interface{} { @@ -306,3 +306,7 @@ func setRate(lim *rate.Limiter, val int) string { lim.SetLimit(rate.Inf) return "unlimited" } + +func (client *ExtentClient) Close() error { + return nil +} diff --git a/sdk/data/stream/extent_handler.go b/sdk/data/stream/extent_handler.go index 9a584be60..e16f69461 100644 --- a/sdk/data/stream/extent_handler.go +++ b/sdk/data/stream/extent_handler.go @@ -485,7 +485,7 @@ func (eh *ExtentHandler) allocateExtent() (err error) { exclude := make(map[string]struct{}) for i := 0; i < MaxSelectDataPartitionForWrite; i++ { - if dp, err = gDataWrapper.GetDataPartitionForWrite(exclude); err != nil { + if dp, err = eh.stream.client.dataWrapper.GetDataPartitionForWrite(exclude); err != nil { log.LogWarnf("allocateExtent: failed to get write data partition, eh(%v) exclude(%v)", eh, exclude) continue } diff --git a/sdk/data/stream/stream_reader.go b/sdk/data/stream/stream_reader.go index 196d1ab16..407c2b29f 100644 --- a/sdk/data/stream/stream_reader.go +++ b/sdk/data/stream/stream_reader.go @@ -76,7 +76,7 @@ func (s *Streamer) GetExtents() error { // GetExtentReader returns the extent reader. // TODO: use memory pool func (s *Streamer) GetExtentReader(ek *proto.ExtentKey) (*ExtentReader, error) { - partition, err := gDataWrapper.GetDataPartition(ek.PartitionId) + partition, err := s.client.dataWrapper.GetDataPartition(ek.PartitionId) if err != nil { return nil, err } diff --git a/sdk/data/stream/stream_writer.go b/sdk/data/stream/stream_writer.go index d4d08d7bf..0920c3e46 100644 --- a/sdk/data/stream/stream_writer.go +++ b/sdk/data/stream/stream_writer.go @@ -319,7 +319,7 @@ func (s *Streamer) doOverwrite(req *ExtentRequest, direct bool) (total int, err return } - if dp, err = gDataWrapper.GetDataPartition(req.ExtentKey.PartitionId); err != nil { + if dp, err = s.client.dataWrapper.GetDataPartition(req.ExtentKey.PartitionId); err != nil { // TODO unhandled error errors.Trace(err, "doOverwrite: ino(%v) failed to get datapartition, ek(%v)", s.inode, req.ExtentKey) return diff --git a/sdk/data/wrapper/wrapper.go b/sdk/data/wrapper/wrapper.go index d5fd7ad61..3d2e08a3d 100644 --- a/sdk/data/wrapper/wrapper.go +++ b/sdk/data/wrapper/wrapper.go @@ -15,23 +15,18 @@ package wrapper import ( - "encoding/json" "fmt" "math/rand" - "net/http" "strings" "sync" "time" "github.com/chubaofs/chubaofs/proto" - "github.com/chubaofs/chubaofs/util" "github.com/chubaofs/chubaofs/util/errors" "github.com/chubaofs/chubaofs/util/log" + masterSDK "github.com/chubaofs/chubaofs/sdk/master" ) -var ( - MasterHelper = util.NewMasterHelper() -) var ( LocalIP string @@ -52,6 +47,7 @@ type Wrapper struct { rwPartition []*DataPartition localLeaderPartitions []*DataPartition followerRead bool + mc *masterSDK.MasterClient } // NewDataPartitionWrapper returns a new data partition wrapper. @@ -59,9 +55,7 @@ func NewDataPartitionWrapper(volName, masterHosts string) (w *Wrapper, err error masters := strings.Split(masterHosts, ",") w = new(Wrapper) w.masters = masters - for _, m := range w.masters { - MasterHelper.AddNode(m) - } + w.mc = masterSDK.NewMasterClient(masters, false) w.volName = volName w.rwPartition = make([]*DataPartition, 0) w.partitions = make(map[uint64]*DataPartition) @@ -85,40 +79,29 @@ func (w *Wrapper) FollowerRead() bool { return w.followerRead } -func (w *Wrapper) updateClusterInfo() error { - body, err := MasterHelper.Request(http.MethodPost, proto.AdminGetIP, nil, nil) - if err != nil { - log.LogWarnf("UpdateClusterInfo request: err(%v)", err) - return err +func (w *Wrapper) updateClusterInfo() (err error) { + var info *proto.ClusterInfo + if info, err = w.mc.AdminAPI().GetClusterInfo(); err != nil { + log.LogWarnf("UpdateClusterInfo: get cluster info fail: err(%v)", err) + return } - - info := new(proto.ClusterInfo) - if err = json.Unmarshal(body, info); err != nil { - log.LogWarnf("UpdateClusterInfo unmarshal: err(%v)", err) - return err - } - log.LogInfof("ClusterInfo: %v", *info) + log.LogInfof("UpdateClusterInfo: get cluster info: cluster(%v) localIP(%v)", info.Cluster, info.Ip) w.clusterName = info.Cluster LocalIP = info.Ip - return nil + return } -func (w *Wrapper) getSimpleVolView() error { - paras := make(map[string]string, 0) - paras["name"] = w.volName - body, err := MasterHelper.Request(http.MethodPost, proto.AdminGetVol, paras, nil) - if err != nil { - log.LogWarnf("getSimpleVolView request: err(%v)", err) - return err - } - - view := new(proto.SimpleVolView) - if err = json.Unmarshal(body, view); err != nil { - log.LogWarnf("getSimpleVolView unmarshal: err(%v)", err) - return err +func (w *Wrapper) getSimpleVolView() (err error) { + var view *proto.SimpleVolView + if view, err = w.mc.AdminAPI().GetVolumeSimpleInfo(w.volName); err != nil { + log.LogWarnf("getSimpleVolView: get volume simple info fail: volume(%v) err(%v)", w.volName, err) + return } w.followerRead = view.FollowerRead - log.LogInfof("SimpleVolView: %v", *view) + log.LogInfof("getSimpleVolView: get volume simple info: ID(%v) name(%v) owner(%v) status(%v) capacity(%v) " + + "metaReplicas(%v) dataReplicas(%v) mpCnt(%v) dpCnt(%v) followerRead(%v)", + view.ID, view.Name, view.Owner, view.Status, view.Capacity, view.MpReplicaNum, view.DpReplicaNum, view.MpCnt, + view.DpCnt, view.FollowerRead) return nil } @@ -132,24 +115,23 @@ func (w *Wrapper) update() { } } -func (w *Wrapper) updateDataPartition() error { - paras := make(map[string]string, 0) - paras["name"] = w.volName - msg, err := MasterHelper.Request(http.MethodGet, proto.ClientDataPartitions, paras, nil) - if err != nil { - return errors.Trace(err, "updateDataPartition: request to master failed!") +func (w *Wrapper) updateDataPartition() (err error) { + + var dpv *proto.DataPartitionsView + if dpv, err = w.mc.ClientAPI().GetDataPartitions(w.volName); err != nil { + log.LogErrorf("updateDataPartition: get data partitions fail: volume(%v) err(%v)", w.volName, err) + return } + log.LogInfof("updateDataPartition: get data partitions: volume(%v) partitions(%v)", len(dpv.DataPartitions)) - log.LogInfof("updateDataPartition: start!") - - view := &DataPartitionView{} - if err = json.Unmarshal(msg, view); err != nil { - return errors.Trace(err, "updateDataPartition: unmarshal failed, msg(%v)", msg) + var convert = func(response *proto.DataPartitionResponse) *DataPartition { + return &DataPartition{ DataPartitionResponse: *response } } rwPartitionGroups := make([]*DataPartition, 0) localLeaderPartitionGroups := make([]*DataPartition, 0) - for _, dp := range view.DataPartitions { + for _, partition := range dpv.DataPartitions { + dp := convert(partition) log.LogInfof("updateDataPartition: dp(%v)", dp) w.replaceOrInsertPartition(dp) if dp.Status == proto.ReadWrite { @@ -167,7 +149,7 @@ func (w *Wrapper) updateDataPartition() error { err = errors.New("updateDataPartition: no writable data partition") } - log.LogInfof("updateDataPartition: end!") + log.LogInfof("updateDataPartition: finish") return err } diff --git a/sdk/master/api_admin.go b/sdk/master/api_admin.go new file mode 100644 index 000000000..2a7fe8573 --- /dev/null +++ b/sdk/master/api_admin.go @@ -0,0 +1,181 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package master + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/chubaofs/chubaofs/master" + "github.com/chubaofs/chubaofs/proto" +) + +type AdminAPI struct { + mc *MasterClient +} + +func (api *AdminAPI) GetCluster() (cv *master.ClusterView, err error) { + var buf []byte + var request = newAPIRequest(http.MethodGet, proto.AdminGetCluster) + if buf, err = api.mc.serveRequest(request); err != nil { + return + } + cv = &master.ClusterView{} + if err = json.Unmarshal(buf, &cv); err != nil { + return + } + return +} + +func (api *AdminAPI) GetDataPartition(volName string, partitionID uint64) (partition *master.DataPartition, err error) { + var buf []byte + var request = newAPIRequest(http.MethodGet, proto.AdminGetDataPartition) + request.addParam("id", strconv.Itoa(int(partitionID))) + request.addParam("name", volName) + if buf, err = api.mc.serveRequest(request); err != nil { + return + } + partition = &master.DataPartition{} + if err = json.Unmarshal(buf, &partition); err != nil { + return + } + return +} + +func (api *AdminAPI) LoadDataPartition(volName string, partitionID uint64) (err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminLoadDataPartition) + request.addParam("id", strconv.Itoa(int(partitionID))) + request.addParam("name", volName) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} + +func (api *AdminAPI) CreateDataPartition(volName string, count int) (err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminCreateDataPartition) + request.addParam("name", volName) + request.addParam("count", strconv.Itoa(count)) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} + +func (api *AdminAPI) DecommissionDataPartition(dataPartitionID uint64, nodeAddr string) (err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminDecommissionDataPartition) + request.addParam("id", strconv.FormatUint(dataPartitionID, 10)) + request.addParam("addr", nodeAddr) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} + +func (api *AdminAPI) DeleteDataReplica(dataPartitionID uint64, nodeAddr string) (err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminDeleteDataReplica) + request.addParam("id", strconv.FormatUint(dataPartitionID, 10)) + request.addParam("addr", nodeAddr) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} + +func (api *AdminAPI) AddDataReplica(dataPartitionID uint64, nodeAddr string) (err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminAddDataReplica) + request.addParam("id", strconv.FormatUint(dataPartitionID, 10)) + request.addParam("addr", nodeAddr) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} + +func (api *AdminAPI) DeleteVolume(volName, authKey string) (err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminDeleteVol) + request.addParam("name", volName) + request.addParam("authKey", authKey) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} + +func (api *AdminAPI) UpdateVolume(volName string, capacity uint64, replicas int, followerRead bool, authKey string) (err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminUpdateVol) + request.addParam("name", volName) + request.addParam("authKey", authKey) + request.addParam("capacity", strconv.FormatUint(capacity, 10)) + request.addParam("replicaNum", strconv.Itoa(replicas)) + request.addParam("followerRead", strconv.FormatBool(followerRead)) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} + +func (api *AdminAPI) CreateVolume(volName, owner string, mpCount int, + dpSize uint64, capacity uint64, replicas int, followerRead bool) (err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminCreateVol) + request.addParam("name", volName) + request.addParam("owner", owner) + request.addParam("mpCount", strconv.Itoa(mpCount)) + request.addParam("size", strconv.FormatUint(dpSize, 10)) + request.addParam("capacity", strconv.FormatUint(capacity, 10)) + request.addParam("followerRead", strconv.FormatBool(followerRead)) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} + +func (api *AdminAPI) GetVolumeSimpleInfo(volName string) (vv *proto.SimpleVolView, err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminGetVol) + request.addParam("name", volName) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + vv = &proto.SimpleVolView{} + if err = json.Unmarshal(data, &vv); err != nil { + return + } + return +} + +func (api *AdminAPI) GetClusterInfo() (ci *proto.ClusterInfo, err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminGetIP) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + ci = &proto.ClusterInfo{} + if err = json.Unmarshal(data, &ci); err != nil { + return + } + return +} + +func (api *AdminAPI) CreateMetaPartition(volName string, inodeStart uint64) (err error) { + var request = newAPIRequest(http.MethodGet, proto.AdminCreateMetaPartition) + request.addParam("name", volName) + request.addParam("start", strconv.FormatUint(inodeStart, 10)) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} diff --git a/sdk/master/api_client.go b/sdk/master/api_client.go new file mode 100644 index 000000000..07d4bc023 --- /dev/null +++ b/sdk/master/api_client.go @@ -0,0 +1,138 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package master + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/chubaofs/chubaofs/master" + "github.com/chubaofs/chubaofs/proto" +) + +type Decoder func([]byte) ([]byte, error) + +func (d Decoder) Decode(raw []byte) ([]byte, error) { + return d(raw) +} + +type ClientAPI struct { + mc *MasterClient +} + +func (api *ClientAPI) GetVolume(volName string, authKey string) (vv *proto.VolView, err error) { + var request = newAPIRequest(http.MethodPost, proto.ClientVol) + request.addParam("name", volName) + request.addParam("authKey", authKey) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + vv = &proto.VolView{} + if err = json.Unmarshal(data, vv); err != nil { + return + } + return +} + +func (api *ClientAPI) GetVolumeWithoutAuthKey(volName string) (vv *proto.VolView, err error) { + var request = newAPIRequest(http.MethodPost, proto.ClientVol) + request.addParam("name", volName) + request.addHeader(proto.SkipOwnerValidation, strconv.FormatBool(true)) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + vv = &proto.VolView{} + if err = json.Unmarshal(data, vv); err != nil { + return + } + return +} + +func (api *ClientAPI) GetVolumeWithAuthnode(volName string, authKey string, token string, decoder Decoder) (vv *proto.VolView, err error) { + var body []byte + var request = newAPIRequest(http.MethodPost, proto.ClientVol) + request.addParam("name", volName) + request.addParam("authKey", authKey) + request.addParam(proto.ClientMessage, token) + body, err = api.mc.serveRequest(request) + if decoder != nil { + if body, err = decoder.Decode(body); err != nil { + return + } + } + vv = &proto.VolView{} + if err = json.Unmarshal(body, vv); err != nil { + return + } + return +} + +func (api *ClientAPI) GetVolumeStat(volName string) (info *proto.VolStatInfo, err error) { + var request = newAPIRequest(http.MethodGet, proto.ClientVolStat) + request.addParam("name", volName) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + info = &proto.VolStatInfo{} + if err = json.Unmarshal(data, info); err != nil { + return + } + return +} + +func (api *ClientAPI) GetMetaPartition(partitionID uint64) (partition *master.MetaPartition, err error) { + var request = newAPIRequest(http.MethodGet, proto.ClientMetaPartition) + request.addParam("id", strconv.FormatUint(partitionID, 10)) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + partition = &master.MetaPartition{} + if err = json.Unmarshal(data, partition); err != nil { + return + } + return +} + +func (api *ClientAPI) GetMetaPartitions(volName string) (views []*proto.MetaPartitionView, err error) { + var request = newAPIRequest(http.MethodGet, proto.ClientMetaPartitions) + request.addParam("name", volName) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + if err = json.Unmarshal(data, &views); err != nil { + return + } + return +} + +func (api *ClientAPI) GetDataPartitions(volName string) (view *proto.DataPartitionsView, err error) { + var request = newAPIRequest(http.MethodGet, proto.ClientDataPartitions) + request.addParam("name", volName) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + view = &proto.DataPartitionsView{} + if err = json.Unmarshal(data, view); err != nil { + return + } + return +} diff --git a/sdk/master/api_node.go b/sdk/master/api_node.go new file mode 100644 index 000000000..8ef6c7035 --- /dev/null +++ b/sdk/master/api_node.go @@ -0,0 +1,105 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package master + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/chubaofs/chubaofs/master" + "github.com/chubaofs/chubaofs/proto" +) + +type NodeAPI struct { + mc *MasterClient +} + +func (api *NodeAPI) AddDataNode(serverAddr string) (id uint64, err error) { + var request = newAPIRequest(http.MethodGet, proto.AddDataNode) + request.addParam("addr", serverAddr) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + id, err = strconv.ParseUint(string(data), 10, 64) + return +} + +func (api *NodeAPI) AddMetaNode(serverAddr string) (id uint64, err error) { + var request = newAPIRequest(http.MethodGet, proto.AddMetaNode) + request.addParam("addr", serverAddr) + var data []byte + if data, err = api.mc.serveRequest(request); err != nil { + return + } + id, err = strconv.ParseUint(string(data), 10, 64) + return +} + +func (api *NodeAPI) GetDataNode(serverHost string) (node *master.DataNode, err error) { + var buf []byte + var request = newAPIRequest(http.MethodGet, proto.GetDataNode) + request.addParam("addr", serverHost) + if buf, err = api.mc.serveRequest(request); err != nil { + return + } + node = &master.DataNode{} + if err = json.Unmarshal(buf, &node); err != nil { + return + } + return +} + +func (api *NodeAPI) GetMetaNode(serverHost string) (node *master.MetaNode, err error) { + var buf []byte + var request = newAPIRequest(http.MethodGet, proto.GetMetaNode) + request.addParam("addr", serverHost) + if buf, err = api.mc.serveRequest(request); err != nil { + return + } + node = &master.MetaNode{} + if err = json.Unmarshal(buf, &node); err != nil { + return + } + return +} + +func (api *NodeAPI) ResponseMetaNodeTask(task *proto.AdminTask) (err error) { + var encoded []byte + if encoded, err = json.Marshal(task); err != nil { + return + } + var request = newAPIRequest(http.MethodPost, proto.GetMetaNodeTaskResponse) + request.addBody(encoded) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} + +func (api *NodeAPI) ResponseDataNodeTask(task *proto.AdminTask) (err error) { + + var encoded []byte + if encoded, err = json.Marshal(task); err != nil { + return + } + var request = newAPIRequest(http.MethodPost, proto.GetDataNodeTaskResponse) + request.addBody(encoded) + if _, err = api.mc.serveRequest(request); err != nil { + return + } + return +} diff --git a/sdk/master/client.go b/sdk/master/client.go new file mode 100644 index 000000000..8c3a57299 --- /dev/null +++ b/sdk/master/client.go @@ -0,0 +1,238 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package master + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/chubaofs/chubaofs/proto" + "io/ioutil" + "net/http" + "strings" + "sync" + "time" + + "github.com/chubaofs/chubaofs/util/log" +) + +const ( + requestTimeout = 30 * time.Second +) + +var ( + ErrNoValidMaster = errors.New("no valid master") +) + +type MasterClient struct { + sync.RWMutex + masters []string + useSSL bool + leaderAddr string +} + +// AddNode add the given address as the master address. +func (c *MasterClient) AddNode(address string) { + c.Lock() + c.updateMaster(address) + c.Unlock() +} + +// Leader returns the current leader address. +func (c *MasterClient) Leader() (addr string) { + c.RLock() + addr = c.leaderAddr + c.RUnlock() + return +} + +func (c *MasterClient) AdminAPI() *AdminAPI { + return &AdminAPI{ + mc: c, + } +} + +func (c *MasterClient) ClientAPI() *ClientAPI { + return &ClientAPI{ + mc: c, + } +} + +func (c *MasterClient) NodeAPI() *NodeAPI { + return &NodeAPI{ + mc: c, + } +} + +// Change the leader address. +func (c *MasterClient) setLeader(addr string) { + c.Lock() + c.leaderAddr = addr + c.Unlock() +} + +func (c *MasterClient) serveRequest(r *request) (repsData []byte, err error) { + leaderAddr, nodes := c.prepareRequest() + host := leaderAddr + for i := -1; i < len(nodes); i++ { + if i == -1 { + if host == "" { + continue + } + } else { + host = nodes[i] + } + var resp *http.Response + var schema string + if c.useSSL { + schema = "https" + } else { + schema = "http" + } + var url = fmt.Sprintf("%s://%s%s", schema, host, + r.path) + resp, err = c.httpRequest(r.method, url, r.params, r.header, r.body) + if err != nil { + log.LogErrorf("serveRequest: send http request fail: method(%v) url(%v) err(%v)", r.method, url, err) + continue + } + stateCode := resp.StatusCode + repsData, err = ioutil.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + log.LogErrorf("serveRequest: read http response body fail: err(%v)", err) + continue + } + switch stateCode { + case http.StatusForbidden: + curMasterAddr := strings.TrimSpace(string(repsData)) + curMasterAddr = strings.Replace(curMasterAddr, "\n", "", -1) + if len(curMasterAddr) == 0 { + log.LogWarnf("serveRequest: server response status 403: request(%s) status"+ + "(403), body is empty", host) + err = ErrNoValidMaster + return + } + repsData, err = c.serveRequest(r) + return + case http.StatusOK: + if leaderAddr != host { + c.setLeader(host) + } + var body = &struct { + Code int32 `json:"code"` + Msg string `json:"msg"` + Data json.RawMessage `json:"data"` + }{} + if err := json.Unmarshal(repsData, body); err != nil { + return nil, fmt.Errorf("unmarshal response body err:%v", err) + + } + // o represent proto.ErrCodeSuccess + if body.Code != 0 { + if body.Code == proto.ErrCodeInvalidTicket { + return nil, proto.ErrInvalidTicket + } else if body.Code == proto.ErrCodeExpiredTicket { + return nil, proto.ErrExpiredTicket + } else { + return nil, fmt.Errorf("request error, code[%d], msg[%s]", body.Code, body.Msg) + } + } + return []byte(body.Data), nil + default: + log.LogErrorf("serveRequest: unknown status: host(%v) uri(%v) status(%v) body(%v).", + resp.Request.URL.String(), host, stateCode, string(repsData)) + continue + } + } + err = ErrNoValidMaster + return +} + +// Nodes returns all master addresses. +func (c *MasterClient) Nodes() (nodes []string) { + c.RLock() + nodes = c.masters + c.RUnlock() + return +} + +// prepareRequest returns the leader address and all master addresses. +func (c *MasterClient) prepareRequest() (addr string, nodes []string) { + c.RLock() + addr = c.leaderAddr + nodes = c.masters + c.RUnlock() + return +} + +func (c *MasterClient) httpRequest(method, url string, param, header map[string]string, reqData []byte) (resp *http.Response, err error) { + client := &http.Client{} + reader := bytes.NewReader(reqData) + client.Timeout = requestTimeout + var req *http.Request + fullUrl := c.mergeRequestUrl(url, param) + log.LogDebugf("httpRequest: merge request url: method(%v) url(%v) bodyLength[%v].", method, fullUrl, len(reqData)) + if req, err = http.NewRequest(method, fullUrl, reader); err != nil { + return + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Connection", "close") + for k, v := range header { + req.Header.Set(k, v) + } + resp, err = client.Do(req) + return +} + +func (c *MasterClient) updateMaster(address string) { + contains := false + for _, master := range c.masters { + if master == address { + contains = true + break + } + } + if !contains { + c.masters = append(c.masters, address) + } + c.leaderAddr = address +} + +func (c *MasterClient) mergeRequestUrl(url string, params map[string]string) string { + if params != nil && len(params) > 0 { + buff := bytes.NewBuffer([]byte(url)) + isFirstParam := true + for k, v := range params { + if isFirstParam { + buff.WriteString("?") + isFirstParam = false + } else { + buff.WriteString("&") + } + buff.WriteString(k) + buff.WriteString("=") + buff.WriteString(v) + } + return buff.String() + } + return url +} + +// NewMasterHelper returns a new MasterHelper instance. +func NewMasterClient(masters []string, useSSL bool) *MasterClient { + return &MasterClient{masters: masters, useSSL: useSSL} +} diff --git a/sdk/master/request.go b/sdk/master/request.go new file mode 100644 index 000000000..305722731 --- /dev/null +++ b/sdk/master/request.go @@ -0,0 +1,44 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package master + +type request struct { + method string + path string + params map[string]string + header map[string]string + body []byte +} + +func (r *request) addParam(key, value string) { + r.params[key] = value +} + +func (r *request) addHeader(key, value string) { + r.header[key] = value +} + +func (r *request) addBody(body []byte) { + r.body = body +} + +func newAPIRequest(method string, path string) *request { + return &request{ + method: method, + path: path, + params: make(map[string]string), + header: make(map[string]string), + } +} diff --git a/sdk/meta/api.go b/sdk/meta/api.go index a65ddd940..ed46be714 100644 --- a/sdk/meta/api.go +++ b/sdk/meta/api.go @@ -20,6 +20,8 @@ import ( "syscall" "time" + "sort" + "github.com/chubaofs/chubaofs/proto" "github.com/chubaofs/chubaofs/util/log" ) @@ -158,6 +160,49 @@ func (mw *MetaWrapper) BatchInodeGet(inodes []uint64) []*proto.InodeInfo { return batchInfos } +// InodeDelete_ll is a low-level api that removes specified inode immediately +// and do not effect extent data managed by this inode. +func (mw *MetaWrapper) InodeDelete_ll(inode uint64) error { + mp := mw.getPartitionByInode(inode) + if mp == nil { + log.LogErrorf("InodeDelete: No such partition, ino(%v)", inode) + return syscall.ENOENT + } + status, err := mw.idelete(mp, inode) + if err != nil || status != statusOK { + return statusToErrno(status) + } + log.LogDebugf("InodeDelete_ll: inode(%v)", inode) + return nil +} + +func (mw *MetaWrapper) BatchGetXAttr(inodes []uint64, keys []string) ([]*proto.XAttrInfo, error) { + batchInfos := make([]*proto.XAttrInfo, 0) + var wg sync.WaitGroup + var errGlobal error + + for _, mp := range mw.partitions { + wg.Add(1) + go func(mp *MetaPartition) { + defer wg.Done() + xAttrInfos, err := mw.batchGetXAttr(mp, inodes, keys) + if err != nil { + errGlobal = err + log.LogErrorf("BatchGetXAttr: Get xattr from partion %s in batch failed cause : %s", mp.PartitionID, err) + return + } + batchInfos = append(batchInfos, xAttrInfos...) + }(mp) + } + + wg.Wait() + if errGlobal != nil { + return nil, errGlobal + } + log.LogDebugf("BatchGetXAttr: inodes(%v) xattrInfos(%v)", len(inodes), len(batchInfos)) + return batchInfos, nil +} + /* * Note that the return value of InodeInfo might be nil without error, * and the caller should make sure InodeInfo is valid before using it. @@ -313,6 +358,34 @@ func (mw *MetaWrapper) ReadDir_ll(parentID uint64) ([]proto.Dentry, error) { return children, nil } +func (mw *MetaWrapper) DentryCreate_ll(parentID uint64, name string, inode uint64, mode uint32) error { + parentMP := mw.getPartitionByInode(parentID) + if parentMP == nil { + return syscall.ENOENT + } + var err error + var status int + if status, err = mw.dcreate(parentMP, parentID, name, inode, mode); err != nil || status != statusOK { + return statusToErrno(status) + } + return nil +} + +func (mw *MetaWrapper) DentryUpdate_ll(parentID uint64, name string, inode uint64) (oldInode uint64, err error) { + parentMP := mw.getPartitionByInode(parentID) + if parentMP == nil { + err = syscall.ENOENT + return + } + var status int + status, oldInode, err = mw.dupdate(parentMP, parentID, name, inode) + if err != nil || status != statusOK { + err = statusToErrno(status) + return + } + return +} + // Used as a callback by stream sdk func (mw *MetaWrapper) AppendExtentKey(inode uint64, ek proto.ExtentKey) error { mp := mw.getPartitionByInode(inode) @@ -329,6 +402,22 @@ func (mw *MetaWrapper) AppendExtentKey(inode uint64, ek proto.ExtentKey) error { return nil } +// AppendExtentKeys append multiple extent key into specified inode with single request. +func (mw *MetaWrapper) AppendExtentKeys(inode uint64, eks []proto.ExtentKey) error { + mp := mw.getPartitionByInode(inode) + if mp == nil { + return syscall.ENOENT + } + + status, err := mw.appendExtentKeys(mp, inode, eks) + if err != nil || status != statusOK { + log.LogErrorf("AppendExtentKeys: inode(%v) extentKeys(%v) err(%v) status(%v)", inode, eks, err, status) + return statusToErrno(status) + } + log.LogDebugf("AppendExtentKeys: ino(%v) extentKeys(%v)", inode, eks) + return nil +} + func (mw *MetaWrapper) GetExtents(inode uint64) (gen uint64, size uint64, extents []proto.ExtentKey, err error) { mp := mw.getPartitionByInode(inode) if mp == nil { @@ -421,3 +510,206 @@ func (mw *MetaWrapper) Setattr(inode uint64, valid, mode, uid, gid uint32) error return nil } + +func (mw *MetaWrapper) InodeCreate_ll(mode, uid, gid uint32, target []byte) (*proto.InodeInfo, error) { + var ( + status int + err error + info *proto.InodeInfo + mp *MetaPartition + rwPartitions []*MetaPartition + ) + + rwPartitions = mw.getRWPartitions() + length := len(rwPartitions) + epoch := atomic.AddUint64(&mw.epoch, 1) + for i := 0; i < length; i++ { + index := (int(epoch) + i) % length + mp = rwPartitions[index] + status, info, err = mw.icreate(mp, mode, uid, gid, target) + if err == nil && status == statusOK { + return info, nil + } + } + return nil, syscall.ENOMEM +} + +// InodeUnlink_ll is a low-level api that makes specified inode link value +1. +func (mw *MetaWrapper) InodeLink_ll(inode uint64) (*proto.InodeInfo, error) { + mp := mw.getPartitionByInode(inode) + if mp == nil { + log.LogErrorf("InodeLink_ll: No such partition, ino(%v)", inode) + return nil, syscall.EINVAL + } + status, info, err := mw.ilink(mp, inode) + if err != nil || status != statusOK { + log.LogErrorf("InodeLink_ll: ino(%v) err(%v) status(%v)", inode, err, status) + return nil, statusToErrno(status) + } + return info, nil +} + +// InodeUnlink_ll is a low-level api that makes specified inode link value -1. +func (mw *MetaWrapper) InodeUnlink_ll(inode uint64) (*proto.InodeInfo, error) { + mp := mw.getPartitionByInode(inode) + if mp == nil { + log.LogErrorf("InodeUnlink_ll: No such partition, ino(%v)", inode) + return nil, syscall.EINVAL + } + status, info, err := mw.iunlink(mp, inode) + if err != nil || status != statusOK { + log.LogErrorf("InodeUnlink_ll: ino(%v) err(%v) status(%v)", inode, err, status) + return nil, statusToErrno(status) + } + return info, nil +} + +func (mw *MetaWrapper) InitMultipart_ll(path string, parentId uint64) (multipartId string, err error) { + mp := mw.getPartitionByInode(parentId) + if mp == nil { + log.LogErrorf("InitMultipart: No such partition, ino(%v)", parentId) + return "", syscall.EINVAL + } + + status, sessionId, err := mw.createSession(mp, path) + if err != nil || status != statusOK { + log.LogErrorf("InitMultipart: err(%v) status(%v)", err, status) + return "", statusToErrno(status) + } + return sessionId, nil +} + +func (mw *MetaWrapper) GetMultipart_ll(multipartId string, parentId uint64) (info *proto.MultipartInfo, err error) { + mp := mw.getPartitionByInode(parentId) + if mp == nil { + log.LogErrorf("GetMultipartRequest: No such partition, ino(%v)", parentId) + return nil, syscall.EINVAL + } + + status, multipartInfo, err := mw.getMultipart(mp, multipartId) + if err != nil || status != statusOK { + log.LogErrorf("GetMultipartRequest: err(%v) status(%v)", err, status) + return nil, statusToErrno(status) + } + return multipartInfo, nil +} + +func (mw *MetaWrapper) AddMultipartPart_ll(multipartId string, parentId uint64, partId uint16, size uint64, md5 string, inode uint64) error { + mp := mw.getPartitionByInode(parentId) + if mp == nil { + log.LogErrorf("AddMultipartPart: No such partition, ino(%v)", parentId) + return syscall.EINVAL + } + status, err := mw.addMultipartPart(mp, multipartId, partId, size, md5, inode) + if err != nil || status != statusOK { + log.LogErrorf("AddMultipartPart: err(%v) status(%v)", err, status) + return statusToErrno(status) + } + return nil +} + +func (mw *MetaWrapper) RemoveMultipart_ll(multipartID string, parentId uint64) error { + mp := mw.getPartitionByInode(parentId) + if mp == nil { + log.LogErrorf("RemoveMultipart_ll: no such partition, ino(%v)", parentId) + return syscall.EINVAL + } + + status, err := mw.removeMultipart(mp, multipartID) + if err != nil || status != statusOK { + log.LogErrorf("RemoveMultipart_ll: partition get multipart fail, partitionID(%v) multipartID(%v) err(%v) status(%v)", + mp.PartitionID, multipartID, err, status) + return statusToErrno(status) + } + return nil +} + +func (mw *MetaWrapper) ListMultipart_ll(prefix, delimiter, keyMarker string, multipartIdMarker string, maxUploads uint64) (sessionResponse []*proto.MultipartInfo, err error) { + partitions := mw.partitions + var wg = sync.WaitGroup{} + //var prefixes = make([]string, 0) + var sessions = make([]*proto.MultipartInfo, 0) + //var allSessions = make([]*proto.ListMultipartResponse, 0) + + for _, mp := range partitions { + wg.Add(1) + go func(mp *MetaPartition) { + defer wg.Done() + status, response, err := mw.listSessions(mp, prefix, delimiter, keyMarker, multipartIdMarker, maxUploads+1) + if err != nil || status != statusOK { + log.LogErrorf("ListMultipart: partition list multipart fail, partitionID(%v) err(%v) status(%v)", + mp.PartitionID, err, status) + err = statusToErrno(status) + return + } + //allSessions = append(allSessions, response) + sessions = append(sessions, response.Multiparts...) + }(mp) + } + + // combine sessions from per partition + wg.Wait() + + // reorder sessions by path + sort.SliceStable(sessions, func(i, j int) bool { + return sessions[i].Path < sessions[j].Path + }) + return sessions, nil +} + +func (mw *MetaWrapper) XAttrSet_ll(inode uint64, name, value []byte) error { + var err error + mp := mw.getPartitionByInode(inode) + if mp == nil { + log.LogErrorf("XAttrSet_ll: no such partition, inode(%v)", inode) + return syscall.ENOENT + } + var status int + status, err = mw.setXAttr(mp, inode, name, value) + if err != nil || status != statusOK { + return statusToErrno(status) + } + log.LogDebugf("XAttrSet_ll: set xattr, inode(%v) name(%v) value(%v) status(%v)", inode, name, value, status) + return nil +} + +func (mw *MetaWrapper) XAttrGet_ll(inode uint64, name string) (*proto.XAttrInfo, error) { + mp := mw.getPartitionByInode(inode) + if mp == nil { + log.LogErrorf("InodeGet_ll: no such partition, ino(%v)", inode) + return nil, syscall.ENOENT + } + + value, status, err := mw.getXAttr(mp, inode, name) + if err != nil || status != statusOK { + return nil, statusToErrno(status) + } + + xAttrValues := make(map[string]string) + xAttrValues[name] = string(value) + + xAttr := &proto.XAttrInfo{ + Inode: inode, + XAttrs: xAttrValues, + } + + log.LogDebugf("XAttrGet_ll: get xattr, inode(%v) name(%v) value(%v)", inode, string(name), string(value)) + return xAttr, nil +} + +// XAttrDel_ll is a low-level meta api that deletes specified xattr. +func (mw *MetaWrapper) XAttrDel_ll(inode uint64, name string) error { + var err error + mp := mw.getPartitionByInode(inode) + if mp == nil { + log.LogErrorf("XAttrDel_ll: no such partition, inode(%v)", inode) + return syscall.ENOENT + } + var status int + status, err = mw.removeXAttr(mp, inode, name) + if err != nil || status != statusOK { + return statusToErrno(status) + } + log.LogDebugf("XAttrDel_ll: remove xattr, inode(%v) name(%v) status(%v)", inode, name, status) + return nil +} diff --git a/sdk/meta/meta.go b/sdk/meta/meta.go index f4d965352..724ff956b 100644 --- a/sdk/meta/meta.go +++ b/sdk/meta/meta.go @@ -16,6 +16,8 @@ package meta import ( "fmt" + "github.com/chubaofs/chubaofs/util/auth" + "github.com/chubaofs/chubaofs/util/cryptoutil" "io/ioutil" "log" "net/http" @@ -26,11 +28,10 @@ import ( "github.com/chubaofs/chubaofs/proto" "github.com/chubaofs/chubaofs/util" - "github.com/chubaofs/chubaofs/util/auth" "github.com/chubaofs/chubaofs/util/btree" - "github.com/chubaofs/chubaofs/util/cryptoutil" "github.com/chubaofs/chubaofs/util/errors" cfslog "github.com/chubaofs/chubaofs/util/log" + masterSDK "github.com/chubaofs/chubaofs/sdk/master" ) const ( @@ -59,12 +60,14 @@ const ( type MetaWrapper struct { sync.RWMutex - cluster string - localIP string - volname string - owner string - master util.MasterHelper - conns *util.ConnectPool + cluster string + localIP string + volname string + ossSecure *OSSSecure + owner string + ownerValidation bool + mc *masterSDK.MasterClient + conns *util.ConnectPool // Partitions and ranges should be modified together. So do not // use partitions and ranges directly. Use the helper functions instead. @@ -87,6 +90,9 @@ type MetaWrapper struct { accessToken proto.APIAccessReq sessionKey string ticketMess auth.TicketMess + + closeCh chan struct{} + closeOnce sync.Once } //the ticket from authnode @@ -97,10 +103,11 @@ type Ticket struct { Ticket string `json:"ticket"` } -func NewMetaWrapper(volname, owner, masterHosts string, authenticate bool, ticketMess auth.TicketMess) (*MetaWrapper, error) { +func NewMetaWrapper(volname, owner, masterHosts string, authenticate, validateOwner bool, ticketMess *auth.TicketMess) (*MetaWrapper, error) { mw := new(MetaWrapper) + mw.closeCh = make(chan struct{}, 1) if authenticate { - ticket, err := getTicketFromAuthnode(owner, ticketMess) + ticket, err := getTicketFromAuthnode(owner, *ticketMess) if err != nil { return nil, errors.Trace(err, "Get ticket from authnode failed!") } @@ -109,21 +116,19 @@ func NewMetaWrapper(volname, owner, masterHosts string, authenticate bool, ticke mw.accessToken.ClientID = owner mw.accessToken.ServiceID = proto.MasterServiceID mw.sessionKey = ticket.SessionKey - mw.ticketMess = ticketMess + mw.ticketMess = *ticketMess } mw.volname = volname mw.owner = owner - master := strings.Split(masterHosts, HostsSeparator) - mw.master = util.NewMasterHelper() - for _, ip := range master { - mw.master.AddNode(ip) - } + mw.ownerValidation = validateOwner + masters := strings.Split(masterHosts, HostsSeparator) + mw.mc = masterSDK.NewMasterClient(masters, false) mw.conns = util.NewConnectPool() mw.partitions = make(map[uint64]*MetaPartition) mw.ranges = btree.New(32) mw.rwPartitions = make([]*MetaPartition, 0) - mw.updateClusterInfo() - mw.updateVolStatInfo() + _ = mw.updateClusterInfo() + _ = mw.updateVolStatInfo() limit := MaxMountRetryLimit retry: @@ -142,6 +147,16 @@ retry: return mw, nil } +func (mw *MetaWrapper) OSSSecure() (accessKey, secretKey string) { + return mw.ossSecure.AccessKey, mw.ossSecure.SecretKey +} + +func (mw *MetaWrapper) Close() { + mw.closeOnce.Do(func() { + close(mw.closeCh) + }) +} + func (mw *MetaWrapper) Cluster() string { return mw.cluster } diff --git a/sdk/meta/operation.go b/sdk/meta/operation.go index 591a99bb9..c71533889 100644 --- a/sdk/meta/operation.go +++ b/sdk/meta/operation.go @@ -648,3 +648,475 @@ func (mw *MetaWrapper) setattr(mp *MetaPartition, inode uint64, valid, mode, uid log.LogDebugf("setattr exit: packet(%v) mp(%v) req(%v)", packet, mp, *req) return statusOK, nil } + +func (mw *MetaWrapper) createSession(mp *MetaPartition, path string) (status int, multipartId string, err error) { + req := &proto.CreateMultipartRequest{ + PartitionId: mp.PartitionID, + VolName: mw.volname, + Path: path, + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpCreateMultipart + err = packet.MarshalData(req) + if err != nil { + log.LogErrorf("create session: err(%v)", err) + return + } + + log.LogDebugf("createSession enter: packet(%v) mp(%v) req(%v)", packet, mp, string(packet.Data)) + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("createSession: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("createSession: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + + resp := new(proto.CreateMultipartResponse) + err = packet.UnmarshalData(resp) + if err != nil { + log.LogErrorf("createSession: packet(%v) mp(%v) req(%v) err(%v) PacketData(%v)", packet, mp, *req, err, string(packet.Data)) + return + } + return statusOK, resp.Info.ID, nil +} + +func (mw *MetaWrapper) getMultipart(mp *MetaPartition, multipartId string) (status int, info *proto.MultipartInfo, err error) { + req := &proto.GetMultipartRequest{ + PartitionId: mp.PartitionID, + VolName: mw.volname, + MultipartId: multipartId, + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpGetMultipart + err = packet.MarshalData(req) + if err != nil { + log.LogErrorf("get session: err(%v)", err) + return + } + + log.LogDebugf("getMultipart enter: packet(%v) mp(%v) req(%v)", packet, mp, string(packet.Data)) + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("getMultipart: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("getMultipart: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + + resp := new(proto.GetMultipartResponse) + err = packet.UnmarshalData(resp) + if err != nil { + log.LogErrorf("getMultipart: packet(%v) mp(%v) req(%v) err(%v) PacketData(%v)", packet, mp, *req, err, string(packet.Data)) + return + } + + return statusOK, resp.Info, nil +} + +func (mw *MetaWrapper) addMultipartPart(mp *MetaPartition, multipartId string, partId uint16, size uint64, md5 string, indoe uint64) (status int, err error) { + part := &proto.MultipartPartInfo{ + ID: partId, + Inode: indoe, + MD5: md5, + Size: size, + } + + req := &proto.AddMultipartPartRequest{ + PartitionId: mp.PartitionID, + VolName: mw.volname, + MultipartId: multipartId, + Part: part, + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpAddMultipartPart + err = packet.MarshalData(req) + if err != nil { + log.LogErrorf("addMultipartPart: marshal packet fail, err(%v)", err) + return + } + + log.LogDebugf("addMultipartPart entry: packet(%v) mp(%v) req(%v)", packet, mp, string(packet.Data)) + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("addMultipartPart: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("addMultipartPart: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + + return statusOK, nil +} + +func (mw *MetaWrapper) idelete(mp *MetaPartition, inode uint64) (status int, err error) { + req := &proto.DeleteInodeRequest{ + VolName: mw.volname, + PartitionId: mp.PartitionID, + Inode: inode, + } + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpMetaDeleteInode + if err = packet.MarshalData(req); err != nil { + log.LogErrorf("delete inode: err[%v]", err) + return + } + log.LogDebugf("delete inode: packet(%v) mp(%v) req(%v)", packet, mp, string(packet.Data)) + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("delete inode: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("idelete: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + log.LogDebugf("idelete: packet(%v) mp(%v) req(%v) ino(%v)", packet, mp, *req, inode) + return statusOK, nil +} + +func (mw *MetaWrapper) removeMultipart(mp *MetaPartition, multipartId string) (status int, err error) { + req := &proto.RemoveMultipartRequest{ + PartitionId: mp.PartitionID, + VolName: mw.volname, + MultipartId: multipartId, + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpRemoveMultipart + if err = packet.MarshalData(req); err != nil { + log.LogErrorf("delete session: err[%v]", err) + return + } + log.LogDebugf("delete session: packet(%v) mp(%v) req(%v)", packet, mp, string(packet.Data)) + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("delete session: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("delete session: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + log.LogDebugf("delete session: packet(%v) mp(%v) req(%v) PacketData(%v)", packet, mp, *req, packet.Data) + return statusOK, nil +} + +func (mw *MetaWrapper) appendExtentKeys(mp *MetaPartition, inode uint64, extents []proto.ExtentKey) (status int, err error) { + req := &proto.AppendExtentKeysRequest{ + VolName: mw.volname, + PartitionId: mp.PartitionID, + Inode: inode, + Extents: extents, + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpMetaBatchExtentsAdd + err = packet.MarshalData(req) + if err != nil { + log.LogErrorf("batch append extent: req(%v) err(%v)", *req, err) + return + } + log.LogDebugf("appendExtentKeys: batch append extent: packet(%v) mp(%v) req(%v)", packet, mp, *req) + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("batch append extent: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("batch append extent: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + + log.LogDebugf("batch append extent: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return +} + +func (mw *MetaWrapper) setXAttr(mp *MetaPartition, inode uint64, name []byte, value []byte) (status int, err error) { + req := &proto.SetXAttrRequest{ + VolName: mw.volname, + PartitionId: mp.PartitionID, + Inode: inode, + Key: string(name), + Value: string(value), + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpMetaSetXAttr + err = packet.MarshalData(req) + if err != nil { + log.LogErrorf("setXAttr: matshal packet fail, err(%v)", err) + return + } + log.LogDebugf("setXAttr: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("setXAttr: send to partition fail, packet(%v) mp(%v) req(%v) err(%v)", + packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("setXAttr: received fail status, packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + + log.LogDebugf("setXAttr: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return +} + +func (mw *MetaWrapper) getXAttr(mp *MetaPartition, inode uint64, name string) (value []byte, status int, err error) { + req := &proto.GetXAttrRequest{ + VolName: mw.volname, + PartitionId: mp.PartitionID, + Inode: inode, + Key: name, + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpMetaGetXAttr + err = packet.MarshalData(req) + if err != nil { + log.LogErrorf("get xattr: req(%v) err(%v)", *req, err) + return + } + log.LogDebugf("get xattr: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("get xattr: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("get xattr: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + + resp := new(proto.GetXAttrResponse) + if err = packet.UnmarshalData(resp); err != nil { + log.LogErrorf("get xattr: packet(%v) mp(%v) req(%v) err(%v) PacketData(%v)", packet, mp, *req, err, string(packet.Data)) + return + } + value = []byte(resp.Value) + + log.LogDebugf("get xattr: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return +} + +func (mw *MetaWrapper) removeXAttr(mp *MetaPartition, inode uint64, name string) (status int, err error) { + req := &proto.RemoveXAttrRequest{ + VolName: mw.volname, + PartitionId: mp.PartitionID, + Inode: inode, + Key: name, + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpMetaRemoveXAttr + if err = packet.MarshalData(req); err != nil { + log.LogErrorf("remove xattr: req(%v) err(%v)", *req, err) + return + } + log.LogErrorf("remove xattr: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + if packet, err = mw.sendToMetaPartition(mp, packet); err != nil { + log.LogErrorf("remove xattr: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("remove xattr: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + + log.LogErrorf("remove xattr: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return +} + +func (mw *MetaWrapper) listXAttr(mp *MetaPartition, inode uint64) (vals map[string][]byte, status int, err error) { + req := &proto.ListXAttrRequest{ + VolName: mw.volname, + PartitionId: mp.PartitionID, + Inode: inode, + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpMetaListXAttr + if err = packet.MarshalData(req); err != nil { + log.LogErrorf("list xattr: req(%v) err(%v)", *req, err) + return + } + log.LogErrorf("list xattr: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + if packet, err = mw.sendToMetaPartition(mp, packet); err != nil { + log.LogErrorf("list xattr: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("list xattr: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + + resp := new(proto.ListXAttrResponse) + if err = packet.UnmarshalData(resp); err != nil { + log.LogErrorf("list xattr: packet(%v) mp(%v) req(%v) err(%v) PacketData(%v)", packet, mp, *req, err, string(packet.Data)) + return + } + + vals = make(map[string][]byte) + for k, v := range resp.XAttr { + vals[k] = []byte(v) + } + + log.LogErrorf("list xattr: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return +} + +func (mw *MetaWrapper) listSessions(mp *MetaPartition, prefix, delimiter, keyMarker string, multipartIdMarker string, maxUploads uint64) (status int, sessions *proto.ListMultipartResponse, err error) { + req := &proto.ListMultipartRequest{ + VolName: mw.volname, + PartitionId: mp.PartitionID, + Marker: keyMarker, + MultipartIdMarker: multipartIdMarker, + Max: maxUploads, + Delimiter: delimiter, + Prefix: prefix, + } + + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpListMultiparts + err = packet.MarshalData(req) + if err != nil { + log.LogErrorf("list sessions : err(%v)", err) + return + } + + log.LogDebugf("listSessions enter: packet(%v) mp(%v) req(%v)", packet, mp, string(packet.Data)) + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("listSessions: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return + } + + status = parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("listSessions: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return + } + + resp := new(proto.ListMultipartResponse) + err = packet.UnmarshalData(resp) + if err != nil { + log.LogErrorf("listSessions: packet(%v) mp(%v) req(%v) err(%v) PacketData(%v)", packet, mp, *req, err, string(packet.Data)) + return + } + + return statusOK, resp, nil +} + +func (mw *MetaWrapper) batchGetXAttr(mp *MetaPartition, inodes []uint64, keys []string) ([]*proto.XAttrInfo, error) { + var ( + err error + ) + req := &proto.BatchGetXAttrRequest{ + VolName: mw.volname, + PartitionId: mp.PartitionID, + Inodes: inodes, + Keys: keys, + } + packet := proto.NewPacketReqID() + packet.Opcode = proto.OpMetaBatchGetXAttr + err = packet.MarshalData(req) + if err != nil { + return nil, err + } + + metric := exporter.NewTPCnt(packet.GetOpMsg()) + defer metric.Set(err) + + packet, err = mw.sendToMetaPartition(mp, packet) + if err != nil { + log.LogErrorf("batchGetXAttr: packet(%v) mp(%v) req(%v) err(%v)", packet, mp, *req, err) + return nil, err + } + + status := parseStatus(packet.ResultCode) + if status != statusOK { + log.LogErrorf("batchIget: packet(%v) mp(%v) req(%v) result(%v)", packet, mp, *req, packet.GetResultMsg()) + return nil, err + } + + resp := new(proto.BatchGetXAttrResponse) + err = packet.UnmarshalData(resp) + if err != nil { + log.LogErrorf("batchIget: packet(%v) mp(%v) err(%v) PacketData(%v)", packet, mp, err, string(packet.Data)) + return nil, err + } + + return resp.XAttrs, nil +} diff --git a/sdk/meta/partition.go b/sdk/meta/partition.go index 250aef70f..7d4a460b4 100644 --- a/sdk/meta/partition.go +++ b/sdk/meta/partition.go @@ -16,7 +16,6 @@ package meta import ( "fmt" - "github.com/chubaofs/chubaofs/util/btree" ) diff --git a/sdk/meta/view.go b/sdk/meta/view.go index 7c65576b0..da0aa775f 100644 --- a/sdk/meta/view.go +++ b/sdk/meta/view.go @@ -21,12 +21,11 @@ import ( "encoding/json" "fmt" "github.com/chubaofs/chubaofs/proto" - "github.com/chubaofs/chubaofs/util" + "github.com/chubaofs/chubaofs/sdk/master" "github.com/chubaofs/chubaofs/util/cryptoutil" "github.com/chubaofs/chubaofs/util/errors" "github.com/chubaofs/chubaofs/util/log" "github.com/jacobsa/daemonize" - "net/http" "os" "strings" "sync/atomic" @@ -38,8 +37,15 @@ const ( ) type VolumeView struct { - VolName string + Name string + Owner string MetaPartitions []*MetaPartition + OSSSecure *OSSSecure +} + +type OSSSecure struct { + AccessKey string + SecretKey string } type VolStatInfo struct { @@ -48,89 +54,100 @@ type VolStatInfo struct { UsedSize uint64 } -// VolName view managements -// -func (mw *MetaWrapper) fetchVolumeView() (*VolumeView, error) { - params := make(map[string]string) - params["name"] = mw.volname - authKey, err := calculateAuthKey(mw.owner) - if err != nil { - return nil, err - } - params["authKey"] = authKey - var dataBody []byte - if mw.authenticate { - mw.accessToken.Type = proto.MsgMasterFetchVolViewReq - tokenMessage, ts, err := genMasterToken(mw.accessToken, mw.sessionKey) - if err != nil { - log.LogWarnf("fetchVolumeView generate token failed: err(%v)", err) - return nil, err +func (mw *MetaWrapper) fetchVolumeView() (view *VolumeView, err error) { + var vv *proto.VolView + if mw.ownerValidation { + var authKey string + if authKey, err = calculateAuthKey(mw.owner); err != nil { + return } - params[proto.ClientMessage] = tokenMessage - body, err := mw.master.Request(http.MethodPost, proto.ClientVol, params, nil) - if err != nil { - log.LogWarnf("fetchVolumeView request: err(%v)", err) - return nil, err - } - dataBody, err = mw.parseRespWithAuth(body, ts) - if err != nil { - log.LogWarnf("fetchVolumeView request: err(%v)", err) - return nil, err + if mw.authenticate { + var ( + tokenMessage string + ts int64 + ) + mw.accessToken.Type = proto.MsgMasterFetchVolViewReq + if tokenMessage, ts, err = genMasterToken(mw.accessToken, mw.sessionKey); err != nil { + log.LogWarnf("fetchVolumeView generate token failed: err(%v)", err) + return nil, err + } + var decoder master.Decoder = func(raw []byte) ([]byte, error) { + return mw.parseAndVerifyResp(raw, ts) + } + if vv, err = mw.mc.ClientAPI().GetVolumeWithAuthnode(mw.volname, authKey, tokenMessage, decoder); err != nil { + return + } + } else { + if vv, err = mw.mc.ClientAPI().GetVolume(mw.volname, authKey); err != nil { + return + } } } else { - body, err := mw.master.Request(http.MethodPost, proto.ClientVol, params, nil) - if err != nil { - log.LogWarnf("fetchVolumeView request: err(%v)", err) - return nil, err + if vv, err = mw.mc.ClientAPI().GetVolumeWithoutAuthKey(mw.volname); err != nil { + return } - dataBody = body } - - view := new(VolumeView) - if err = json.Unmarshal(dataBody, view); err != nil { - log.LogWarnf("fetchVolumeView unmarshal: err(%v) body(%v)", err, string(dataBody)) - return nil, err + var convert = func(volView *proto.VolView) *VolumeView { + result := &VolumeView{ + Name: volView.Name, + Owner: volView.Owner, + MetaPartitions: make([]*MetaPartition, len(volView.MetaPartitions)), + OSSSecure: &OSSSecure{}, + } + if volView.OSSSecure != nil { + result.OSSSecure.AccessKey = volView.OSSSecure.AccessKey + result.OSSSecure.SecretKey = volView.OSSSecure.SecretKey + } + for i, mp := range volView.MetaPartitions { + result.MetaPartitions[i] = &MetaPartition{ + PartitionID: mp.PartitionID, + Start: mp.Start, + End: mp.End, + Members: mp.Members, + LeaderAddr: mp.LeaderAddr, + Status: mp.Status, + } + } + return result } - return view, nil + view = convert(vv) + return } // fetch and update cluster info if successful -func (mw *MetaWrapper) updateClusterInfo() error { - body, err := mw.master.Request(http.MethodPost, proto.AdminGetIP, nil, nil) - if err != nil { - log.LogWarnf("updateClusterInfo request: err(%v)", err) - return err +func (mw *MetaWrapper) updateClusterInfo() (err error) { + var info *proto.ClusterInfo + if info, err = mw.mc.AdminAPI().GetClusterInfo(); err != nil { + log.LogWarnf("updateClusterInfo: get cluster info fail: err(%v)", err) + return } - - info := new(proto.ClusterInfo) - if err = json.Unmarshal(body, info); err != nil { - log.LogWarnf("updateClusterInfo unmarshal: err(%v)", err) - return err - } - log.LogInfof("ClusterInfo: %v", *info) + log.LogInfof("updateClusterInfo: get cluster info: cluster(%v) localIP(%v)", + info.Cluster, info.Ip) mw.cluster = info.Cluster mw.localIP = info.Ip - return nil + return } -func (mw *MetaWrapper) updateVolStatInfo() error { - params := make(map[string]string) - params["name"] = mw.volname - body, err := mw.master.Request(http.MethodPost, proto.ClientVolStat, params, nil) - if err != nil { - log.LogWarnf("updateVolStatInfo request: err(%v)", err) - return err +func (mw *MetaWrapper) updateVolStatInfo() (err error) { + + var convert = func(info *proto.VolStatInfo) *VolStatInfo { + return &VolStatInfo{ + Name: info.Name, + TotalSize: info.TotalSize, + UsedSize: info.UsedSize, + } } - info := new(VolStatInfo) - if err = json.Unmarshal(body, info); err != nil { - log.LogWarnf("updateVolStatInfo unmarshal: err(%v)", err) - return err + var volStatInfo *proto.VolStatInfo + if volStatInfo, err = mw.mc.ClientAPI().GetVolumeStat(mw.volname); err != nil { + log.LogWarnf("updateVolStatInfo: get volume status fail: volume(%v) err(%v)", mw.volname, err) + return } + var info = convert(volStatInfo) atomic.StoreUint64(&mw.totalSize, info.TotalSize) atomic.StoreUint64(&mw.usedSize, info.UsedSize) log.LogInfof("VolStatInfo: info(%v)", *info) - return nil + return } func (mw *MetaWrapper) updateMetaPartitions() error { @@ -138,7 +155,7 @@ func (mw *MetaWrapper) updateMetaPartitions() error { if err != nil { log.LogInfof("error: %v", err.Error()) switch err { - case util.ErrExpiredTicket: + case proto.ErrExpiredTicket: if e := mw.updateTicket(); e != nil { log.LogFlush() daemonize.SignalOutcome(err) @@ -146,7 +163,7 @@ func (mw *MetaWrapper) updateMetaPartitions() error { } log.LogInfof("updateTicket: ok!") return err - case util.ErrInvalidTicket: + case proto.ErrInvalidTicket: log.LogFlush() daemonize.SignalOutcome(err) os.Exit(1) @@ -163,9 +180,10 @@ func (mw *MetaWrapper) updateMetaPartitions() error { rwPartitions = append(rwPartitions, mp) } } + mw.ossSecure = view.OSSSecure if len(rwPartitions) == 0 { - log.LogInfof("updateMetaPartition: no rw partitions") + log.LogInfof("updateMetaPartition: no valid partitions") return nil } @@ -181,8 +199,15 @@ func (mw *MetaWrapper) refresh() { for { select { case <-t.C: - mw.updateMetaPartitions() - mw.updateVolStatInfo() + var err error + if err = mw.updateMetaPartitions(); err != nil { + log.LogErrorf("updateMetaPartition fail cause: %v", err) + } + if err = mw.updateVolStatInfo(); err != nil { + log.LogErrorf("updateVolStatInfo fail cause: %v", err) + } + case <-mw.closeCh: + return } } } @@ -230,36 +255,14 @@ func (mw *MetaWrapper) updateTicket() error { return nil } -func (mw *MetaWrapper) verifyResponse(checkMsg string, serviceID string, ts int64) (err error) { - var ( - sessionKey []byte - plaintext []byte - resp proto.APIAccessResp - ) - - if sessionKey, err = cryptoutil.Base64Decode(mw.sessionKey); err != nil { - return - } - - if plaintext, err = cryptoutil.DecodeMessage(checkMsg, sessionKey); err != nil { - return - } - - if err = json.Unmarshal(plaintext, &resp); err != nil { - return - } - return proto.VerifyAPIRespComm(&resp, mw.accessToken.Type, mw.owner, serviceID, ts) - -} - -func (mw *MetaWrapper) parseRespWithAuth(body []byte, ts int64) (dataBody []byte, err error) { - getVolResp := new(proto.GetVolResponse) - if err = json.Unmarshal(body, getVolResp); err != nil { - log.LogWarnf("fetchVolumeView unmarshal: err(%v) body(%v)", err, string(body)) +func (mw *MetaWrapper) parseAndVerifyResp(body []byte, ts int64) (dataBody []byte, err error) { + var resp proto.MasterAPIAccessResp + if resp, err = mw.parseRespWithAuth(body); err != nil { + log.LogWarnf("fetchVolumeView parse response failed: err(%v) body(%v)", err, string(body)) return nil, err } - if err = mw.verifyResponse(getVolResp.CheckMsg, proto.MasterServiceID, ts); err != nil { - log.LogWarnf("fetchVolumeView verify response: err(%v) body(%v)", err, getVolResp.CheckMsg) + if err = proto.VerifyAPIRespComm(&(resp.APIResp), mw.accessToken.Type, mw.owner, proto.MasterServiceID, ts); err != nil { + log.LogWarnf("fetchVolumeView verify response: err(%v)", err) return nil, err } var viewBody = &struct { @@ -267,7 +270,7 @@ func (mw *MetaWrapper) parseRespWithAuth(body []byte, ts int64) (dataBody []byte Msg string `json:"msg"` Data json.RawMessage }{} - if err = json.Unmarshal(getVolResp.VolViewCache, viewBody); err != nil { + if err = json.Unmarshal(resp.Data, viewBody); err != nil { log.LogWarnf("VolViewCache unmarshal: err(%v) body(%v)", err, viewBody) return nil, err } @@ -276,3 +279,29 @@ func (mw *MetaWrapper) parseRespWithAuth(body []byte, ts int64) (dataBody []byte } return viewBody.Data, err } + +func (mw *MetaWrapper) parseRespWithAuth(body []byte) (resp proto.MasterAPIAccessResp, err error) { + var ( + message string + sessionKey []byte + plaintext []byte + ) + + if err = json.Unmarshal(body, &message); err != nil { + return + } + + if sessionKey, err = cryptoutil.Base64Decode(mw.sessionKey); err != nil { + return + } + + if plaintext, err = cryptoutil.DecodeMessage(message, sessionKey); err != nil { + return + } + + if err = json.Unmarshal(plaintext, &resp); err != nil { + return + } + + return +} diff --git a/storage/extent.go b/storage/extent.go index e072da128..40f5f364a 100644 --- a/storage/extent.go +++ b/storage/extent.go @@ -353,8 +353,7 @@ func (e *Extent) DeleteTiny(offset, size int64) (hasDelete bool, err error) { hasDelete = true return true, nil } - err = syscall.Fallocate(int(e.file.Fd()), FallocFLPunchHole|FallocFLKeepSize, offset, size) - + err = fallocate(int(e.file.Fd()), FallocFLPunchHole|FallocFLKeepSize, offset, size) return } @@ -388,7 +387,7 @@ func (e *Extent) TinyExtentRecover(data []byte, offset, size int64, crc uint32, if err = syscall.Ftruncate(int(e.file.Fd()), offset+size); err != nil { return err } - err = syscall.Fallocate(int(e.file.Fd()), FallocFLPunchHole|FallocFLKeepSize, offset, size) + err = fallocate(int(e.file.Fd()), FallocFLPunchHole|FallocFLKeepSize, offset, size) } else { _, err = e.file.WriteAt(data[:size], int64(offset)) } diff --git a/storage/fallocate_darwin.go b/storage/fallocate_darwin.go new file mode 100644 index 000000000..b4abbfaaa --- /dev/null +++ b/storage/fallocate_darwin.go @@ -0,0 +1,6 @@ +package storage + +func fallocate(fd int, mode uint32, off int64, len int64) (err error) { + panic("Fallocate is required") + return nil +} diff --git a/storage/fallocate_linux.go b/storage/fallocate_linux.go new file mode 100644 index 000000000..10ac46bc9 --- /dev/null +++ b/storage/fallocate_linux.go @@ -0,0 +1,7 @@ +package storage + +import "syscall" + +func fallocate(fd int, mode uint32, off int64, len int64) (err error) { + return syscall.Fallocate(fd, mode, off, len) +} diff --git a/storage/fallocate_windows.go b/storage/fallocate_windows.go new file mode 100644 index 000000000..b4abbfaaa --- /dev/null +++ b/storage/fallocate_windows.go @@ -0,0 +1,6 @@ +package storage + +func fallocate(fd int, mode uint32, off int64, len int64) (err error) { + panic("Fallocate is required") + return nil +} diff --git a/storage/persistence_crc.go b/storage/persistence_crc.go index d23589d18..131c23cf8 100644 --- a/storage/persistence_crc.go +++ b/storage/persistence_crc.go @@ -19,7 +19,6 @@ import ( "github.com/chubaofs/chubaofs/util" "github.com/chubaofs/chubaofs/util/log" "sync/atomic" - "syscall" ) type BlockCrc struct { @@ -52,7 +51,7 @@ func (s *ExtentStore) PersistenceBlockCrc(e *Extent, blockNo int, blockCrc uint3 } func (s *ExtentStore) DeleteBlockCrc(extentID uint64) (err error) { - err = syscall.Fallocate(int(s.verifyExtentFp.Fd()), FallocFLPunchHole|FallocFLKeepSize, + err = fallocate(int(s.verifyExtentFp.Fd()), FallocFLPunchHole|FallocFLKeepSize, int64(util.BlockHeaderSize*extentID), util.BlockHeaderSize) return @@ -80,7 +79,7 @@ func (s *ExtentStore) PreAllocSpaceOnVerfiyFile(currExtentID uint64) { prevAllocSpaceExtentID := int64(atomic.LoadUint64(&s.hasAllocSpaceExtentIDOnVerfiyFile)) endAllocSpaceExtentID := int64(prevAllocSpaceExtentID + 1000) size := int64(1000 * util.BlockHeaderSize) - err := syscall.Fallocate(int(s.verifyExtentFp.Fd()), 1, prevAllocSpaceExtentID*util.BlockHeaderSize, size) + err := fallocate(int(s.verifyExtentFp.Fd()), 1, prevAllocSpaceExtentID*util.BlockHeaderSize, size) if err != nil { return } diff --git a/util/caps/caps.go b/util/caps/caps.go index 29cf35347..25bc96f44 100644 --- a/util/caps/caps.go +++ b/util/caps/caps.go @@ -9,42 +9,37 @@ import ( // Caps defines the capability type type Caps struct { - API []string - VOL []string + API []string + OwnerVOL []string + NoneOwnerVOL []string } // ContainCaps whether contain a capability with kind func (c *Caps) ContainCaps(cat string, cap string) (r bool) { - r = false if cat == "API" { - for _, s := range c.API { - a := strings.Split(s, ":") - b := strings.Split(cap, ":") - i := 0 - for ; i < 3; i++ { - if a[i] != "*" && a[i] != b[i] { - break - } - } - if i == 3 { - r = true + return traversalCaps(c.API, cap) + } else if cat == "OwnerVOL" { + return traversalCaps(c.OwnerVOL, cap) + } else if cat == "NoneOwnerVOL" { + return traversalCaps(c.NoneOwnerVOL, cap) + } + return false +} + +func traversalCaps(caps []string, cap string) (r bool) { + r = false + for _, s := range caps { + a := strings.Split(s, ":") + b := strings.Split(cap, ":") + i := 0 + for ; i < 3; i++ { + if a[i] != "*" && a[i] != b[i] { break } } - } else if cat == "VOL" { - for _, s := range c.VOL { - a := strings.Split(s, ":") - b := strings.Split(cap, ":") - i := 0 - for ; i < 3; i++ { - if a[i] != "*" && a[i] != b[i] { - break - } - } - if i == 3 { - r = true - break - } + if i == 3 { + r = true + break } } return @@ -74,21 +69,29 @@ func (c *Caps) Dump() (d string) { // Union union caps func (c *Caps) Union(caps *Caps) { c.API = append(c.API, caps.API...) - c.VOL = append(c.VOL, caps.VOL...) + c.OwnerVOL = append(c.OwnerVOL, caps.OwnerVOL...) + c.NoneOwnerVOL = append(c.NoneOwnerVOL, caps.NoneOwnerVOL...) c.cleanDup() } func (c *Caps) check() (err error) { apiRe := regexp.MustCompile("^[A-Za-z0-9*]{1,20}:[A-Za-z0-9*]{1,20}:[A-Za-z0-9*]{1,20}$") volRe := regexp.MustCompile("^[A-Za-z0-9*]{1,20}:[a-zA-Z0-9_-]{3,256}:[A-Za-z0-9*]{1,20}$") - for _, cap := range c.API { - if !apiRe.MatchString(cap) { - err = fmt.Errorf("invalid cap [%s]", cap) - return - } + if err = checkRegexp(apiRe, c.API); err != nil { + return } - for _, cap := range c.VOL { - if !volRe.MatchString(cap) { + if err = checkRegexp(volRe, c.OwnerVOL); err != nil { + return + } + if err = checkRegexp(volRe, c.NoneOwnerVOL); err != nil { + return + } + return +} + +func checkRegexp(re *regexp.Regexp, caps []string) (err error) { + for _, cap := range caps { + if !re.MatchString(cap) { err = fmt.Errorf("invalid cap [%s]", cap) return } @@ -98,60 +101,48 @@ func (c *Caps) check() (err error) { // Delete delete caps func (c *Caps) Delete(caps *Caps) { - mapi := make(map[string]bool) - mvol := make(map[string]bool) - for _, item := range c.API { - mapi[item] = true + c.API = deleteCaps(c.API, caps.API) + c.OwnerVOL = deleteCaps(c.OwnerVOL, caps.OwnerVOL) + c.NoneOwnerVOL = deleteCaps(c.NoneOwnerVOL, caps.NoneOwnerVOL) + return +} + +func deleteCaps(caps []string, deleteCaps []string) []string { + m := make(map[string]bool) + for _, item := range caps { + m[item] = true } - for _, item := range c.VOL { - mvol[item] = true + caps = []string{} + for _, item := range deleteCaps { + delete(m, item) } - c.API = []string{} - c.VOL = []string{} - for _, item := range caps.API { - delete(mapi, item) - } - for _, item := range caps.VOL { - delete(mvol, item) - } - for k := range mapi { - c.API = append(c.API, k) - } - for k := range mvol { - c.VOL = append(c.VOL, k) + for k := range m { + caps = append(caps, k) } + return caps } func (c *Caps) cleanDup() { - API := make([]string, 0) - VOL := make([]string, 0) - mapi := make(map[string]map[string]bool) - mvol := make(map[string]map[string]bool) - for _, cap := range c.API { - a := strings.Split(cap, ":") - key1 := a[0] - key2 := a[1] + ":" + a[2] - if _, ok := mapi[key1]; !ok { - mapi[key1] = make(map[string]bool, 0) - } - if _, ok := mapi[key1][key2]; !ok { - API = append(API, cap) - mapi[key1][key2] = true - } - } - for _, cap := range c.VOL { - a := strings.Split(cap, ":") - key1 := a[0] - key2 := a[1] + ":" + a[2] - if _, ok := mvol[key1]; !ok { - mvol[key1] = make(map[string]bool, 0) - } - if _, ok := mvol[key1][key2]; !ok { - VOL = append(VOL, cap) - mvol[key1][key2] = true - } - } - c.API = API - c.VOL = VOL + c.API = cleanCaps(c.API) + c.OwnerVOL = cleanCaps(c.OwnerVOL) + c.NoneOwnerVOL = cleanCaps(c.NoneOwnerVOL) return } + +func cleanCaps(caps []string) []string { + newCaps := make([]string, 0) + m := make(map[string]map[string]bool) + for _, cap := range caps { + a := strings.Split(cap, ":") + key1 := a[0] + key2 := a[1] + ":" + a[2] + if _, ok := m[key1]; !ok { + m[key1] = make(map[string]bool, 0) + } + if _, ok := m[key1][key2]; !ok { + newCaps = append(newCaps, cap) + m[key1][key2] = true + } + } + return newCaps +} diff --git a/util/keystore/keystore.go b/util/keystore/keystore.go index e88c136d2..db151e8eb 100644 --- a/util/keystore/keystore.go +++ b/util/keystore/keystore.go @@ -17,11 +17,18 @@ var roleSet = map[string]bool{ // KeyInfo defines the key info structure in key store type KeyInfo struct { - ID string `json:"id"` - Key []byte `json:"key"` - Ts int64 `json:"create_ts"` - Role string `json:"role"` - Caps []byte `json:"caps"` + ID string `json:"id"` + AuthKey []byte `json:"auth_key"` + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + Ts int64 `json:"create_ts"` + Role string `json:"role"` + Caps []byte `json:"caps"` +} + +type AccessKeyInfo struct { + AccessKey string `json:"access_key"` + ID string `json:"id"` } // DumpJSONFile dump KeyInfo to file in json format @@ -49,14 +56,18 @@ func (u *KeyInfo) DumpJSONFile(filename string) (err error) { // DumpJSONStr dump KeyInfo to string in json format func (u *KeyInfo) DumpJSONStr() (r string, err error) { dumpInfo := struct { - ID string `json:"id"` - Key []byte `json:"key"` - Ts int64 `json:"create_ts"` - Role string `json:"role"` - Caps string `json:"caps"` + ID string `json:"id"` + AuthKey []byte `json:"auth_key"` + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + Ts int64 `json:"create_ts"` + Role string `json:"role"` + Caps string `json:"caps"` }{ u.ID, - u.Key, + u.AuthKey, + u.AccessKey, + u.SecretKey, u.Ts, u.Role, string(u.Caps), @@ -110,3 +121,12 @@ func (u *KeyInfo) IsValidKeyInfo() (err error) { } return } + +func (u *KeyInfo) IsValidAK() (err error) { + re := regexp.MustCompile("^[A-Za-z0-9]{16}$") + if !re.MatchString(u.AccessKey) { + err = fmt.Errorf("invalid AccessKey [%s]", u.AccessKey) + return + } + return +} diff --git a/util/master_helper.go b/util/master_helper.go index ef2c634d2..e09a5d292 100644 --- a/util/master_helper.go +++ b/util/master_helper.go @@ -33,8 +33,6 @@ const ( var ( ErrNoValidMaster = errors.New("no valid master") - ErrInvalidTicket = errors.New("invalid ticket") - ErrExpiredTicket = errors.New("expired ticket") ) // MasterHelper defines the helper struct to manage the master. @@ -42,7 +40,7 @@ type MasterHelper interface { AddNode(address string) Nodes() []string Leader() string - Request(method, path string, param map[string]string, body []byte) (data []byte, err error) + Request(method, path string, param, header map[string]string, body []byte) (data []byte, err error) } type masterHelper struct { @@ -74,12 +72,12 @@ func (helper *masterHelper) setLeader(addr string) { } // Request sends out the request through the helper. -func (helper *masterHelper) Request(method, path string, param map[string]string, reqData []byte) (respData []byte, err error) { - respData, err = helper.request(method, path, param, reqData) +func (helper *masterHelper) Request(method, path string, param, header map[string]string, reqData []byte) (respData []byte, err error) { + respData, err = helper.request(method, path, param, header, reqData) return } -func (helper *masterHelper) request(method, path string, param map[string]string, reqData []byte) (repsData []byte, err error) { +func (helper *masterHelper) request(method, path string, param, header map[string]string, reqData []byte) (repsData []byte, err error) { leaderAddr, nodes := helper.prepareRequest() host := leaderAddr for i := -1; i < len(nodes); i++ { @@ -92,7 +90,7 @@ func (helper *masterHelper) request(method, path string, param map[string]string } var resp *http.Response resp, err = helper.httpRequest(method, fmt.Sprintf("http://%s%s", host, - path), param, reqData) + path), param, header, reqData) if err != nil { log.LogErrorf("[masterHelper] %s", err) continue @@ -114,16 +112,16 @@ func (helper *masterHelper) request(method, path string, param map[string]string err = ErrNoValidMaster return } - repsData, err = helper.request(method, path, param, reqData) + repsData, err = helper.request(method, path, param, header, reqData) return case http.StatusOK: if leaderAddr != host { helper.setLeader(host) } var body = &struct { - Code int32 `json:"code"` - Msg string `json:"msg"` - Data json.RawMessage + Code int32 `json:"code"` + Msg string `json:"msg"` + Data json.RawMessage `json:"data"` }{} if err := json.Unmarshal(repsData, body); err != nil { return nil, fmt.Errorf("unmarshal response body err:%v", err) @@ -131,13 +129,7 @@ func (helper *masterHelper) request(method, path string, param map[string]string } // o represent proto.ErrCodeSuccess if body.Code != 0 { - if body.Code == 37 { - return nil, ErrInvalidTicket - } else if body.Code == 38 { - return nil, ErrExpiredTicket - } else { - return nil, fmt.Errorf("request error, code[%d], msg[%s]", body.Code, body.Msg) - } + return nil, fmt.Errorf("request error, code[%d], msg[%s]", body.Code, body.Msg) } return []byte(body.Data), nil default: @@ -167,7 +159,7 @@ func (helper *masterHelper) prepareRequest() (addr string, nodes []string) { return } -func (helper *masterHelper) httpRequest(method, url string, param map[string]string, reqData []byte) (resp *http.Response, err error) { +func (helper *masterHelper) httpRequest(method, url string, param, header map[string]string, reqData []byte) (resp *http.Response, err error) { client := &http.Client{} reader := bytes.NewReader(reqData) client.Timeout = requestTimeout @@ -179,6 +171,9 @@ func (helper *masterHelper) httpRequest(method, url string, param map[string]str } req.Header.Set("Content-Type", "application/json") req.Header.Set("Connection", "close") + for k, v := range header { + req.Header.Set(k, v) + } resp, err = client.Do(req) return } diff --git a/util/string.go b/util/string.go new file mode 100644 index 000000000..8ae4aaa76 --- /dev/null +++ b/util/string.go @@ -0,0 +1,67 @@ +// Copyright 2018 The Chubao Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package util + +import ( + "math/rand" + "strings" + "time" +) + +func SubString(sourceString string, begin, end int) string { + bytes := []byte(sourceString) + stringLength := len(bytes) + + if begin < 0 { + begin = 0 + } + if end > stringLength { + end = stringLength + } + return string(bytes[begin:end]) +} + +type RandomSeed byte + +func (s RandomSeed) Runes() []rune { + sourceBuilder := strings.Builder{} + if s&Numeric > 0 { + sourceBuilder.WriteString("0123456789") + } + if s&LowerLetter > 0 { + sourceBuilder.WriteString("abcdefghijklmnopqrstuvwxyz") + } + if s&UpperLetter > 0 { + sourceBuilder.WriteString("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + } + return []rune(sourceBuilder.String()) +} + +const ( + Numeric RandomSeed = 1 << iota + LowerLetter + UpperLetter +) + +func RandomString(length int, seed RandomSeed) string { + runs := seed.Runes() + result := "" + for i := 0; i < length; i++ { + rand.Seed(time.Now().UnixNano()) + randNumber := rand.Intn(len(runs)) + result += string(runs[randNumber]) + } + return result +} diff --git a/vendor/bazil.org/fuse/.gitattributes b/vendor/bazil.org/fuse/.gitattributes new file mode 100644 index 000000000..b65f2a9ff --- /dev/null +++ b/vendor/bazil.org/fuse/.gitattributes @@ -0,0 +1,2 @@ +*.go filter=gofmt +*.cgo filter=gofmt diff --git a/vendor/bazil.org/fuse/.gitignore b/vendor/bazil.org/fuse/.gitignore new file mode 100644 index 000000000..53589948c --- /dev/null +++ b/vendor/bazil.org/fuse/.gitignore @@ -0,0 +1,11 @@ +*~ +.#* +## the next line needs to start with a backslash to avoid looking like +## a comment +\#*# +.*.swp + +*.test + +/clockfs +/hellofs diff --git a/vendor/github.com/davecgh/go-spew/LICENSE b/vendor/github.com/davecgh/go-spew/LICENSE deleted file mode 100644 index bc52e96f2..000000000 --- a/vendor/github.com/davecgh/go-spew/LICENSE +++ /dev/null @@ -1,15 +0,0 @@ -ISC License - -Copyright (c) 2012-2016 Dave Collins - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/vendor/github.com/davecgh/go-spew/spew/bypass.go b/vendor/github.com/davecgh/go-spew/spew/bypass.go deleted file mode 100644 index 792994785..000000000 --- a/vendor/github.com/davecgh/go-spew/spew/bypass.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) 2015-2016 Dave Collins -// -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -// NOTE: Due to the following build constraints, this file will only be compiled -// when the code is not running on Google App Engine, compiled by GopherJS, and -// "-tags safe" is not added to the go build command line. The "disableunsafe" -// tag is deprecated and thus should not be used. -// Go versions prior to 1.4 are disabled because they use a different layout -// for interfaces which make the implementation of unsafeReflectValue more complex. -// +build !js,!appengine,!safe,!disableunsafe,go1.4 - -package spew - -import ( - "reflect" - "unsafe" -) - -const ( - // UnsafeDisabled is a build-time constant which specifies whether or - // not access to the unsafe package is available. - UnsafeDisabled = false - - // ptrSize is the size of a pointer on the current arch. - ptrSize = unsafe.Sizeof((*byte)(nil)) -) - -type flag uintptr - -var ( - // flagRO indicates whether the value field of a reflect.Value - // is read-only. - flagRO flag - - // flagAddr indicates whether the address of the reflect.Value's - // value may be taken. - flagAddr flag -) - -// flagKindMask holds the bits that make up the kind -// part of the flags field. In all the supported versions, -// it is in the lower 5 bits. -const flagKindMask = flag(0x1f) - -// Different versions of Go have used different -// bit layouts for the flags type. This table -// records the known combinations. -var okFlags = []struct { - ro, addr flag -}{{ - // From Go 1.4 to 1.5 - ro: 1 << 5, - addr: 1 << 7, -}, { - // Up to Go tip. - ro: 1<<5 | 1<<6, - addr: 1 << 8, -}} - -var flagValOffset = func() uintptr { - field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") - if !ok { - panic("reflect.Value has no flag field") - } - return field.Offset -}() - -// flagField returns a pointer to the flag field of a reflect.Value. -func flagField(v *reflect.Value) *flag { - return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset)) -} - -// unsafeReflectValue converts the passed reflect.Value into a one that bypasses -// the typical safety restrictions preventing access to unaddressable and -// unexported data. It works by digging the raw pointer to the underlying -// value out of the protected value and generating a new unprotected (unsafe) -// reflect.Value to it. -// -// This allows us to check for implementations of the Stringer and error -// interfaces to be used for pretty printing ordinarily unaddressable and -// inaccessible values such as unexported struct fields. -func unsafeReflectValue(v reflect.Value) reflect.Value { - if !v.IsValid() || (v.CanInterface() && v.CanAddr()) { - return v - } - flagFieldPtr := flagField(&v) - *flagFieldPtr &^= flagRO - *flagFieldPtr |= flagAddr - return v -} - -// Sanity checks against future reflect package changes -// to the type or semantics of the Value.flag field. -func init() { - field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") - if !ok { - panic("reflect.Value has no flag field") - } - if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() { - panic("reflect.Value flag field has changed kind") - } - type t0 int - var t struct { - A t0 - // t0 will have flagEmbedRO set. - t0 - // a will have flagStickyRO set - a t0 - } - vA := reflect.ValueOf(t).FieldByName("A") - va := reflect.ValueOf(t).FieldByName("a") - vt0 := reflect.ValueOf(t).FieldByName("t0") - - // Infer flagRO from the difference between the flags - // for the (otherwise identical) fields in t. - flagPublic := *flagField(&vA) - flagWithRO := *flagField(&va) | *flagField(&vt0) - flagRO = flagPublic ^ flagWithRO - - // Infer flagAddr from the difference between a value - // taken from a pointer and not. - vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A") - flagNoPtr := *flagField(&vA) - flagPtr := *flagField(&vPtrA) - flagAddr = flagNoPtr ^ flagPtr - - // Check that the inferred flags tally with one of the known versions. - for _, f := range okFlags { - if flagRO == f.ro && flagAddr == f.addr { - return - } - } - panic("reflect.Value read-only flag has changed semantics") -} diff --git a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go b/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go deleted file mode 100644 index 205c28d68..000000000 --- a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2015-2016 Dave Collins -// -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -// NOTE: Due to the following build constraints, this file will only be compiled -// when the code is running on Google App Engine, compiled by GopherJS, or -// "-tags safe" is added to the go build command line. The "disableunsafe" -// tag is deprecated and thus should not be used. -// +build js appengine safe disableunsafe !go1.4 - -package spew - -import "reflect" - -const ( - // UnsafeDisabled is a build-time constant which specifies whether or - // not access to the unsafe package is available. - UnsafeDisabled = true -) - -// unsafeReflectValue typically converts the passed reflect.Value into a one -// that bypasses the typical safety restrictions preventing access to -// unaddressable and unexported data. However, doing this relies on access to -// the unsafe package. This is a stub version which simply returns the passed -// reflect.Value when the unsafe package is not available. -func unsafeReflectValue(v reflect.Value) reflect.Value { - return v -} diff --git a/vendor/github.com/davecgh/go-spew/spew/common.go b/vendor/github.com/davecgh/go-spew/spew/common.go deleted file mode 100644 index 1be8ce945..000000000 --- a/vendor/github.com/davecgh/go-spew/spew/common.go +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "bytes" - "fmt" - "io" - "reflect" - "sort" - "strconv" -) - -// Some constants in the form of bytes to avoid string overhead. This mirrors -// the technique used in the fmt package. -var ( - panicBytes = []byte("(PANIC=") - plusBytes = []byte("+") - iBytes = []byte("i") - trueBytes = []byte("true") - falseBytes = []byte("false") - interfaceBytes = []byte("(interface {})") - commaNewlineBytes = []byte(",\n") - newlineBytes = []byte("\n") - openBraceBytes = []byte("{") - openBraceNewlineBytes = []byte("{\n") - closeBraceBytes = []byte("}") - asteriskBytes = []byte("*") - colonBytes = []byte(":") - colonSpaceBytes = []byte(": ") - openParenBytes = []byte("(") - closeParenBytes = []byte(")") - spaceBytes = []byte(" ") - pointerChainBytes = []byte("->") - nilAngleBytes = []byte("") - maxNewlineBytes = []byte("\n") - maxShortBytes = []byte("") - circularBytes = []byte("") - circularShortBytes = []byte("") - invalidAngleBytes = []byte("") - openBracketBytes = []byte("[") - closeBracketBytes = []byte("]") - percentBytes = []byte("%") - precisionBytes = []byte(".") - openAngleBytes = []byte("<") - closeAngleBytes = []byte(">") - openMapBytes = []byte("map[") - closeMapBytes = []byte("]") - lenEqualsBytes = []byte("len=") - capEqualsBytes = []byte("cap=") -) - -// hexDigits is used to map a decimal value to a hex digit. -var hexDigits = "0123456789abcdef" - -// catchPanic handles any panics that might occur during the handleMethods -// calls. -func catchPanic(w io.Writer, v reflect.Value) { - if err := recover(); err != nil { - w.Write(panicBytes) - fmt.Fprintf(w, "%v", err) - w.Write(closeParenBytes) - } -} - -// handleMethods attempts to call the Error and String methods on the underlying -// type the passed reflect.Value represents and outputes the result to Writer w. -// -// It handles panics in any called methods by catching and displaying the error -// as the formatted value. -func handleMethods(cs *ConfigState, w io.Writer, v reflect.Value) (handled bool) { - // We need an interface to check if the type implements the error or - // Stringer interface. However, the reflect package won't give us an - // interface on certain things like unexported struct fields in order - // to enforce visibility rules. We use unsafe, when it's available, - // to bypass these restrictions since this package does not mutate the - // values. - if !v.CanInterface() { - if UnsafeDisabled { - return false - } - - v = unsafeReflectValue(v) - } - - // Choose whether or not to do error and Stringer interface lookups against - // the base type or a pointer to the base type depending on settings. - // Technically calling one of these methods with a pointer receiver can - // mutate the value, however, types which choose to satisify an error or - // Stringer interface with a pointer receiver should not be mutating their - // state inside these interface methods. - if !cs.DisablePointerMethods && !UnsafeDisabled && !v.CanAddr() { - v = unsafeReflectValue(v) - } - if v.CanAddr() { - v = v.Addr() - } - - // Is it an error or Stringer? - switch iface := v.Interface().(type) { - case error: - defer catchPanic(w, v) - if cs.ContinueOnMethod { - w.Write(openParenBytes) - w.Write([]byte(iface.Error())) - w.Write(closeParenBytes) - w.Write(spaceBytes) - return false - } - - w.Write([]byte(iface.Error())) - return true - - case fmt.Stringer: - defer catchPanic(w, v) - if cs.ContinueOnMethod { - w.Write(openParenBytes) - w.Write([]byte(iface.String())) - w.Write(closeParenBytes) - w.Write(spaceBytes) - return false - } - w.Write([]byte(iface.String())) - return true - } - return false -} - -// printBool outputs a boolean value as true or false to Writer w. -func printBool(w io.Writer, val bool) { - if val { - w.Write(trueBytes) - } else { - w.Write(falseBytes) - } -} - -// printInt outputs a signed integer value to Writer w. -func printInt(w io.Writer, val int64, base int) { - w.Write([]byte(strconv.FormatInt(val, base))) -} - -// printUint outputs an unsigned integer value to Writer w. -func printUint(w io.Writer, val uint64, base int) { - w.Write([]byte(strconv.FormatUint(val, base))) -} - -// printFloat outputs a floating point value using the specified precision, -// which is expected to be 32 or 64bit, to Writer w. -func printFloat(w io.Writer, val float64, precision int) { - w.Write([]byte(strconv.FormatFloat(val, 'g', -1, precision))) -} - -// printComplex outputs a complex value using the specified float precision -// for the real and imaginary parts to Writer w. -func printComplex(w io.Writer, c complex128, floatPrecision int) { - r := real(c) - w.Write(openParenBytes) - w.Write([]byte(strconv.FormatFloat(r, 'g', -1, floatPrecision))) - i := imag(c) - if i >= 0 { - w.Write(plusBytes) - } - w.Write([]byte(strconv.FormatFloat(i, 'g', -1, floatPrecision))) - w.Write(iBytes) - w.Write(closeParenBytes) -} - -// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x' -// prefix to Writer w. -func printHexPtr(w io.Writer, p uintptr) { - // Null pointer. - num := uint64(p) - if num == 0 { - w.Write(nilAngleBytes) - return - } - - // Max uint64 is 16 bytes in hex + 2 bytes for '0x' prefix - buf := make([]byte, 18) - - // It's simpler to construct the hex string right to left. - base := uint64(16) - i := len(buf) - 1 - for num >= base { - buf[i] = hexDigits[num%base] - num /= base - i-- - } - buf[i] = hexDigits[num] - - // Add '0x' prefix. - i-- - buf[i] = 'x' - i-- - buf[i] = '0' - - // Strip unused leading bytes. - buf = buf[i:] - w.Write(buf) -} - -// valuesSorter implements sort.Interface to allow a slice of reflect.Value -// elements to be sorted. -type valuesSorter struct { - values []reflect.Value - strings []string // either nil or same len and values - cs *ConfigState -} - -// newValuesSorter initializes a valuesSorter instance, which holds a set of -// surrogate keys on which the data should be sorted. It uses flags in -// ConfigState to decide if and how to populate those surrogate keys. -func newValuesSorter(values []reflect.Value, cs *ConfigState) sort.Interface { - vs := &valuesSorter{values: values, cs: cs} - if canSortSimply(vs.values[0].Kind()) { - return vs - } - if !cs.DisableMethods { - vs.strings = make([]string, len(values)) - for i := range vs.values { - b := bytes.Buffer{} - if !handleMethods(cs, &b, vs.values[i]) { - vs.strings = nil - break - } - vs.strings[i] = b.String() - } - } - if vs.strings == nil && cs.SpewKeys { - vs.strings = make([]string, len(values)) - for i := range vs.values { - vs.strings[i] = Sprintf("%#v", vs.values[i].Interface()) - } - } - return vs -} - -// canSortSimply tests whether a reflect.Kind is a primitive that can be sorted -// directly, or whether it should be considered for sorting by surrogate keys -// (if the ConfigState allows it). -func canSortSimply(kind reflect.Kind) bool { - // This switch parallels valueSortLess, except for the default case. - switch kind { - case reflect.Bool: - return true - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: - return true - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - return true - case reflect.Float32, reflect.Float64: - return true - case reflect.String: - return true - case reflect.Uintptr: - return true - case reflect.Array: - return true - } - return false -} - -// Len returns the number of values in the slice. It is part of the -// sort.Interface implementation. -func (s *valuesSorter) Len() int { - return len(s.values) -} - -// Swap swaps the values at the passed indices. It is part of the -// sort.Interface implementation. -func (s *valuesSorter) Swap(i, j int) { - s.values[i], s.values[j] = s.values[j], s.values[i] - if s.strings != nil { - s.strings[i], s.strings[j] = s.strings[j], s.strings[i] - } -} - -// valueSortLess returns whether the first value should sort before the second -// value. It is used by valueSorter.Less as part of the sort.Interface -// implementation. -func valueSortLess(a, b reflect.Value) bool { - switch a.Kind() { - case reflect.Bool: - return !a.Bool() && b.Bool() - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: - return a.Int() < b.Int() - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - return a.Uint() < b.Uint() - case reflect.Float32, reflect.Float64: - return a.Float() < b.Float() - case reflect.String: - return a.String() < b.String() - case reflect.Uintptr: - return a.Uint() < b.Uint() - case reflect.Array: - // Compare the contents of both arrays. - l := a.Len() - for i := 0; i < l; i++ { - av := a.Index(i) - bv := b.Index(i) - if av.Interface() == bv.Interface() { - continue - } - return valueSortLess(av, bv) - } - } - return a.String() < b.String() -} - -// Less returns whether the value at index i should sort before the -// value at index j. It is part of the sort.Interface implementation. -func (s *valuesSorter) Less(i, j int) bool { - if s.strings == nil { - return valueSortLess(s.values[i], s.values[j]) - } - return s.strings[i] < s.strings[j] -} - -// sortValues is a sort function that handles both native types and any type that -// can be converted to error or Stringer. Other inputs are sorted according to -// their Value.String() value to ensure display stability. -func sortValues(values []reflect.Value, cs *ConfigState) { - if len(values) == 0 { - return - } - sort.Sort(newValuesSorter(values, cs)) -} diff --git a/vendor/github.com/davecgh/go-spew/spew/config.go b/vendor/github.com/davecgh/go-spew/spew/config.go deleted file mode 100644 index 2e3d22f31..000000000 --- a/vendor/github.com/davecgh/go-spew/spew/config.go +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "bytes" - "fmt" - "io" - "os" -) - -// ConfigState houses the configuration options used by spew to format and -// display values. There is a global instance, Config, that is used to control -// all top-level Formatter and Dump functionality. Each ConfigState instance -// provides methods equivalent to the top-level functions. -// -// The zero value for ConfigState provides no indentation. You would typically -// want to set it to a space or a tab. -// -// Alternatively, you can use NewDefaultConfig to get a ConfigState instance -// with default settings. See the documentation of NewDefaultConfig for default -// values. -type ConfigState struct { - // Indent specifies the string to use for each indentation level. The - // global config instance that all top-level functions use set this to a - // single space by default. If you would like more indentation, you might - // set this to a tab with "\t" or perhaps two spaces with " ". - Indent string - - // MaxDepth controls the maximum number of levels to descend into nested - // data structures. The default, 0, means there is no limit. - // - // NOTE: Circular data structures are properly detected, so it is not - // necessary to set this value unless you specifically want to limit deeply - // nested data structures. - MaxDepth int - - // DisableMethods specifies whether or not error and Stringer interfaces are - // invoked for types that implement them. - DisableMethods bool - - // DisablePointerMethods specifies whether or not to check for and invoke - // error and Stringer interfaces on types which only accept a pointer - // receiver when the current type is not a pointer. - // - // NOTE: This might be an unsafe action since calling one of these methods - // with a pointer receiver could technically mutate the value, however, - // in practice, types which choose to satisify an error or Stringer - // interface with a pointer receiver should not be mutating their state - // inside these interface methods. As a result, this option relies on - // access to the unsafe package, so it will not have any effect when - // running in environments without access to the unsafe package such as - // Google App Engine or with the "safe" build tag specified. - DisablePointerMethods bool - - // DisablePointerAddresses specifies whether to disable the printing of - // pointer addresses. This is useful when diffing data structures in tests. - DisablePointerAddresses bool - - // DisableCapacities specifies whether to disable the printing of capacities - // for arrays, slices, maps and channels. This is useful when diffing - // data structures in tests. - DisableCapacities bool - - // ContinueOnMethod specifies whether or not recursion should continue once - // a custom error or Stringer interface is invoked. The default, false, - // means it will print the results of invoking the custom error or Stringer - // interface and return immediately instead of continuing to recurse into - // the internals of the data type. - // - // NOTE: This flag does not have any effect if method invocation is disabled - // via the DisableMethods or DisablePointerMethods options. - ContinueOnMethod bool - - // SortKeys specifies map keys should be sorted before being printed. Use - // this to have a more deterministic, diffable output. Note that only - // native types (bool, int, uint, floats, uintptr and string) and types - // that support the error or Stringer interfaces (if methods are - // enabled) are supported, with other types sorted according to the - // reflect.Value.String() output which guarantees display stability. - SortKeys bool - - // SpewKeys specifies that, as a last resort attempt, map keys should - // be spewed to strings and sorted by those strings. This is only - // considered if SortKeys is true. - SpewKeys bool -} - -// Config is the active configuration of the top-level functions. -// The configuration can be changed by modifying the contents of spew.Config. -var Config = ConfigState{Indent: " "} - -// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the formatted string as a value that satisfies error. See NewFormatter -// for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Errorf(format, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Errorf(format string, a ...interface{}) (err error) { - return fmt.Errorf(format, c.convertArgs(a)...) -} - -// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprint(w, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Fprint(w io.Writer, a ...interface{}) (n int, err error) { - return fmt.Fprint(w, c.convertArgs(a)...) -} - -// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprintf(w, format, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { - return fmt.Fprintf(w, format, c.convertArgs(a)...) -} - -// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it -// passed with a Formatter interface returned by c.NewFormatter. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprintln(w, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Fprintln(w io.Writer, a ...interface{}) (n int, err error) { - return fmt.Fprintln(w, c.convertArgs(a)...) -} - -// Print is a wrapper for fmt.Print that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Print(c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Print(a ...interface{}) (n int, err error) { - return fmt.Print(c.convertArgs(a)...) -} - -// Printf is a wrapper for fmt.Printf that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Printf(format, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Printf(format string, a ...interface{}) (n int, err error) { - return fmt.Printf(format, c.convertArgs(a)...) -} - -// Println is a wrapper for fmt.Println that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Println(c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Println(a ...interface{}) (n int, err error) { - return fmt.Println(c.convertArgs(a)...) -} - -// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprint(c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Sprint(a ...interface{}) string { - return fmt.Sprint(c.convertArgs(a)...) -} - -// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprintf(format, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Sprintf(format string, a ...interface{}) string { - return fmt.Sprintf(format, c.convertArgs(a)...) -} - -// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it -// were passed with a Formatter interface returned by c.NewFormatter. It -// returns the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprintln(c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Sprintln(a ...interface{}) string { - return fmt.Sprintln(c.convertArgs(a)...) -} - -/* -NewFormatter returns a custom formatter that satisfies the fmt.Formatter -interface. As a result, it integrates cleanly with standard fmt package -printing functions. The formatter is useful for inline printing of smaller data -types similar to the standard %v format specifier. - -The custom formatter only responds to the %v (most compact), %+v (adds pointer -addresses), %#v (adds types), and %#+v (adds types and pointer addresses) verb -combinations. Any other verbs such as %x and %q will be sent to the the -standard fmt package for formatting. In addition, the custom formatter ignores -the width and precision arguments (however they will still work on the format -specifiers not handled by the custom formatter). - -Typically this function shouldn't be called directly. It is much easier to make -use of the custom formatter by calling one of the convenience functions such as -c.Printf, c.Println, or c.Printf. -*/ -func (c *ConfigState) NewFormatter(v interface{}) fmt.Formatter { - return newFormatter(c, v) -} - -// Fdump formats and displays the passed arguments to io.Writer w. It formats -// exactly the same as Dump. -func (c *ConfigState) Fdump(w io.Writer, a ...interface{}) { - fdump(c, w, a...) -} - -/* -Dump displays the passed parameters to standard out with newlines, customizable -indentation, and additional debug information such as complete types and all -pointer addresses used to indirect to the final value. It provides the -following features over the built-in printing facilities provided by the fmt -package: - - * Pointers are dereferenced and followed - * Circular data structures are detected and handled properly - * Custom Stringer/error interfaces are optionally invoked, including - on unexported types - * Custom types which only implement the Stringer/error interfaces via - a pointer receiver are optionally invoked when passing non-pointer - variables - * Byte arrays and slices are dumped like the hexdump -C command which - includes offsets, byte values in hex, and ASCII output - -The configuration options are controlled by modifying the public members -of c. See ConfigState for options documentation. - -See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to -get the formatted result as a string. -*/ -func (c *ConfigState) Dump(a ...interface{}) { - fdump(c, os.Stdout, a...) -} - -// Sdump returns a string with the passed arguments formatted exactly the same -// as Dump. -func (c *ConfigState) Sdump(a ...interface{}) string { - var buf bytes.Buffer - fdump(c, &buf, a...) - return buf.String() -} - -// convertArgs accepts a slice of arguments and returns a slice of the same -// length with each argument converted to a spew Formatter interface using -// the ConfigState associated with s. -func (c *ConfigState) convertArgs(args []interface{}) (formatters []interface{}) { - formatters = make([]interface{}, len(args)) - for index, arg := range args { - formatters[index] = newFormatter(c, arg) - } - return formatters -} - -// NewDefaultConfig returns a ConfigState with the following default settings. -// -// Indent: " " -// MaxDepth: 0 -// DisableMethods: false -// DisablePointerMethods: false -// ContinueOnMethod: false -// SortKeys: false -func NewDefaultConfig() *ConfigState { - return &ConfigState{Indent: " "} -} diff --git a/vendor/github.com/davecgh/go-spew/spew/doc.go b/vendor/github.com/davecgh/go-spew/spew/doc.go deleted file mode 100644 index aacaac6f1..000000000 --- a/vendor/github.com/davecgh/go-spew/spew/doc.go +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* -Package spew implements a deep pretty printer for Go data structures to aid in -debugging. - -A quick overview of the additional features spew provides over the built-in -printing facilities for Go data types are as follows: - - * Pointers are dereferenced and followed - * Circular data structures are detected and handled properly - * Custom Stringer/error interfaces are optionally invoked, including - on unexported types - * Custom types which only implement the Stringer/error interfaces via - a pointer receiver are optionally invoked when passing non-pointer - variables - * Byte arrays and slices are dumped like the hexdump -C command which - includes offsets, byte values in hex, and ASCII output (only when using - Dump style) - -There are two different approaches spew allows for dumping Go data structures: - - * Dump style which prints with newlines, customizable indentation, - and additional debug information such as types and all pointer addresses - used to indirect to the final value - * A custom Formatter interface that integrates cleanly with the standard fmt - package and replaces %v, %+v, %#v, and %#+v to provide inline printing - similar to the default %v while providing the additional functionality - outlined above and passing unsupported format verbs such as %x and %q - along to fmt - -Quick Start - -This section demonstrates how to quickly get started with spew. See the -sections below for further details on formatting and configuration options. - -To dump a variable with full newlines, indentation, type, and pointer -information use Dump, Fdump, or Sdump: - spew.Dump(myVar1, myVar2, ...) - spew.Fdump(someWriter, myVar1, myVar2, ...) - str := spew.Sdump(myVar1, myVar2, ...) - -Alternatively, if you would prefer to use format strings with a compacted inline -printing style, use the convenience wrappers Printf, Fprintf, etc with -%v (most compact), %+v (adds pointer addresses), %#v (adds types), or -%#+v (adds types and pointer addresses): - spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2) - spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) - spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2) - spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) - -Configuration Options - -Configuration of spew is handled by fields in the ConfigState type. For -convenience, all of the top-level functions use a global state available -via the spew.Config global. - -It is also possible to create a ConfigState instance that provides methods -equivalent to the top-level functions. This allows concurrent configuration -options. See the ConfigState documentation for more details. - -The following configuration options are available: - * Indent - String to use for each indentation level for Dump functions. - It is a single space by default. A popular alternative is "\t". - - * MaxDepth - Maximum number of levels to descend into nested data structures. - There is no limit by default. - - * DisableMethods - Disables invocation of error and Stringer interface methods. - Method invocation is enabled by default. - - * DisablePointerMethods - Disables invocation of error and Stringer interface methods on types - which only accept pointer receivers from non-pointer variables. - Pointer method invocation is enabled by default. - - * DisablePointerAddresses - DisablePointerAddresses specifies whether to disable the printing of - pointer addresses. This is useful when diffing data structures in tests. - - * DisableCapacities - DisableCapacities specifies whether to disable the printing of - capacities for arrays, slices, maps and channels. This is useful when - diffing data structures in tests. - - * ContinueOnMethod - Enables recursion into types after invoking error and Stringer interface - methods. Recursion after method invocation is disabled by default. - - * SortKeys - Specifies map keys should be sorted before being printed. Use - this to have a more deterministic, diffable output. Note that - only native types (bool, int, uint, floats, uintptr and string) - and types which implement error or Stringer interfaces are - supported with other types sorted according to the - reflect.Value.String() output which guarantees display - stability. Natural map order is used by default. - - * SpewKeys - Specifies that, as a last resort attempt, map keys should be - spewed to strings and sorted by those strings. This is only - considered if SortKeys is true. - -Dump Usage - -Simply call spew.Dump with a list of variables you want to dump: - - spew.Dump(myVar1, myVar2, ...) - -You may also call spew.Fdump if you would prefer to output to an arbitrary -io.Writer. For example, to dump to standard error: - - spew.Fdump(os.Stderr, myVar1, myVar2, ...) - -A third option is to call spew.Sdump to get the formatted output as a string: - - str := spew.Sdump(myVar1, myVar2, ...) - -Sample Dump Output - -See the Dump example for details on the setup of the types and variables being -shown here. - - (main.Foo) { - unexportedField: (*main.Bar)(0xf84002e210)({ - flag: (main.Flag) flagTwo, - data: (uintptr) - }), - ExportedField: (map[interface {}]interface {}) (len=1) { - (string) (len=3) "one": (bool) true - } - } - -Byte (and uint8) arrays and slices are displayed uniquely like the hexdump -C -command as shown. - ([]uint8) (len=32 cap=32) { - 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... | - 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0| - 00000020 31 32 |12| - } - -Custom Formatter - -Spew provides a custom formatter that implements the fmt.Formatter interface -so that it integrates cleanly with standard fmt package printing functions. The -formatter is useful for inline printing of smaller data types similar to the -standard %v format specifier. - -The custom formatter only responds to the %v (most compact), %+v (adds pointer -addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb -combinations. Any other verbs such as %x and %q will be sent to the the -standard fmt package for formatting. In addition, the custom formatter ignores -the width and precision arguments (however they will still work on the format -specifiers not handled by the custom formatter). - -Custom Formatter Usage - -The simplest way to make use of the spew custom formatter is to call one of the -convenience functions such as spew.Printf, spew.Println, or spew.Printf. The -functions have syntax you are most likely already familiar with: - - spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2) - spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) - spew.Println(myVar, myVar2) - spew.Fprintf(os.Stderr, "myVar1: %v -- myVar2: %+v", myVar1, myVar2) - spew.Fprintf(os.Stderr, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) - -See the Index for the full list convenience functions. - -Sample Formatter Output - -Double pointer to a uint8: - %v: <**>5 - %+v: <**>(0xf8400420d0->0xf8400420c8)5 - %#v: (**uint8)5 - %#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5 - -Pointer to circular struct with a uint8 field and a pointer to itself: - %v: <*>{1 <*>} - %+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)} - %#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)} - %#+v: (*main.circular)(0xf84003e260){ui8:(uint8)1 c:(*main.circular)(0xf84003e260)} - -See the Printf example for details on the setup of variables being shown -here. - -Errors - -Since it is possible for custom Stringer/error interfaces to panic, spew -detects them and handles them internally by printing the panic information -inline with the output. Since spew is intended to provide deep pretty printing -capabilities on structures, it intentionally does not return any errors. -*/ -package spew diff --git a/vendor/github.com/davecgh/go-spew/spew/dump.go b/vendor/github.com/davecgh/go-spew/spew/dump.go deleted file mode 100644 index f78d89fc1..000000000 --- a/vendor/github.com/davecgh/go-spew/spew/dump.go +++ /dev/null @@ -1,509 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "bytes" - "encoding/hex" - "fmt" - "io" - "os" - "reflect" - "regexp" - "strconv" - "strings" -) - -var ( - // uint8Type is a reflect.Type representing a uint8. It is used to - // convert cgo types to uint8 slices for hexdumping. - uint8Type = reflect.TypeOf(uint8(0)) - - // cCharRE is a regular expression that matches a cgo char. - // It is used to detect character arrays to hexdump them. - cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`) - - // cUnsignedCharRE is a regular expression that matches a cgo unsigned - // char. It is used to detect unsigned character arrays to hexdump - // them. - cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`) - - // cUint8tCharRE is a regular expression that matches a cgo uint8_t. - // It is used to detect uint8_t arrays to hexdump them. - cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`) -) - -// dumpState contains information about the state of a dump operation. -type dumpState struct { - w io.Writer - depth int - pointers map[uintptr]int - ignoreNextType bool - ignoreNextIndent bool - cs *ConfigState -} - -// indent performs indentation according to the depth level and cs.Indent -// option. -func (d *dumpState) indent() { - if d.ignoreNextIndent { - d.ignoreNextIndent = false - return - } - d.w.Write(bytes.Repeat([]byte(d.cs.Indent), d.depth)) -} - -// unpackValue returns values inside of non-nil interfaces when possible. -// This is useful for data types like structs, arrays, slices, and maps which -// can contain varying types packed inside an interface. -func (d *dumpState) unpackValue(v reflect.Value) reflect.Value { - if v.Kind() == reflect.Interface && !v.IsNil() { - v = v.Elem() - } - return v -} - -// dumpPtr handles formatting of pointers by indirecting them as necessary. -func (d *dumpState) dumpPtr(v reflect.Value) { - // Remove pointers at or below the current depth from map used to detect - // circular refs. - for k, depth := range d.pointers { - if depth >= d.depth { - delete(d.pointers, k) - } - } - - // Keep list of all dereferenced pointers to show later. - pointerChain := make([]uintptr, 0) - - // Figure out how many levels of indirection there are by dereferencing - // pointers and unpacking interfaces down the chain while detecting circular - // references. - nilFound := false - cycleFound := false - indirects := 0 - ve := v - for ve.Kind() == reflect.Ptr { - if ve.IsNil() { - nilFound = true - break - } - indirects++ - addr := ve.Pointer() - pointerChain = append(pointerChain, addr) - if pd, ok := d.pointers[addr]; ok && pd < d.depth { - cycleFound = true - indirects-- - break - } - d.pointers[addr] = d.depth - - ve = ve.Elem() - if ve.Kind() == reflect.Interface { - if ve.IsNil() { - nilFound = true - break - } - ve = ve.Elem() - } - } - - // Display type information. - d.w.Write(openParenBytes) - d.w.Write(bytes.Repeat(asteriskBytes, indirects)) - d.w.Write([]byte(ve.Type().String())) - d.w.Write(closeParenBytes) - - // Display pointer information. - if !d.cs.DisablePointerAddresses && len(pointerChain) > 0 { - d.w.Write(openParenBytes) - for i, addr := range pointerChain { - if i > 0 { - d.w.Write(pointerChainBytes) - } - printHexPtr(d.w, addr) - } - d.w.Write(closeParenBytes) - } - - // Display dereferenced value. - d.w.Write(openParenBytes) - switch { - case nilFound: - d.w.Write(nilAngleBytes) - - case cycleFound: - d.w.Write(circularBytes) - - default: - d.ignoreNextType = true - d.dump(ve) - } - d.w.Write(closeParenBytes) -} - -// dumpSlice handles formatting of arrays and slices. Byte (uint8 under -// reflection) arrays and slices are dumped in hexdump -C fashion. -func (d *dumpState) dumpSlice(v reflect.Value) { - // Determine whether this type should be hex dumped or not. Also, - // for types which should be hexdumped, try to use the underlying data - // first, then fall back to trying to convert them to a uint8 slice. - var buf []uint8 - doConvert := false - doHexDump := false - numEntries := v.Len() - if numEntries > 0 { - vt := v.Index(0).Type() - vts := vt.String() - switch { - // C types that need to be converted. - case cCharRE.MatchString(vts): - fallthrough - case cUnsignedCharRE.MatchString(vts): - fallthrough - case cUint8tCharRE.MatchString(vts): - doConvert = true - - // Try to use existing uint8 slices and fall back to converting - // and copying if that fails. - case vt.Kind() == reflect.Uint8: - // We need an addressable interface to convert the type - // to a byte slice. However, the reflect package won't - // give us an interface on certain things like - // unexported struct fields in order to enforce - // visibility rules. We use unsafe, when available, to - // bypass these restrictions since this package does not - // mutate the values. - vs := v - if !vs.CanInterface() || !vs.CanAddr() { - vs = unsafeReflectValue(vs) - } - if !UnsafeDisabled { - vs = vs.Slice(0, numEntries) - - // Use the existing uint8 slice if it can be - // type asserted. - iface := vs.Interface() - if slice, ok := iface.([]uint8); ok { - buf = slice - doHexDump = true - break - } - } - - // The underlying data needs to be converted if it can't - // be type asserted to a uint8 slice. - doConvert = true - } - - // Copy and convert the underlying type if needed. - if doConvert && vt.ConvertibleTo(uint8Type) { - // Convert and copy each element into a uint8 byte - // slice. - buf = make([]uint8, numEntries) - for i := 0; i < numEntries; i++ { - vv := v.Index(i) - buf[i] = uint8(vv.Convert(uint8Type).Uint()) - } - doHexDump = true - } - } - - // Hexdump the entire slice as needed. - if doHexDump { - indent := strings.Repeat(d.cs.Indent, d.depth) - str := indent + hex.Dump(buf) - str = strings.Replace(str, "\n", "\n"+indent, -1) - str = strings.TrimRight(str, d.cs.Indent) - d.w.Write([]byte(str)) - return - } - - // Recursively call dump for each item. - for i := 0; i < numEntries; i++ { - d.dump(d.unpackValue(v.Index(i))) - if i < (numEntries - 1) { - d.w.Write(commaNewlineBytes) - } else { - d.w.Write(newlineBytes) - } - } -} - -// dump is the main workhorse for dumping a value. It uses the passed reflect -// value to figure out what kind of object we are dealing with and formats it -// appropriately. It is a recursive function, however circular data structures -// are detected and handled properly. -func (d *dumpState) dump(v reflect.Value) { - // Handle invalid reflect values immediately. - kind := v.Kind() - if kind == reflect.Invalid { - d.w.Write(invalidAngleBytes) - return - } - - // Handle pointers specially. - if kind == reflect.Ptr { - d.indent() - d.dumpPtr(v) - return - } - - // Print type information unless already handled elsewhere. - if !d.ignoreNextType { - d.indent() - d.w.Write(openParenBytes) - d.w.Write([]byte(v.Type().String())) - d.w.Write(closeParenBytes) - d.w.Write(spaceBytes) - } - d.ignoreNextType = false - - // Display length and capacity if the built-in len and cap functions - // work with the value's kind and the len/cap itself is non-zero. - valueLen, valueCap := 0, 0 - switch v.Kind() { - case reflect.Array, reflect.Slice, reflect.Chan: - valueLen, valueCap = v.Len(), v.Cap() - case reflect.Map, reflect.String: - valueLen = v.Len() - } - if valueLen != 0 || !d.cs.DisableCapacities && valueCap != 0 { - d.w.Write(openParenBytes) - if valueLen != 0 { - d.w.Write(lenEqualsBytes) - printInt(d.w, int64(valueLen), 10) - } - if !d.cs.DisableCapacities && valueCap != 0 { - if valueLen != 0 { - d.w.Write(spaceBytes) - } - d.w.Write(capEqualsBytes) - printInt(d.w, int64(valueCap), 10) - } - d.w.Write(closeParenBytes) - d.w.Write(spaceBytes) - } - - // Call Stringer/error interfaces if they exist and the handle methods flag - // is enabled - if !d.cs.DisableMethods { - if (kind != reflect.Invalid) && (kind != reflect.Interface) { - if handled := handleMethods(d.cs, d.w, v); handled { - return - } - } - } - - switch kind { - case reflect.Invalid: - // Do nothing. We should never get here since invalid has already - // been handled above. - - case reflect.Bool: - printBool(d.w, v.Bool()) - - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: - printInt(d.w, v.Int(), 10) - - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - printUint(d.w, v.Uint(), 10) - - case reflect.Float32: - printFloat(d.w, v.Float(), 32) - - case reflect.Float64: - printFloat(d.w, v.Float(), 64) - - case reflect.Complex64: - printComplex(d.w, v.Complex(), 32) - - case reflect.Complex128: - printComplex(d.w, v.Complex(), 64) - - case reflect.Slice: - if v.IsNil() { - d.w.Write(nilAngleBytes) - break - } - fallthrough - - case reflect.Array: - d.w.Write(openBraceNewlineBytes) - d.depth++ - if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { - d.indent() - d.w.Write(maxNewlineBytes) - } else { - d.dumpSlice(v) - } - d.depth-- - d.indent() - d.w.Write(closeBraceBytes) - - case reflect.String: - d.w.Write([]byte(strconv.Quote(v.String()))) - - case reflect.Interface: - // The only time we should get here is for nil interfaces due to - // unpackValue calls. - if v.IsNil() { - d.w.Write(nilAngleBytes) - } - - case reflect.Ptr: - // Do nothing. We should never get here since pointers have already - // been handled above. - - case reflect.Map: - // nil maps should be indicated as different than empty maps - if v.IsNil() { - d.w.Write(nilAngleBytes) - break - } - - d.w.Write(openBraceNewlineBytes) - d.depth++ - if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { - d.indent() - d.w.Write(maxNewlineBytes) - } else { - numEntries := v.Len() - keys := v.MapKeys() - if d.cs.SortKeys { - sortValues(keys, d.cs) - } - for i, key := range keys { - d.dump(d.unpackValue(key)) - d.w.Write(colonSpaceBytes) - d.ignoreNextIndent = true - d.dump(d.unpackValue(v.MapIndex(key))) - if i < (numEntries - 1) { - d.w.Write(commaNewlineBytes) - } else { - d.w.Write(newlineBytes) - } - } - } - d.depth-- - d.indent() - d.w.Write(closeBraceBytes) - - case reflect.Struct: - d.w.Write(openBraceNewlineBytes) - d.depth++ - if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { - d.indent() - d.w.Write(maxNewlineBytes) - } else { - vt := v.Type() - numFields := v.NumField() - for i := 0; i < numFields; i++ { - d.indent() - vtf := vt.Field(i) - d.w.Write([]byte(vtf.Name)) - d.w.Write(colonSpaceBytes) - d.ignoreNextIndent = true - d.dump(d.unpackValue(v.Field(i))) - if i < (numFields - 1) { - d.w.Write(commaNewlineBytes) - } else { - d.w.Write(newlineBytes) - } - } - } - d.depth-- - d.indent() - d.w.Write(closeBraceBytes) - - case reflect.Uintptr: - printHexPtr(d.w, uintptr(v.Uint())) - - case reflect.UnsafePointer, reflect.Chan, reflect.Func: - printHexPtr(d.w, v.Pointer()) - - // There were not any other types at the time this code was written, but - // fall back to letting the default fmt package handle it in case any new - // types are added. - default: - if v.CanInterface() { - fmt.Fprintf(d.w, "%v", v.Interface()) - } else { - fmt.Fprintf(d.w, "%v", v.String()) - } - } -} - -// fdump is a helper function to consolidate the logic from the various public -// methods which take varying writers and config states. -func fdump(cs *ConfigState, w io.Writer, a ...interface{}) { - for _, arg := range a { - if arg == nil { - w.Write(interfaceBytes) - w.Write(spaceBytes) - w.Write(nilAngleBytes) - w.Write(newlineBytes) - continue - } - - d := dumpState{w: w, cs: cs} - d.pointers = make(map[uintptr]int) - d.dump(reflect.ValueOf(arg)) - d.w.Write(newlineBytes) - } -} - -// Fdump formats and displays the passed arguments to io.Writer w. It formats -// exactly the same as Dump. -func Fdump(w io.Writer, a ...interface{}) { - fdump(&Config, w, a...) -} - -// Sdump returns a string with the passed arguments formatted exactly the same -// as Dump. -func Sdump(a ...interface{}) string { - var buf bytes.Buffer - fdump(&Config, &buf, a...) - return buf.String() -} - -/* -Dump displays the passed parameters to standard out with newlines, customizable -indentation, and additional debug information such as complete types and all -pointer addresses used to indirect to the final value. It provides the -following features over the built-in printing facilities provided by the fmt -package: - - * Pointers are dereferenced and followed - * Circular data structures are detected and handled properly - * Custom Stringer/error interfaces are optionally invoked, including - on unexported types - * Custom types which only implement the Stringer/error interfaces via - a pointer receiver are optionally invoked when passing non-pointer - variables - * Byte arrays and slices are dumped like the hexdump -C command which - includes offsets, byte values in hex, and ASCII output - -The configuration options are controlled by an exported package global, -spew.Config. See ConfigState for options documentation. - -See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to -get the formatted result as a string. -*/ -func Dump(a ...interface{}) { - fdump(&Config, os.Stdout, a...) -} diff --git a/vendor/github.com/davecgh/go-spew/spew/format.go b/vendor/github.com/davecgh/go-spew/spew/format.go deleted file mode 100644 index b04edb7d7..000000000 --- a/vendor/github.com/davecgh/go-spew/spew/format.go +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "bytes" - "fmt" - "reflect" - "strconv" - "strings" -) - -// supportedFlags is a list of all the character flags supported by fmt package. -const supportedFlags = "0-+# " - -// formatState implements the fmt.Formatter interface and contains information -// about the state of a formatting operation. The NewFormatter function can -// be used to get a new Formatter which can be used directly as arguments -// in standard fmt package printing calls. -type formatState struct { - value interface{} - fs fmt.State - depth int - pointers map[uintptr]int - ignoreNextType bool - cs *ConfigState -} - -// buildDefaultFormat recreates the original format string without precision -// and width information to pass in to fmt.Sprintf in the case of an -// unrecognized type. Unless new types are added to the language, this -// function won't ever be called. -func (f *formatState) buildDefaultFormat() (format string) { - buf := bytes.NewBuffer(percentBytes) - - for _, flag := range supportedFlags { - if f.fs.Flag(int(flag)) { - buf.WriteRune(flag) - } - } - - buf.WriteRune('v') - - format = buf.String() - return format -} - -// constructOrigFormat recreates the original format string including precision -// and width information to pass along to the standard fmt package. This allows -// automatic deferral of all format strings this package doesn't support. -func (f *formatState) constructOrigFormat(verb rune) (format string) { - buf := bytes.NewBuffer(percentBytes) - - for _, flag := range supportedFlags { - if f.fs.Flag(int(flag)) { - buf.WriteRune(flag) - } - } - - if width, ok := f.fs.Width(); ok { - buf.WriteString(strconv.Itoa(width)) - } - - if precision, ok := f.fs.Precision(); ok { - buf.Write(precisionBytes) - buf.WriteString(strconv.Itoa(precision)) - } - - buf.WriteRune(verb) - - format = buf.String() - return format -} - -// unpackValue returns values inside of non-nil interfaces when possible and -// ensures that types for values which have been unpacked from an interface -// are displayed when the show types flag is also set. -// This is useful for data types like structs, arrays, slices, and maps which -// can contain varying types packed inside an interface. -func (f *formatState) unpackValue(v reflect.Value) reflect.Value { - if v.Kind() == reflect.Interface { - f.ignoreNextType = false - if !v.IsNil() { - v = v.Elem() - } - } - return v -} - -// formatPtr handles formatting of pointers by indirecting them as necessary. -func (f *formatState) formatPtr(v reflect.Value) { - // Display nil if top level pointer is nil. - showTypes := f.fs.Flag('#') - if v.IsNil() && (!showTypes || f.ignoreNextType) { - f.fs.Write(nilAngleBytes) - return - } - - // Remove pointers at or below the current depth from map used to detect - // circular refs. - for k, depth := range f.pointers { - if depth >= f.depth { - delete(f.pointers, k) - } - } - - // Keep list of all dereferenced pointers to possibly show later. - pointerChain := make([]uintptr, 0) - - // Figure out how many levels of indirection there are by derferencing - // pointers and unpacking interfaces down the chain while detecting circular - // references. - nilFound := false - cycleFound := false - indirects := 0 - ve := v - for ve.Kind() == reflect.Ptr { - if ve.IsNil() { - nilFound = true - break - } - indirects++ - addr := ve.Pointer() - pointerChain = append(pointerChain, addr) - if pd, ok := f.pointers[addr]; ok && pd < f.depth { - cycleFound = true - indirects-- - break - } - f.pointers[addr] = f.depth - - ve = ve.Elem() - if ve.Kind() == reflect.Interface { - if ve.IsNil() { - nilFound = true - break - } - ve = ve.Elem() - } - } - - // Display type or indirection level depending on flags. - if showTypes && !f.ignoreNextType { - f.fs.Write(openParenBytes) - f.fs.Write(bytes.Repeat(asteriskBytes, indirects)) - f.fs.Write([]byte(ve.Type().String())) - f.fs.Write(closeParenBytes) - } else { - if nilFound || cycleFound { - indirects += strings.Count(ve.Type().String(), "*") - } - f.fs.Write(openAngleBytes) - f.fs.Write([]byte(strings.Repeat("*", indirects))) - f.fs.Write(closeAngleBytes) - } - - // Display pointer information depending on flags. - if f.fs.Flag('+') && (len(pointerChain) > 0) { - f.fs.Write(openParenBytes) - for i, addr := range pointerChain { - if i > 0 { - f.fs.Write(pointerChainBytes) - } - printHexPtr(f.fs, addr) - } - f.fs.Write(closeParenBytes) - } - - // Display dereferenced value. - switch { - case nilFound: - f.fs.Write(nilAngleBytes) - - case cycleFound: - f.fs.Write(circularShortBytes) - - default: - f.ignoreNextType = true - f.format(ve) - } -} - -// format is the main workhorse for providing the Formatter interface. It -// uses the passed reflect value to figure out what kind of object we are -// dealing with and formats it appropriately. It is a recursive function, -// however circular data structures are detected and handled properly. -func (f *formatState) format(v reflect.Value) { - // Handle invalid reflect values immediately. - kind := v.Kind() - if kind == reflect.Invalid { - f.fs.Write(invalidAngleBytes) - return - } - - // Handle pointers specially. - if kind == reflect.Ptr { - f.formatPtr(v) - return - } - - // Print type information unless already handled elsewhere. - if !f.ignoreNextType && f.fs.Flag('#') { - f.fs.Write(openParenBytes) - f.fs.Write([]byte(v.Type().String())) - f.fs.Write(closeParenBytes) - } - f.ignoreNextType = false - - // Call Stringer/error interfaces if they exist and the handle methods - // flag is enabled. - if !f.cs.DisableMethods { - if (kind != reflect.Invalid) && (kind != reflect.Interface) { - if handled := handleMethods(f.cs, f.fs, v); handled { - return - } - } - } - - switch kind { - case reflect.Invalid: - // Do nothing. We should never get here since invalid has already - // been handled above. - - case reflect.Bool: - printBool(f.fs, v.Bool()) - - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: - printInt(f.fs, v.Int(), 10) - - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - printUint(f.fs, v.Uint(), 10) - - case reflect.Float32: - printFloat(f.fs, v.Float(), 32) - - case reflect.Float64: - printFloat(f.fs, v.Float(), 64) - - case reflect.Complex64: - printComplex(f.fs, v.Complex(), 32) - - case reflect.Complex128: - printComplex(f.fs, v.Complex(), 64) - - case reflect.Slice: - if v.IsNil() { - f.fs.Write(nilAngleBytes) - break - } - fallthrough - - case reflect.Array: - f.fs.Write(openBracketBytes) - f.depth++ - if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { - f.fs.Write(maxShortBytes) - } else { - numEntries := v.Len() - for i := 0; i < numEntries; i++ { - if i > 0 { - f.fs.Write(spaceBytes) - } - f.ignoreNextType = true - f.format(f.unpackValue(v.Index(i))) - } - } - f.depth-- - f.fs.Write(closeBracketBytes) - - case reflect.String: - f.fs.Write([]byte(v.String())) - - case reflect.Interface: - // The only time we should get here is for nil interfaces due to - // unpackValue calls. - if v.IsNil() { - f.fs.Write(nilAngleBytes) - } - - case reflect.Ptr: - // Do nothing. We should never get here since pointers have already - // been handled above. - - case reflect.Map: - // nil maps should be indicated as different than empty maps - if v.IsNil() { - f.fs.Write(nilAngleBytes) - break - } - - f.fs.Write(openMapBytes) - f.depth++ - if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { - f.fs.Write(maxShortBytes) - } else { - keys := v.MapKeys() - if f.cs.SortKeys { - sortValues(keys, f.cs) - } - for i, key := range keys { - if i > 0 { - f.fs.Write(spaceBytes) - } - f.ignoreNextType = true - f.format(f.unpackValue(key)) - f.fs.Write(colonBytes) - f.ignoreNextType = true - f.format(f.unpackValue(v.MapIndex(key))) - } - } - f.depth-- - f.fs.Write(closeMapBytes) - - case reflect.Struct: - numFields := v.NumField() - f.fs.Write(openBraceBytes) - f.depth++ - if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { - f.fs.Write(maxShortBytes) - } else { - vt := v.Type() - for i := 0; i < numFields; i++ { - if i > 0 { - f.fs.Write(spaceBytes) - } - vtf := vt.Field(i) - if f.fs.Flag('+') || f.fs.Flag('#') { - f.fs.Write([]byte(vtf.Name)) - f.fs.Write(colonBytes) - } - f.format(f.unpackValue(v.Field(i))) - } - } - f.depth-- - f.fs.Write(closeBraceBytes) - - case reflect.Uintptr: - printHexPtr(f.fs, uintptr(v.Uint())) - - case reflect.UnsafePointer, reflect.Chan, reflect.Func: - printHexPtr(f.fs, v.Pointer()) - - // There were not any other types at the time this code was written, but - // fall back to letting the default fmt package handle it if any get added. - default: - format := f.buildDefaultFormat() - if v.CanInterface() { - fmt.Fprintf(f.fs, format, v.Interface()) - } else { - fmt.Fprintf(f.fs, format, v.String()) - } - } -} - -// Format satisfies the fmt.Formatter interface. See NewFormatter for usage -// details. -func (f *formatState) Format(fs fmt.State, verb rune) { - f.fs = fs - - // Use standard formatting for verbs that are not v. - if verb != 'v' { - format := f.constructOrigFormat(verb) - fmt.Fprintf(fs, format, f.value) - return - } - - if f.value == nil { - if fs.Flag('#') { - fs.Write(interfaceBytes) - } - fs.Write(nilAngleBytes) - return - } - - f.format(reflect.ValueOf(f.value)) -} - -// newFormatter is a helper function to consolidate the logic from the various -// public methods which take varying config states. -func newFormatter(cs *ConfigState, v interface{}) fmt.Formatter { - fs := &formatState{value: v, cs: cs} - fs.pointers = make(map[uintptr]int) - return fs -} - -/* -NewFormatter returns a custom formatter that satisfies the fmt.Formatter -interface. As a result, it integrates cleanly with standard fmt package -printing functions. The formatter is useful for inline printing of smaller data -types similar to the standard %v format specifier. - -The custom formatter only responds to the %v (most compact), %+v (adds pointer -addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb -combinations. Any other verbs such as %x and %q will be sent to the the -standard fmt package for formatting. In addition, the custom formatter ignores -the width and precision arguments (however they will still work on the format -specifiers not handled by the custom formatter). - -Typically this function shouldn't be called directly. It is much easier to make -use of the custom formatter by calling one of the convenience functions such as -Printf, Println, or Fprintf. -*/ -func NewFormatter(v interface{}) fmt.Formatter { - return newFormatter(&Config, v) -} diff --git a/vendor/github.com/davecgh/go-spew/spew/spew.go b/vendor/github.com/davecgh/go-spew/spew/spew.go deleted file mode 100644 index 32c0e3388..000000000 --- a/vendor/github.com/davecgh/go-spew/spew/spew.go +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "fmt" - "io" -) - -// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the formatted string as a value that satisfies error. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Errorf(format, spew.NewFormatter(a), spew.NewFormatter(b)) -func Errorf(format string, a ...interface{}) (err error) { - return fmt.Errorf(format, convertArgs(a)...) -} - -// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprint(w, spew.NewFormatter(a), spew.NewFormatter(b)) -func Fprint(w io.Writer, a ...interface{}) (n int, err error) { - return fmt.Fprint(w, convertArgs(a)...) -} - -// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprintf(w, format, spew.NewFormatter(a), spew.NewFormatter(b)) -func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { - return fmt.Fprintf(w, format, convertArgs(a)...) -} - -// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it -// passed with a default Formatter interface returned by NewFormatter. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprintln(w, spew.NewFormatter(a), spew.NewFormatter(b)) -func Fprintln(w io.Writer, a ...interface{}) (n int, err error) { - return fmt.Fprintln(w, convertArgs(a)...) -} - -// Print is a wrapper for fmt.Print that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Print(spew.NewFormatter(a), spew.NewFormatter(b)) -func Print(a ...interface{}) (n int, err error) { - return fmt.Print(convertArgs(a)...) -} - -// Printf is a wrapper for fmt.Printf that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Printf(format, spew.NewFormatter(a), spew.NewFormatter(b)) -func Printf(format string, a ...interface{}) (n int, err error) { - return fmt.Printf(format, convertArgs(a)...) -} - -// Println is a wrapper for fmt.Println that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Println(spew.NewFormatter(a), spew.NewFormatter(b)) -func Println(a ...interface{}) (n int, err error) { - return fmt.Println(convertArgs(a)...) -} - -// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprint(spew.NewFormatter(a), spew.NewFormatter(b)) -func Sprint(a ...interface{}) string { - return fmt.Sprint(convertArgs(a)...) -} - -// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprintf(format, spew.NewFormatter(a), spew.NewFormatter(b)) -func Sprintf(format string, a ...interface{}) string { - return fmt.Sprintf(format, convertArgs(a)...) -} - -// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it -// were passed with a default Formatter interface returned by NewFormatter. It -// returns the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprintln(spew.NewFormatter(a), spew.NewFormatter(b)) -func Sprintln(a ...interface{}) string { - return fmt.Sprintln(convertArgs(a)...) -} - -// convertArgs accepts a slice of arguments and returns a slice of the same -// length with each argument converted to a default spew Formatter interface. -func convertArgs(args []interface{}) (formatters []interface{}) { - formatters = make([]interface{}, len(args)) - for index, arg := range args { - formatters[index] = NewFormatter(arg) - } - return formatters -} diff --git a/vendor/github.com/edsrzf/mmap-go/.gitignore b/vendor/github.com/edsrzf/mmap-go/.gitignore new file mode 100644 index 000000000..9aa02c1ed --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/.gitignore @@ -0,0 +1,8 @@ +*.out +*.5 +*.6 +*.8 +*.swp +_obj +_test +testdata diff --git a/vendor/github.com/edsrzf/mmap-go/LICENSE b/vendor/github.com/edsrzf/mmap-go/LICENSE new file mode 100644 index 000000000..8f05f338a --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2011, Evan Shaw +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/github.com/edsrzf/mmap-go/README.md b/vendor/github.com/edsrzf/mmap-go/README.md new file mode 100644 index 000000000..4cc2bfe1c --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/README.md @@ -0,0 +1,12 @@ +mmap-go +======= + +mmap-go is a portable mmap package for the [Go programming language](http://golang.org). +It has been tested on Linux (386, amd64), OS X, and Windows (386). It should also +work on other Unix-like platforms, but hasn't been tested with them. I'm interested +to hear about the results. + +I haven't been able to add more features without adding significant complexity, +so mmap-go doesn't support mprotect, mincore, and maybe a few other things. +If you're running on a Unix-like platform and need some of these features, +I suggest Gustavo Niemeyer's [gommap](http://labix.org/gommap). diff --git a/vendor/github.com/edsrzf/mmap-go/mmap.go b/vendor/github.com/edsrzf/mmap-go/mmap.go new file mode 100644 index 000000000..29655bd22 --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/mmap.go @@ -0,0 +1,117 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file defines the common package interface and contains a little bit of +// factored out logic. + +// Package mmap allows mapping files into memory. It tries to provide a simple, reasonably portable interface, +// but doesn't go out of its way to abstract away every little platform detail. +// This specifically means: +// * forked processes may or may not inherit mappings +// * a file's timestamp may or may not be updated by writes through mappings +// * specifying a size larger than the file's actual size can increase the file's size +// * If the mapped file is being modified by another process while your program's running, don't expect consistent results between platforms +package mmap + +import ( + "errors" + "os" + "reflect" + "unsafe" +) + +const ( + // RDONLY maps the memory read-only. + // Attempts to write to the MMap object will result in undefined behavior. + RDONLY = 0 + // RDWR maps the memory as read-write. Writes to the MMap object will update the + // underlying file. + RDWR = 1 << iota + // COPY maps the memory as copy-on-write. Writes to the MMap object will affect + // memory, but the underlying file will remain unchanged. + COPY + // If EXEC is set, the mapped memory is marked as executable. + EXEC +) + +const ( + // If the ANON flag is set, the mapped memory will not be backed by a file. + ANON = 1 << iota +) + +// MMap represents a file mapped into memory. +type MMap []byte + +// Map maps an entire file into memory. +// If ANON is set in flags, f is ignored. +func Map(f *os.File, prot, flags int) (MMap, error) { + return MapRegion(f, -1, prot, flags, 0) +} + +// MapRegion maps part of a file into memory. +// The offset parameter must be a multiple of the system's page size. +// If length < 0, the entire file will be mapped. +// If ANON is set in flags, f is ignored. +func MapRegion(f *os.File, length int, prot, flags int, offset int64) (MMap, error) { + if offset%int64(os.Getpagesize()) != 0 { + return nil, errors.New("offset parameter must be a multiple of the system's page size") + } + + var fd uintptr + if flags&ANON == 0 { + fd = uintptr(f.Fd()) + if length < 0 { + fi, err := f.Stat() + if err != nil { + return nil, err + } + length = int(fi.Size()) + } + } else { + if length <= 0 { + return nil, errors.New("anonymous mapping requires non-zero length") + } + fd = ^uintptr(0) + } + return mmap(length, uintptr(prot), uintptr(flags), fd, offset) +} + +func (m *MMap) header() *reflect.SliceHeader { + return (*reflect.SliceHeader)(unsafe.Pointer(m)) +} + +func (m *MMap) addrLen() (uintptr, uintptr) { + header := m.header() + return header.Data, uintptr(header.Len) +} + +// Lock keeps the mapped region in physical memory, ensuring that it will not be +// swapped out. +func (m MMap) Lock() error { + return m.lock() +} + +// Unlock reverses the effect of Lock, allowing the mapped region to potentially +// be swapped out. +// If m is already unlocked, aan error will result. +func (m MMap) Unlock() error { + return m.unlock() +} + +// Flush synchronizes the mapping's contents to the file's contents on disk. +func (m MMap) Flush() error { + return m.flush() +} + +// Unmap deletes the memory mapped region, flushes any remaining changes, and sets +// m to nil. +// Trying to read or write any remaining references to m after Unmap is called will +// result in undefined behavior. +// Unmap should only be called on the slice value that was originally returned from +// a call to Map. Calling Unmap on a derived slice may cause errors. +func (m *MMap) Unmap() error { + err := m.unmap() + *m = nil + return err +} diff --git a/vendor/github.com/edsrzf/mmap-go/mmap_unix.go b/vendor/github.com/edsrzf/mmap-go/mmap_unix.go new file mode 100644 index 000000000..25b13e51f --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/mmap_unix.go @@ -0,0 +1,51 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux openbsd solaris netbsd + +package mmap + +import ( + "golang.org/x/sys/unix" +) + +func mmap(len int, inprot, inflags, fd uintptr, off int64) ([]byte, error) { + flags := unix.MAP_SHARED + prot := unix.PROT_READ + switch { + case inprot© != 0: + prot |= unix.PROT_WRITE + flags = unix.MAP_PRIVATE + case inprot&RDWR != 0: + prot |= unix.PROT_WRITE + } + if inprot&EXEC != 0 { + prot |= unix.PROT_EXEC + } + if inflags&ANON != 0 { + flags |= unix.MAP_ANON + } + + b, err := unix.Mmap(int(fd), off, len, prot, flags) + if err != nil { + return nil, err + } + return b, nil +} + +func (m MMap) flush() error { + return unix.Msync([]byte(m), unix.MS_SYNC) +} + +func (m MMap) lock() error { + return unix.Mlock([]byte(m)) +} + +func (m MMap) unlock() error { + return unix.Munlock([]byte(m)) +} + +func (m MMap) unmap() error { + return unix.Munmap([]byte(m)) +} diff --git a/vendor/github.com/edsrzf/mmap-go/mmap_windows.go b/vendor/github.com/edsrzf/mmap-go/mmap_windows.go new file mode 100644 index 000000000..7910da257 --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/mmap_windows.go @@ -0,0 +1,143 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mmap + +import ( + "errors" + "os" + "sync" + + "golang.org/x/sys/windows" +) + +// mmap on Windows is a two-step process. +// First, we call CreateFileMapping to get a handle. +// Then, we call MapviewToFile to get an actual pointer into memory. +// Because we want to emulate a POSIX-style mmap, we don't want to expose +// the handle -- only the pointer. We also want to return only a byte slice, +// not a struct, so it's convenient to manipulate. + +// We keep this map so that we can get back the original handle from the memory address. + +type addrinfo struct { + file windows.Handle + mapview windows.Handle +} + +var handleLock sync.Mutex +var handleMap = map[uintptr]*addrinfo{} + +func mmap(len int, prot, flags, hfile uintptr, off int64) ([]byte, error) { + flProtect := uint32(windows.PAGE_READONLY) + dwDesiredAccess := uint32(windows.FILE_MAP_READ) + switch { + case prot© != 0: + flProtect = windows.PAGE_WRITECOPY + dwDesiredAccess = windows.FILE_MAP_COPY + case prot&RDWR != 0: + flProtect = windows.PAGE_READWRITE + dwDesiredAccess = windows.FILE_MAP_WRITE + } + if prot&EXEC != 0 { + flProtect <<= 4 + dwDesiredAccess |= windows.FILE_MAP_EXECUTE + } + + // The maximum size is the area of the file, starting from 0, + // that we wish to allow to be mappable. It is the sum of + // the length the user requested, plus the offset where that length + // is starting from. This does not map the data into memory. + maxSizeHigh := uint32((off + int64(len)) >> 32) + maxSizeLow := uint32((off + int64(len)) & 0xFFFFFFFF) + // TODO: Do we need to set some security attributes? It might help portability. + h, errno := windows.CreateFileMapping(windows.Handle(hfile), nil, flProtect, maxSizeHigh, maxSizeLow, nil) + if h == 0 { + return nil, os.NewSyscallError("CreateFileMapping", errno) + } + + // Actually map a view of the data into memory. The view's size + // is the length the user requested. + fileOffsetHigh := uint32(off >> 32) + fileOffsetLow := uint32(off & 0xFFFFFFFF) + addr, errno := windows.MapViewOfFile(h, dwDesiredAccess, fileOffsetHigh, fileOffsetLow, uintptr(len)) + if addr == 0 { + return nil, os.NewSyscallError("MapViewOfFile", errno) + } + handleLock.Lock() + handleMap[addr] = &addrinfo{ + file: windows.Handle(hfile), + mapview: h, + } + handleLock.Unlock() + + m := MMap{} + dh := m.header() + dh.Data = addr + dh.Len = len + dh.Cap = dh.Len + + return m, nil +} + +func (m MMap) flush() error { + addr, len := m.addrLen() + errno := windows.FlushViewOfFile(addr, len) + if errno != nil { + return os.NewSyscallError("FlushViewOfFile", errno) + } + + handleLock.Lock() + defer handleLock.Unlock() + handle, ok := handleMap[addr] + if !ok { + // should be impossible; we would've errored above + return errors.New("unknown base address") + } + + errno = windows.FlushFileBuffers(handle.file) + return os.NewSyscallError("FlushFileBuffers", errno) +} + +func (m MMap) lock() error { + addr, len := m.addrLen() + errno := windows.VirtualLock(addr, len) + return os.NewSyscallError("VirtualLock", errno) +} + +func (m MMap) unlock() error { + addr, len := m.addrLen() + errno := windows.VirtualUnlock(addr, len) + return os.NewSyscallError("VirtualUnlock", errno) +} + +func (m MMap) unmap() error { + err := m.flush() + if err != nil { + return err + } + + addr := m.header().Data + // Lock the UnmapViewOfFile along with the handleMap deletion. + // As soon as we unmap the view, the OS is free to give the + // same addr to another new map. We don't want another goroutine + // to insert and remove the same addr into handleMap while + // we're trying to remove our old addr/handle pair. + handleLock.Lock() + defer handleLock.Unlock() + err = windows.UnmapViewOfFile(addr) + if err != nil { + return err + } + + handle, ok := handleMap[addr] + if !ok { + // should be impossible; we would've errored above + return errors.New("unknown base address") + } + delete(handleMap, addr) + + e := windows.CloseHandle(windows.Handle(handle.mapview)) + return os.NewSyscallError("CloseHandle", e) +} diff --git a/vendor/github.com/google/uuid/.travis.yml b/vendor/github.com/google/uuid/.travis.yml new file mode 100644 index 000000000..d8156a60b --- /dev/null +++ b/vendor/github.com/google/uuid/.travis.yml @@ -0,0 +1,9 @@ +language: go + +go: + - 1.4.3 + - 1.5.3 + - tip + +script: + - go test -v ./... diff --git a/vendor/github.com/google/uuid/CONTRIBUTING.md b/vendor/github.com/google/uuid/CONTRIBUTING.md new file mode 100644 index 000000000..04fdf09f1 --- /dev/null +++ b/vendor/github.com/google/uuid/CONTRIBUTING.md @@ -0,0 +1,10 @@ +# How to contribute + +We definitely welcome patches and contribution to this project! + +### Legal requirements + +In order to protect both you and ourselves, you will need to sign the +[Contributor License Agreement](https://cla.developers.google.com/clas). + +You may have already signed it for other Google projects. diff --git a/vendor/github.com/google/uuid/CONTRIBUTORS b/vendor/github.com/google/uuid/CONTRIBUTORS new file mode 100644 index 000000000..b4bb97f6b --- /dev/null +++ b/vendor/github.com/google/uuid/CONTRIBUTORS @@ -0,0 +1,9 @@ +Paul Borman +bmatsuo +shawnps +theory +jboverfelt +dsymonds +cd1 +wallclockbuilder +dansouza diff --git a/vendor/github.com/google/uuid/LICENSE b/vendor/github.com/google/uuid/LICENSE new file mode 100644 index 000000000..5dc68268d --- /dev/null +++ b/vendor/github.com/google/uuid/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009,2014 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/google/uuid/README.md b/vendor/github.com/google/uuid/README.md new file mode 100644 index 000000000..9d92c11f1 --- /dev/null +++ b/vendor/github.com/google/uuid/README.md @@ -0,0 +1,19 @@ +# uuid ![build status](https://travis-ci.org/google/uuid.svg?branch=master) +The uuid package generates and inspects UUIDs based on +[RFC 4122](http://tools.ietf.org/html/rfc4122) +and DCE 1.1: Authentication and Security Services. + +This package is based on the github.com/pborman/uuid package (previously named +code.google.com/p/go-uuid). It differs from these earlier packages in that +a UUID is a 16 byte array rather than a byte slice. One loss due to this +change is the ability to represent an invalid UUID (vs a NIL UUID). + +###### Install +`go get github.com/google/uuid` + +###### Documentation +[![GoDoc](https://godoc.org/github.com/google/uuid?status.svg)](http://godoc.org/github.com/google/uuid) + +Full `go doc` style documentation for the package can be viewed online without +installing this package by using the GoDoc site here: +http://godoc.org/github.com/google/uuid diff --git a/vendor/github.com/google/uuid/dce.go b/vendor/github.com/google/uuid/dce.go new file mode 100644 index 000000000..fa820b9d3 --- /dev/null +++ b/vendor/github.com/google/uuid/dce.go @@ -0,0 +1,80 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" + "fmt" + "os" +) + +// A Domain represents a Version 2 domain +type Domain byte + +// Domain constants for DCE Security (Version 2) UUIDs. +const ( + Person = Domain(0) + Group = Domain(1) + Org = Domain(2) +) + +// NewDCESecurity returns a DCE Security (Version 2) UUID. +// +// The domain should be one of Person, Group or Org. +// On a POSIX system the id should be the users UID for the Person +// domain and the users GID for the Group. The meaning of id for +// the domain Org or on non-POSIX systems is site defined. +// +// For a given domain/id pair the same token may be returned for up to +// 7 minutes and 10 seconds. +func NewDCESecurity(domain Domain, id uint32) (UUID, error) { + uuid, err := NewUUID() + if err == nil { + uuid[6] = (uuid[6] & 0x0f) | 0x20 // Version 2 + uuid[9] = byte(domain) + binary.BigEndian.PutUint32(uuid[0:], id) + } + return uuid, err +} + +// NewDCEPerson returns a DCE Security (Version 2) UUID in the person +// domain with the id returned by os.Getuid. +// +// NewDCESecurity(Person, uint32(os.Getuid())) +func NewDCEPerson() (UUID, error) { + return NewDCESecurity(Person, uint32(os.Getuid())) +} + +// NewDCEGroup returns a DCE Security (Version 2) UUID in the group +// domain with the id returned by os.Getgid. +// +// NewDCESecurity(Group, uint32(os.Getgid())) +func NewDCEGroup() (UUID, error) { + return NewDCESecurity(Group, uint32(os.Getgid())) +} + +// Domain returns the domain for a Version 2 UUID. Domains are only defined +// for Version 2 UUIDs. +func (uuid UUID) Domain() Domain { + return Domain(uuid[9]) +} + +// ID returns the id for a Version 2 UUID. IDs are only defined for Version 2 +// UUIDs. +func (uuid UUID) ID() uint32 { + return binary.BigEndian.Uint32(uuid[0:4]) +} + +func (d Domain) String() string { + switch d { + case Person: + return "Person" + case Group: + return "Group" + case Org: + return "Org" + } + return fmt.Sprintf("Domain%d", int(d)) +} diff --git a/vendor/github.com/google/uuid/doc.go b/vendor/github.com/google/uuid/doc.go new file mode 100644 index 000000000..5b8a4b9af --- /dev/null +++ b/vendor/github.com/google/uuid/doc.go @@ -0,0 +1,12 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package uuid generates and inspects UUIDs. +// +// UUIDs are based on RFC 4122 and DCE 1.1: Authentication and Security +// Services. +// +// A UUID is a 16 byte (128 bit) array. UUIDs may be used as keys to +// maps or compared directly. +package uuid diff --git a/vendor/github.com/google/uuid/go.mod b/vendor/github.com/google/uuid/go.mod new file mode 100644 index 000000000..fc84cd79d --- /dev/null +++ b/vendor/github.com/google/uuid/go.mod @@ -0,0 +1 @@ +module github.com/google/uuid diff --git a/vendor/github.com/google/uuid/hash.go b/vendor/github.com/google/uuid/hash.go new file mode 100644 index 000000000..b17461631 --- /dev/null +++ b/vendor/github.com/google/uuid/hash.go @@ -0,0 +1,53 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "crypto/md5" + "crypto/sha1" + "hash" +) + +// Well known namespace IDs and UUIDs +var ( + NameSpaceDNS = Must(Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")) + NameSpaceURL = Must(Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) + NameSpaceOID = Must(Parse("6ba7b812-9dad-11d1-80b4-00c04fd430c8")) + NameSpaceX500 = Must(Parse("6ba7b814-9dad-11d1-80b4-00c04fd430c8")) + Nil UUID // empty UUID, all zeros +) + +// NewHash returns a new UUID derived from the hash of space concatenated with +// data generated by h. The hash should be at least 16 byte in length. The +// first 16 bytes of the hash are used to form the UUID. The version of the +// UUID will be the lower 4 bits of version. NewHash is used to implement +// NewMD5 and NewSHA1. +func NewHash(h hash.Hash, space UUID, data []byte, version int) UUID { + h.Reset() + h.Write(space[:]) + h.Write(data) + s := h.Sum(nil) + var uuid UUID + copy(uuid[:], s) + uuid[6] = (uuid[6] & 0x0f) | uint8((version&0xf)<<4) + uuid[8] = (uuid[8] & 0x3f) | 0x80 // RFC 4122 variant + return uuid +} + +// NewMD5 returns a new MD5 (Version 3) UUID based on the +// supplied name space and data. It is the same as calling: +// +// NewHash(md5.New(), space, data, 3) +func NewMD5(space UUID, data []byte) UUID { + return NewHash(md5.New(), space, data, 3) +} + +// NewSHA1 returns a new SHA1 (Version 5) UUID based on the +// supplied name space and data. It is the same as calling: +// +// NewHash(sha1.New(), space, data, 5) +func NewSHA1(space UUID, data []byte) UUID { + return NewHash(sha1.New(), space, data, 5) +} diff --git a/vendor/github.com/google/uuid/marshal.go b/vendor/github.com/google/uuid/marshal.go new file mode 100644 index 000000000..7f9e0c6c0 --- /dev/null +++ b/vendor/github.com/google/uuid/marshal.go @@ -0,0 +1,37 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import "fmt" + +// MarshalText implements encoding.TextMarshaler. +func (uuid UUID) MarshalText() ([]byte, error) { + var js [36]byte + encodeHex(js[:], uuid) + return js[:], nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (uuid *UUID) UnmarshalText(data []byte) error { + id, err := ParseBytes(data) + if err == nil { + *uuid = id + } + return err +} + +// MarshalBinary implements encoding.BinaryMarshaler. +func (uuid UUID) MarshalBinary() ([]byte, error) { + return uuid[:], nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +func (uuid *UUID) UnmarshalBinary(data []byte) error { + if len(data) != 16 { + return fmt.Errorf("invalid UUID (got %d bytes)", len(data)) + } + copy(uuid[:], data) + return nil +} diff --git a/vendor/github.com/google/uuid/node.go b/vendor/github.com/google/uuid/node.go new file mode 100644 index 000000000..d651a2b06 --- /dev/null +++ b/vendor/github.com/google/uuid/node.go @@ -0,0 +1,90 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "sync" +) + +var ( + nodeMu sync.Mutex + ifname string // name of interface being used + nodeID [6]byte // hardware for version 1 UUIDs + zeroID [6]byte // nodeID with only 0's +) + +// NodeInterface returns the name of the interface from which the NodeID was +// derived. The interface "user" is returned if the NodeID was set by +// SetNodeID. +func NodeInterface() string { + defer nodeMu.Unlock() + nodeMu.Lock() + return ifname +} + +// SetNodeInterface selects the hardware address to be used for Version 1 UUIDs. +// If name is "" then the first usable interface found will be used or a random +// Node ID will be generated. If a named interface cannot be found then false +// is returned. +// +// SetNodeInterface never fails when name is "". +func SetNodeInterface(name string) bool { + defer nodeMu.Unlock() + nodeMu.Lock() + return setNodeInterface(name) +} + +func setNodeInterface(name string) bool { + iname, addr := getHardwareInterface(name) // null implementation for js + if iname != "" && addr != nil { + ifname = iname + copy(nodeID[:], addr) + return true + } + + // We found no interfaces with a valid hardware address. If name + // does not specify a specific interface generate a random Node ID + // (section 4.1.6) + if name == "" { + ifname = "random" + randomBits(nodeID[:]) + return true + } + return false +} + +// NodeID returns a slice of a copy of the current Node ID, setting the Node ID +// if not already set. +func NodeID() []byte { + defer nodeMu.Unlock() + nodeMu.Lock() + if nodeID == zeroID { + setNodeInterface("") + } + nid := nodeID + return nid[:] +} + +// SetNodeID sets the Node ID to be used for Version 1 UUIDs. The first 6 bytes +// of id are used. If id is less than 6 bytes then false is returned and the +// Node ID is not set. +func SetNodeID(id []byte) bool { + if len(id) < 6 { + return false + } + defer nodeMu.Unlock() + nodeMu.Lock() + copy(nodeID[:], id) + ifname = "user" + return true +} + +// NodeID returns the 6 byte node id encoded in uuid. It returns nil if uuid is +// not valid. The NodeID is only well defined for version 1 and 2 UUIDs. +func (uuid UUID) NodeID() []byte { + var node [6]byte + copy(node[:], uuid[10:]) + return node[:] +} diff --git a/vendor/github.com/google/uuid/node_js.go b/vendor/github.com/google/uuid/node_js.go new file mode 100644 index 000000000..24b78edc9 --- /dev/null +++ b/vendor/github.com/google/uuid/node_js.go @@ -0,0 +1,12 @@ +// Copyright 2017 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js + +package uuid + +// getHardwareInterface returns nil values for the JS version of the code. +// This remvoves the "net" dependency, because it is not used in the browser. +// Using the "net" library inflates the size of the transpiled JS code by 673k bytes. +func getHardwareInterface(name string) (string, []byte) { return "", nil } diff --git a/vendor/github.com/google/uuid/node_net.go b/vendor/github.com/google/uuid/node_net.go new file mode 100644 index 000000000..0cbbcddbd --- /dev/null +++ b/vendor/github.com/google/uuid/node_net.go @@ -0,0 +1,33 @@ +// Copyright 2017 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !js + +package uuid + +import "net" + +var interfaces []net.Interface // cached list of interfaces + +// getHardwareInterface returns the name and hardware address of interface name. +// If name is "" then the name and hardware address of one of the system's +// interfaces is returned. If no interfaces are found (name does not exist or +// there are no interfaces) then "", nil is returned. +// +// Only addresses of at least 6 bytes are returned. +func getHardwareInterface(name string) (string, []byte) { + if interfaces == nil { + var err error + interfaces, err = net.Interfaces() + if err != nil { + return "", nil + } + } + for _, ifs := range interfaces { + if len(ifs.HardwareAddr) >= 6 && (name == "" || name == ifs.Name) { + return ifs.Name, ifs.HardwareAddr + } + } + return "", nil +} diff --git a/vendor/github.com/google/uuid/sql.go b/vendor/github.com/google/uuid/sql.go new file mode 100644 index 000000000..f326b54db --- /dev/null +++ b/vendor/github.com/google/uuid/sql.go @@ -0,0 +1,59 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "database/sql/driver" + "fmt" +) + +// Scan implements sql.Scanner so UUIDs can be read from databases transparently +// Currently, database types that map to string and []byte are supported. Please +// consult database-specific driver documentation for matching types. +func (uuid *UUID) Scan(src interface{}) error { + switch src := src.(type) { + case nil: + return nil + + case string: + // if an empty UUID comes from a table, we return a null UUID + if src == "" { + return nil + } + + // see Parse for required string format + u, err := Parse(src) + if err != nil { + return fmt.Errorf("Scan: %v", err) + } + + *uuid = u + + case []byte: + // if an empty UUID comes from a table, we return a null UUID + if len(src) == 0 { + return nil + } + + // assumes a simple slice of bytes if 16 bytes + // otherwise attempts to parse + if len(src) != 16 { + return uuid.Scan(string(src)) + } + copy((*uuid)[:], src) + + default: + return fmt.Errorf("Scan: unable to scan type %T into UUID", src) + } + + return nil +} + +// Value implements sql.Valuer so that UUIDs can be written to databases +// transparently. Currently, UUIDs map to strings. Please consult +// database-specific driver documentation for matching types. +func (uuid UUID) Value() (driver.Value, error) { + return uuid.String(), nil +} diff --git a/vendor/github.com/google/uuid/time.go b/vendor/github.com/google/uuid/time.go new file mode 100644 index 000000000..e6ef06cdc --- /dev/null +++ b/vendor/github.com/google/uuid/time.go @@ -0,0 +1,123 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" + "sync" + "time" +) + +// A Time represents a time as the number of 100's of nanoseconds since 15 Oct +// 1582. +type Time int64 + +const ( + lillian = 2299160 // Julian day of 15 Oct 1582 + unix = 2440587 // Julian day of 1 Jan 1970 + epoch = unix - lillian // Days between epochs + g1582 = epoch * 86400 // seconds between epochs + g1582ns100 = g1582 * 10000000 // 100s of a nanoseconds between epochs +) + +var ( + timeMu sync.Mutex + lasttime uint64 // last time we returned + clockSeq uint16 // clock sequence for this run + + timeNow = time.Now // for testing +) + +// UnixTime converts t the number of seconds and nanoseconds using the Unix +// epoch of 1 Jan 1970. +func (t Time) UnixTime() (sec, nsec int64) { + sec = int64(t - g1582ns100) + nsec = (sec % 10000000) * 100 + sec /= 10000000 + return sec, nsec +} + +// GetTime returns the current Time (100s of nanoseconds since 15 Oct 1582) and +// clock sequence as well as adjusting the clock sequence as needed. An error +// is returned if the current time cannot be determined. +func GetTime() (Time, uint16, error) { + defer timeMu.Unlock() + timeMu.Lock() + return getTime() +} + +func getTime() (Time, uint16, error) { + t := timeNow() + + // If we don't have a clock sequence already, set one. + if clockSeq == 0 { + setClockSequence(-1) + } + now := uint64(t.UnixNano()/100) + g1582ns100 + + // If time has gone backwards with this clock sequence then we + // increment the clock sequence + if now <= lasttime { + clockSeq = ((clockSeq + 1) & 0x3fff) | 0x8000 + } + lasttime = now + return Time(now), clockSeq, nil +} + +// ClockSequence returns the current clock sequence, generating one if not +// already set. The clock sequence is only used for Version 1 UUIDs. +// +// The uuid package does not use global static storage for the clock sequence or +// the last time a UUID was generated. Unless SetClockSequence is used, a new +// random clock sequence is generated the first time a clock sequence is +// requested by ClockSequence, GetTime, or NewUUID. (section 4.2.1.1) +func ClockSequence() int { + defer timeMu.Unlock() + timeMu.Lock() + return clockSequence() +} + +func clockSequence() int { + if clockSeq == 0 { + setClockSequence(-1) + } + return int(clockSeq & 0x3fff) +} + +// SetClockSequence sets the clock sequence to the lower 14 bits of seq. Setting to +// -1 causes a new sequence to be generated. +func SetClockSequence(seq int) { + defer timeMu.Unlock() + timeMu.Lock() + setClockSequence(seq) +} + +func setClockSequence(seq int) { + if seq == -1 { + var b [2]byte + randomBits(b[:]) // clock sequence + seq = int(b[0])<<8 | int(b[1]) + } + oldSeq := clockSeq + clockSeq = uint16(seq&0x3fff) | 0x8000 // Set our variant + if oldSeq != clockSeq { + lasttime = 0 + } +} + +// Time returns the time in 100s of nanoseconds since 15 Oct 1582 encoded in +// uuid. The time is only defined for version 1 and 2 UUIDs. +func (uuid UUID) Time() Time { + time := int64(binary.BigEndian.Uint32(uuid[0:4])) + time |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32 + time |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48 + return Time(time) +} + +// ClockSequence returns the clock sequence encoded in uuid. +// The clock sequence is only well defined for version 1 and 2 UUIDs. +func (uuid UUID) ClockSequence() int { + return int(binary.BigEndian.Uint16(uuid[8:10])) & 0x3fff +} diff --git a/vendor/github.com/google/uuid/util.go b/vendor/github.com/google/uuid/util.go new file mode 100644 index 000000000..5ea6c7378 --- /dev/null +++ b/vendor/github.com/google/uuid/util.go @@ -0,0 +1,43 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "io" +) + +// randomBits completely fills slice b with random data. +func randomBits(b []byte) { + if _, err := io.ReadFull(rander, b); err != nil { + panic(err.Error()) // rand should never fail + } +} + +// xvalues returns the value of a byte as a hexadecimal digit or 255. +var xvalues = [256]byte{ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +} + +// xtob converts hex characters x1 and x2 into a byte. +func xtob(x1, x2 byte) (byte, bool) { + b1 := xvalues[x1] + b2 := xvalues[x2] + return (b1 << 4) | b2, b1 != 255 && b2 != 255 +} diff --git a/vendor/github.com/google/uuid/uuid.go b/vendor/github.com/google/uuid/uuid.go new file mode 100644 index 000000000..524404cc5 --- /dev/null +++ b/vendor/github.com/google/uuid/uuid.go @@ -0,0 +1,245 @@ +// Copyright 2018 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "strings" +) + +// A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC +// 4122. +type UUID [16]byte + +// A Version represents a UUID's version. +type Version byte + +// A Variant represents a UUID's variant. +type Variant byte + +// Constants returned by Variant. +const ( + Invalid = Variant(iota) // Invalid UUID + RFC4122 // The variant specified in RFC4122 + Reserved // Reserved, NCS backward compatibility. + Microsoft // Reserved, Microsoft Corporation backward compatibility. + Future // Reserved for future definition. +) + +var rander = rand.Reader // random function + +// Parse decodes s into a UUID or returns an error. Both the standard UUID +// forms of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded as well as the +// Microsoft encoding {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} and the raw hex +// encoding: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx. +func Parse(s string) (UUID, error) { + var uuid UUID + switch len(s) { + // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36: + + // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36 + 9: + if strings.ToLower(s[:9]) != "urn:uuid:" { + return uuid, fmt.Errorf("invalid urn prefix: %q", s[:9]) + } + s = s[9:] + + // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} + case 36 + 2: + s = s[1:] + + // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + case 32: + var ok bool + for i := range uuid { + uuid[i], ok = xtob(s[i*2], s[i*2+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + } + return uuid, nil + default: + return uuid, fmt.Errorf("invalid UUID length: %d", len(s)) + } + // s is now at least 36 bytes long + // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { + return uuid, errors.New("invalid UUID format") + } + for i, x := range [16]int{ + 0, 2, 4, 6, + 9, 11, + 14, 16, + 19, 21, + 24, 26, 28, 30, 32, 34} { + v, ok := xtob(s[x], s[x+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + uuid[i] = v + } + return uuid, nil +} + +// ParseBytes is like Parse, except it parses a byte slice instead of a string. +func ParseBytes(b []byte) (UUID, error) { + var uuid UUID + switch len(b) { + case 36: // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36 + 9: // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + if !bytes.Equal(bytes.ToLower(b[:9]), []byte("urn:uuid:")) { + return uuid, fmt.Errorf("invalid urn prefix: %q", b[:9]) + } + b = b[9:] + case 36 + 2: // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} + b = b[1:] + case 32: // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + var ok bool + for i := 0; i < 32; i += 2 { + uuid[i/2], ok = xtob(b[i], b[i+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + } + return uuid, nil + default: + return uuid, fmt.Errorf("invalid UUID length: %d", len(b)) + } + // s is now at least 36 bytes long + // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + if b[8] != '-' || b[13] != '-' || b[18] != '-' || b[23] != '-' { + return uuid, errors.New("invalid UUID format") + } + for i, x := range [16]int{ + 0, 2, 4, 6, + 9, 11, + 14, 16, + 19, 21, + 24, 26, 28, 30, 32, 34} { + v, ok := xtob(b[x], b[x+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + uuid[i] = v + } + return uuid, nil +} + +// MustParse is like Parse but panics if the string cannot be parsed. +// It simplifies safe initialization of global variables holding compiled UUIDs. +func MustParse(s string) UUID { + uuid, err := Parse(s) + if err != nil { + panic(`uuid: Parse(` + s + `): ` + err.Error()) + } + return uuid +} + +// FromBytes creates a new UUID from a byte slice. Returns an error if the slice +// does not have a length of 16. The bytes are copied from the slice. +func FromBytes(b []byte) (uuid UUID, err error) { + err = uuid.UnmarshalBinary(b) + return uuid, err +} + +// Must returns uuid if err is nil and panics otherwise. +func Must(uuid UUID, err error) UUID { + if err != nil { + panic(err) + } + return uuid +} + +// String returns the string form of uuid, xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +// , or "" if uuid is invalid. +func (uuid UUID) String() string { + var buf [36]byte + encodeHex(buf[:], uuid) + return string(buf[:]) +} + +// URN returns the RFC 2141 URN form of uuid, +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx, or "" if uuid is invalid. +func (uuid UUID) URN() string { + var buf [36 + 9]byte + copy(buf[:], "urn:uuid:") + encodeHex(buf[9:], uuid) + return string(buf[:]) +} + +func encodeHex(dst []byte, uuid UUID) { + hex.Encode(dst, uuid[:4]) + dst[8] = '-' + hex.Encode(dst[9:13], uuid[4:6]) + dst[13] = '-' + hex.Encode(dst[14:18], uuid[6:8]) + dst[18] = '-' + hex.Encode(dst[19:23], uuid[8:10]) + dst[23] = '-' + hex.Encode(dst[24:], uuid[10:]) +} + +// Variant returns the variant encoded in uuid. +func (uuid UUID) Variant() Variant { + switch { + case (uuid[8] & 0xc0) == 0x80: + return RFC4122 + case (uuid[8] & 0xe0) == 0xc0: + return Microsoft + case (uuid[8] & 0xe0) == 0xe0: + return Future + default: + return Reserved + } +} + +// Version returns the version of uuid. +func (uuid UUID) Version() Version { + return Version(uuid[6] >> 4) +} + +func (v Version) String() string { + if v > 15 { + return fmt.Sprintf("BAD_VERSION_%d", v) + } + return fmt.Sprintf("VERSION_%d", v) +} + +func (v Variant) String() string { + switch v { + case RFC4122: + return "RFC4122" + case Reserved: + return "Reserved" + case Microsoft: + return "Microsoft" + case Future: + return "Future" + case Invalid: + return "Invalid" + } + return fmt.Sprintf("BadVariant%d", int(v)) +} + +// SetRand sets the random number generator to r, which implements io.Reader. +// If r.Read returns an error when the package requests random data then +// a panic will be issued. +// +// Calling SetRand with nil sets the random number generator to the default +// generator. +func SetRand(r io.Reader) { + if r == nil { + rander = rand.Reader + return + } + rander = r +} diff --git a/vendor/github.com/google/uuid/version1.go b/vendor/github.com/google/uuid/version1.go new file mode 100644 index 000000000..199a1ac65 --- /dev/null +++ b/vendor/github.com/google/uuid/version1.go @@ -0,0 +1,44 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" +) + +// NewUUID returns a Version 1 UUID based on the current NodeID and clock +// sequence, and the current time. If the NodeID has not been set by SetNodeID +// or SetNodeInterface then it will be set automatically. If the NodeID cannot +// be set NewUUID returns nil. If clock sequence has not been set by +// SetClockSequence then it will be set automatically. If GetTime fails to +// return the current NewUUID returns nil and an error. +// +// In most cases, New should be used. +func NewUUID() (UUID, error) { + nodeMu.Lock() + if nodeID == zeroID { + setNodeInterface("") + } + nodeMu.Unlock() + + var uuid UUID + now, seq, err := GetTime() + if err != nil { + return uuid, err + } + + timeLow := uint32(now & 0xffffffff) + timeMid := uint16((now >> 32) & 0xffff) + timeHi := uint16((now >> 48) & 0x0fff) + timeHi |= 0x1000 // Version 1 + + binary.BigEndian.PutUint32(uuid[0:], timeLow) + binary.BigEndian.PutUint16(uuid[4:], timeMid) + binary.BigEndian.PutUint16(uuid[6:], timeHi) + binary.BigEndian.PutUint16(uuid[8:], seq) + copy(uuid[10:], nodeID[:]) + + return uuid, nil +} diff --git a/vendor/github.com/google/uuid/version4.go b/vendor/github.com/google/uuid/version4.go new file mode 100644 index 000000000..84af91c9f --- /dev/null +++ b/vendor/github.com/google/uuid/version4.go @@ -0,0 +1,38 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import "io" + +// New creates a new random UUID or panics. New is equivalent to +// the expression +// +// uuid.Must(uuid.NewRandom()) +func New() UUID { + return Must(NewRandom()) +} + +// NewRandom returns a Random (Version 4) UUID. +// +// The strength of the UUIDs is based on the strength of the crypto/rand +// package. +// +// A note about uniqueness derived from the UUID Wikipedia entry: +// +// Randomly generated UUIDs have 122 random bits. One's annual risk of being +// hit by a meteorite is estimated to be one chance in 17 billion, that +// means the probability is about 0.00000000006 (6 × 10−11), +// equivalent to the odds of creating a few tens of trillions of UUIDs in a +// year and having one duplicate. +func NewRandom() (UUID, error) { + var uuid UUID + _, err := io.ReadFull(rander, uuid[:]) + if err != nil { + return Nil, err + } + uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4 + uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 + return uuid, nil +} diff --git a/vendor/github.com/gorilla/mux/AUTHORS b/vendor/github.com/gorilla/mux/AUTHORS new file mode 100644 index 000000000..b722392ee --- /dev/null +++ b/vendor/github.com/gorilla/mux/AUTHORS @@ -0,0 +1,8 @@ +# This is the official list of gorilla/mux authors for copyright purposes. +# +# Please keep the list sorted. + +Google LLC (https://opensource.google.com/) +Kamil Kisielk +Matt Silverlock +Rodrigo Moraes (https://github.com/moraes) diff --git a/vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md b/vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..232be82e4 --- /dev/null +++ b/vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md @@ -0,0 +1,11 @@ +**What version of Go are you running?** (Paste the output of `go version`) + + +**What version of gorilla/mux are you at?** (Paste the output of `git rev-parse HEAD` inside `$GOPATH/src/github.com/gorilla/mux`) + + +**Describe your problem** (and what you have tried so far) + + +**Paste a minimal, runnable, reproduction of your issue below** (use backticks to format it) + diff --git a/vendor/github.com/gorilla/mux/LICENSE b/vendor/github.com/gorilla/mux/LICENSE new file mode 100644 index 000000000..6903df638 --- /dev/null +++ b/vendor/github.com/gorilla/mux/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012-2018 The Gorilla Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/gorilla/mux/README.md b/vendor/github.com/gorilla/mux/README.md new file mode 100644 index 000000000..c661599ab --- /dev/null +++ b/vendor/github.com/gorilla/mux/README.md @@ -0,0 +1,649 @@ +# gorilla/mux + +[![GoDoc](https://godoc.org/github.com/gorilla/mux?status.svg)](https://godoc.org/github.com/gorilla/mux) +[![Build Status](https://travis-ci.org/gorilla/mux.svg?branch=master)](https://travis-ci.org/gorilla/mux) +[![Sourcegraph](https://sourcegraph.com/github.com/gorilla/mux/-/badge.svg)](https://sourcegraph.com/github.com/gorilla/mux?badge) + +![Gorilla Logo](http://www.gorillatoolkit.org/static/images/gorilla-icon-64.png) + +https://www.gorillatoolkit.org/pkg/mux + +Package `gorilla/mux` implements a request router and dispatcher for matching incoming requests to +their respective handler. + +The name mux stands for "HTTP request multiplexer". Like the standard `http.ServeMux`, `mux.Router` matches incoming requests against a list of registered routes and calls a handler for the route that matches the URL or other conditions. The main features are: + +* It implements the `http.Handler` interface so it is compatible with the standard `http.ServeMux`. +* Requests can be matched based on URL host, path, path prefix, schemes, header and query values, HTTP methods or using custom matchers. +* URL hosts, paths and query values can have variables with an optional regular expression. +* Registered URLs can be built, or "reversed", which helps maintaining references to resources. +* Routes can be used as subrouters: nested routes are only tested if the parent route matches. This is useful to define groups of routes that share common conditions like a host, a path prefix or other repeated attributes. As a bonus, this optimizes request matching. + +--- + +* [Install](#install) +* [Examples](#examples) +* [Matching Routes](#matching-routes) +* [Static Files](#static-files) +* [Registered URLs](#registered-urls) +* [Walking Routes](#walking-routes) +* [Graceful Shutdown](#graceful-shutdown) +* [Middleware](#middleware) +* [Testing Handlers](#testing-handlers) +* [Full Example](#full-example) + +--- + +## Install + +With a [correctly configured](https://golang.org/doc/install#testing) Go toolchain: + +```sh +go get -u github.com/gorilla/mux +``` + +## Examples + +Let's start registering a couple of URL paths and handlers: + +```go +func main() { + r := mux.NewRouter() + r.HandleFunc("/", HomeHandler) + r.HandleFunc("/products", ProductsHandler) + r.HandleFunc("/articles", ArticlesHandler) + http.Handle("/", r) +} +``` + +Here we register three routes mapping URL paths to handlers. This is equivalent to how `http.HandleFunc()` works: if an incoming request URL matches one of the paths, the corresponding handler is called passing (`http.ResponseWriter`, `*http.Request`) as parameters. + +Paths can have variables. They are defined using the format `{name}` or `{name:pattern}`. If a regular expression pattern is not defined, the matched variable will be anything until the next slash. For example: + +```go +r := mux.NewRouter() +r.HandleFunc("/products/{key}", ProductHandler) +r.HandleFunc("/articles/{category}/", ArticlesCategoryHandler) +r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler) +``` + +The names are used to create a map of route variables which can be retrieved calling `mux.Vars()`: + +```go +func ArticlesCategoryHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "Category: %v\n", vars["category"]) +} +``` + +And this is all you need to know about the basic usage. More advanced options are explained below. + +### Matching Routes + +Routes can also be restricted to a domain or subdomain. Just define a host pattern to be matched. They can also have variables: + +```go +r := mux.NewRouter() +// Only matches if domain is "www.example.com". +r.Host("www.example.com") +// Matches a dynamic subdomain. +r.Host("{subdomain:[a-z]+}.example.com") +``` + +There are several other matchers that can be added. To match path prefixes: + +```go +r.PathPrefix("/products/") +``` + +...or HTTP methods: + +```go +r.Methods("GET", "POST") +``` + +...or URL schemes: + +```go +r.Schemes("https") +``` + +...or header values: + +```go +r.Headers("X-Requested-With", "XMLHttpRequest") +``` + +...or query values: + +```go +r.Queries("key", "value") +``` + +...or to use a custom matcher function: + +```go +r.MatcherFunc(func(r *http.Request, rm *RouteMatch) bool { + return r.ProtoMajor == 0 +}) +``` + +...and finally, it is possible to combine several matchers in a single route: + +```go +r.HandleFunc("/products", ProductsHandler). + Host("www.example.com"). + Methods("GET"). + Schemes("http") +``` + +Routes are tested in the order they were added to the router. If two routes match, the first one wins: + +```go +r := mux.NewRouter() +r.HandleFunc("/specific", specificHandler) +r.PathPrefix("/").Handler(catchAllHandler) +``` + +Setting the same matching conditions again and again can be boring, so we have a way to group several routes that share the same requirements. We call it "subrouting". + +For example, let's say we have several URLs that should only match when the host is `www.example.com`. Create a route for that host and get a "subrouter" from it: + +```go +r := mux.NewRouter() +s := r.Host("www.example.com").Subrouter() +``` + +Then register routes in the subrouter: + +```go +s.HandleFunc("/products/", ProductsHandler) +s.HandleFunc("/products/{key}", ProductHandler) +s.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler) +``` + +The three URL paths we registered above will only be tested if the domain is `www.example.com`, because the subrouter is tested first. This is not only convenient, but also optimizes request matching. You can create subrouters combining any attribute matchers accepted by a route. + +Subrouters can be used to create domain or path "namespaces": you define subrouters in a central place and then parts of the app can register its paths relatively to a given subrouter. + +There's one more thing about subroutes. When a subrouter has a path prefix, the inner routes use it as base for their paths: + +```go +r := mux.NewRouter() +s := r.PathPrefix("/products").Subrouter() +// "/products/" +s.HandleFunc("/", ProductsHandler) +// "/products/{key}/" +s.HandleFunc("/{key}/", ProductHandler) +// "/products/{key}/details" +s.HandleFunc("/{key}/details", ProductDetailsHandler) +``` + + +### Static Files + +Note that the path provided to `PathPrefix()` represents a "wildcard": calling +`PathPrefix("/static/").Handler(...)` means that the handler will be passed any +request that matches "/static/\*". This makes it easy to serve static files with mux: + +```go +func main() { + var dir string + + flag.StringVar(&dir, "dir", ".", "the directory to serve files from. Defaults to the current dir") + flag.Parse() + r := mux.NewRouter() + + // This will serve files under http://localhost:8000/static/ + r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir)))) + + srv := &http.Server{ + Handler: r, + Addr: "127.0.0.1:8000", + // Good practice: enforce timeouts for servers you create! + WriteTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + } + + log.Fatal(srv.ListenAndServe()) +} +``` + +### Registered URLs + +Now let's see how to build registered URLs. + +Routes can be named. All routes that define a name can have their URLs built, or "reversed". We define a name calling `Name()` on a route. For example: + +```go +r := mux.NewRouter() +r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). + Name("article") +``` + +To build a URL, get the route and call the `URL()` method, passing a sequence of key/value pairs for the route variables. For the previous route, we would do: + +```go +url, err := r.Get("article").URL("category", "technology", "id", "42") +``` + +...and the result will be a `url.URL` with the following path: + +``` +"/articles/technology/42" +``` + +This also works for host and query value variables: + +```go +r := mux.NewRouter() +r.Host("{subdomain}.example.com"). + Path("/articles/{category}/{id:[0-9]+}"). + Queries("filter", "{filter}"). + HandlerFunc(ArticleHandler). + Name("article") + +// url.String() will be "http://news.example.com/articles/technology/42?filter=gorilla" +url, err := r.Get("article").URL("subdomain", "news", + "category", "technology", + "id", "42", + "filter", "gorilla") +``` + +All variables defined in the route are required, and their values must conform to the corresponding patterns. These requirements guarantee that a generated URL will always match a registered route -- the only exception is for explicitly defined "build-only" routes which never match. + +Regex support also exists for matching Headers within a route. For example, we could do: + +```go +r.HeadersRegexp("Content-Type", "application/(text|json)") +``` + +...and the route will match both requests with a Content-Type of `application/json` as well as `application/text` + +There's also a way to build only the URL host or path for a route: use the methods `URLHost()` or `URLPath()` instead. For the previous route, we would do: + +```go +// "http://news.example.com/" +host, err := r.Get("article").URLHost("subdomain", "news") + +// "/articles/technology/42" +path, err := r.Get("article").URLPath("category", "technology", "id", "42") +``` + +And if you use subrouters, host and path defined separately can be built as well: + +```go +r := mux.NewRouter() +s := r.Host("{subdomain}.example.com").Subrouter() +s.Path("/articles/{category}/{id:[0-9]+}"). + HandlerFunc(ArticleHandler). + Name("article") + +// "http://news.example.com/articles/technology/42" +url, err := r.Get("article").URL("subdomain", "news", + "category", "technology", + "id", "42") +``` + +### Walking Routes + +The `Walk` function on `mux.Router` can be used to visit all of the routes that are registered on a router. For example, +the following prints all of the registered routes: + +```go +package main + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gorilla/mux" +) + +func handler(w http.ResponseWriter, r *http.Request) { + return +} + +func main() { + r := mux.NewRouter() + r.HandleFunc("/", handler) + r.HandleFunc("/products", handler).Methods("POST") + r.HandleFunc("/articles", handler).Methods("GET") + r.HandleFunc("/articles/{id}", handler).Methods("GET", "PUT") + r.HandleFunc("/authors", handler).Queries("surname", "{surname}") + err := r.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + pathTemplate, err := route.GetPathTemplate() + if err == nil { + fmt.Println("ROUTE:", pathTemplate) + } + pathRegexp, err := route.GetPathRegexp() + if err == nil { + fmt.Println("Path regexp:", pathRegexp) + } + queriesTemplates, err := route.GetQueriesTemplates() + if err == nil { + fmt.Println("Queries templates:", strings.Join(queriesTemplates, ",")) + } + queriesRegexps, err := route.GetQueriesRegexp() + if err == nil { + fmt.Println("Queries regexps:", strings.Join(queriesRegexps, ",")) + } + methods, err := route.GetMethods() + if err == nil { + fmt.Println("Methods:", strings.Join(methods, ",")) + } + fmt.Println() + return nil + }) + + if err != nil { + fmt.Println(err) + } + + http.Handle("/", r) +} +``` + +### Graceful Shutdown + +Go 1.8 introduced the ability to [gracefully shutdown](https://golang.org/doc/go1.8#http_shutdown) a `*http.Server`. Here's how to do that alongside `mux`: + +```go +package main + +import ( + "context" + "flag" + "log" + "net/http" + "os" + "os/signal" + "time" + + "github.com/gorilla/mux" +) + +func main() { + var wait time.Duration + flag.DurationVar(&wait, "graceful-timeout", time.Second * 15, "the duration for which the server gracefully wait for existing connections to finish - e.g. 15s or 1m") + flag.Parse() + + r := mux.NewRouter() + // Add your routes as needed + + srv := &http.Server{ + Addr: "0.0.0.0:8080", + // Good practice to set timeouts to avoid Slowloris attacks. + WriteTimeout: time.Second * 15, + ReadTimeout: time.Second * 15, + IdleTimeout: time.Second * 60, + Handler: r, // Pass our instance of gorilla/mux in. + } + + // Run our server in a goroutine so that it doesn't block. + go func() { + if err := srv.ListenAndServe(); err != nil { + log.Println(err) + } + }() + + c := make(chan os.Signal, 1) + // We'll accept graceful shutdowns when quit via SIGINT (Ctrl+C) + // SIGKILL, SIGQUIT or SIGTERM (Ctrl+/) will not be caught. + signal.Notify(c, os.Interrupt) + + // Block until we receive our signal. + <-c + + // Create a deadline to wait for. + ctx, cancel := context.WithTimeout(context.Background(), wait) + defer cancel() + // Doesn't block if no connections, but will otherwise wait + // until the timeout deadline. + srv.Shutdown(ctx) + // Optionally, you could run srv.Shutdown in a goroutine and block on + // <-ctx.Done() if your application should wait for other services + // to finalize based on context cancellation. + log.Println("shutting down") + os.Exit(0) +} +``` + +### Middleware + +Mux supports the addition of middlewares to a [Router](https://godoc.org/github.com/gorilla/mux#Router), which are executed in the order they are added if a match is found, including its subrouters. +Middlewares are (typically) small pieces of code which take one request, do something with it, and pass it down to another middleware or the final handler. Some common use cases for middleware are request logging, header manipulation, or `ResponseWriter` hijacking. + +Mux middlewares are defined using the de facto standard type: + +```go +type MiddlewareFunc func(http.Handler) http.Handler +``` + +Typically, the returned handler is a closure which does something with the http.ResponseWriter and http.Request passed to it, and then calls the handler passed as parameter to the MiddlewareFunc. This takes advantage of closures being able access variables from the context where they are created, while retaining the signature enforced by the receivers. + +A very basic middleware which logs the URI of the request being handled could be written as: + +```go +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Do stuff here + log.Println(r.RequestURI) + // Call the next handler, which can be another middleware in the chain, or the final handler. + next.ServeHTTP(w, r) + }) +} +``` + +Middlewares can be added to a router using `Router.Use()`: + +```go +r := mux.NewRouter() +r.HandleFunc("/", handler) +r.Use(loggingMiddleware) +``` + +A more complex authentication middleware, which maps session token to users, could be written as: + +```go +// Define our struct +type authenticationMiddleware struct { + tokenUsers map[string]string +} + +// Initialize it somewhere +func (amw *authenticationMiddleware) Populate() { + amw.tokenUsers["00000000"] = "user0" + amw.tokenUsers["aaaaaaaa"] = "userA" + amw.tokenUsers["05f717e5"] = "randomUser" + amw.tokenUsers["deadbeef"] = "user0" +} + +// Middleware function, which will be called for each request +func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("X-Session-Token") + + if user, found := amw.tokenUsers[token]; found { + // We found the token in our map + log.Printf("Authenticated user %s\n", user) + // Pass down the request to the next middleware (or final handler) + next.ServeHTTP(w, r) + } else { + // Write an error and stop the handler chain + http.Error(w, "Forbidden", http.StatusForbidden) + } + }) +} +``` + +```go +r := mux.NewRouter() +r.HandleFunc("/", handler) + +amw := authenticationMiddleware{} +amw.Populate() + +r.Use(amw.Middleware) +``` + +Note: The handler chain will be stopped if your middleware doesn't call `next.ServeHTTP()` with the corresponding parameters. This can be used to abort a request if the middleware writer wants to. Middlewares _should_ write to `ResponseWriter` if they _are_ going to terminate the request, and they _should not_ write to `ResponseWriter` if they _are not_ going to terminate it. + +### Testing Handlers + +Testing handlers in a Go web application is straightforward, and _mux_ doesn't complicate this any further. Given two files: `endpoints.go` and `endpoints_test.go`, here's how we'd test an application using _mux_. + +First, our simple HTTP handler: + +```go +// endpoints.go +package main + +func HealthCheckHandler(w http.ResponseWriter, r *http.Request) { + // A very simple health check. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + // In the future we could report back on the status of our DB, or our cache + // (e.g. Redis) by performing a simple PING, and include them in the response. + io.WriteString(w, `{"alive": true}`) +} + +func main() { + r := mux.NewRouter() + r.HandleFunc("/health", HealthCheckHandler) + + log.Fatal(http.ListenAndServe("localhost:8080", r)) +} +``` + +Our test code: + +```go +// endpoints_test.go +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestHealthCheckHandler(t *testing.T) { + // Create a request to pass to our handler. We don't have any query parameters for now, so we'll + // pass 'nil' as the third parameter. + req, err := http.NewRequest("GET", "/health", nil) + if err != nil { + t.Fatal(err) + } + + // We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response. + rr := httptest.NewRecorder() + handler := http.HandlerFunc(HealthCheckHandler) + + // Our handlers satisfy http.Handler, so we can call their ServeHTTP method + // directly and pass in our Request and ResponseRecorder. + handler.ServeHTTP(rr, req) + + // Check the status code is what we expect. + if status := rr.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code: got %v want %v", + status, http.StatusOK) + } + + // Check the response body is what we expect. + expected := `{"alive": true}` + if rr.Body.String() != expected { + t.Errorf("handler returned unexpected body: got %v want %v", + rr.Body.String(), expected) + } +} +``` + +In the case that our routes have [variables](#examples), we can pass those in the request. We could write +[table-driven tests](https://dave.cheney.net/2013/06/09/writing-table-driven-tests-in-go) to test multiple +possible route variables as needed. + +```go +// endpoints.go +func main() { + r := mux.NewRouter() + // A route with a route variable: + r.HandleFunc("/metrics/{type}", MetricsHandler) + + log.Fatal(http.ListenAndServe("localhost:8080", r)) +} +``` + +Our test file, with a table-driven test of `routeVariables`: + +```go +// endpoints_test.go +func TestMetricsHandler(t *testing.T) { + tt := []struct{ + routeVariable string + shouldPass bool + }{ + {"goroutines", true}, + {"heap", true}, + {"counters", true}, + {"queries", true}, + {"adhadaeqm3k", false}, + } + + for _, tc := range tt { + path := fmt.Sprintf("/metrics/%s", tc.routeVariable) + req, err := http.NewRequest("GET", path, nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + + // Need to create a router that we can pass the request through so that the vars will be added to the context + router := mux.NewRouter() + router.HandleFunc("/metrics/{type}", MetricsHandler) + router.ServeHTTP(rr, req) + + // In this case, our MetricsHandler returns a non-200 response + // for a route variable it doesn't know about. + if rr.Code == http.StatusOK && !tc.shouldPass { + t.Errorf("handler should have failed on routeVariable %s: got %v want %v", + tc.routeVariable, rr.Code, http.StatusOK) + } + } +} +``` + +## Full Example + +Here's a complete, runnable example of a small `mux` based server: + +```go +package main + +import ( + "net/http" + "log" + "github.com/gorilla/mux" +) + +func YourHandler(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Gorilla!\n")) +} + +func main() { + r := mux.NewRouter() + // Routes consist of a path and a handler function. + r.HandleFunc("/", YourHandler) + + // Bind to a port and pass our router in + log.Fatal(http.ListenAndServe(":8000", r)) +} +``` + +## License + +BSD licensed. See the LICENSE file for details. diff --git a/vendor/github.com/gorilla/mux/bench_test.go b/vendor/github.com/gorilla/mux/bench_test.go new file mode 100644 index 000000000..522156dcc --- /dev/null +++ b/vendor/github.com/gorilla/mux/bench_test.go @@ -0,0 +1,49 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func BenchmarkMux(b *testing.B) { + router := new(Router) + handler := func(w http.ResponseWriter, r *http.Request) {} + router.HandleFunc("/v1/{v1}", handler) + + request, _ := http.NewRequest("GET", "/v1/anything", nil) + for i := 0; i < b.N; i++ { + router.ServeHTTP(nil, request) + } +} + +func BenchmarkMuxAlternativeInRegexp(b *testing.B) { + router := new(Router) + handler := func(w http.ResponseWriter, r *http.Request) {} + router.HandleFunc("/v1/{v1:(?:a|b)}", handler) + + requestA, _ := http.NewRequest("GET", "/v1/a", nil) + requestB, _ := http.NewRequest("GET", "/v1/b", nil) + for i := 0; i < b.N; i++ { + router.ServeHTTP(nil, requestA) + router.ServeHTTP(nil, requestB) + } +} + +func BenchmarkManyPathVariables(b *testing.B) { + router := new(Router) + handler := func(w http.ResponseWriter, r *http.Request) {} + router.HandleFunc("/v1/{v1}/{v2}/{v3}/{v4}/{v5}", handler) + + matchingRequest, _ := http.NewRequest("GET", "/v1/1/2/3/4/5", nil) + notMatchingRequest, _ := http.NewRequest("GET", "/v1/1/2/3/4", nil) + recorder := httptest.NewRecorder() + for i := 0; i < b.N; i++ { + router.ServeHTTP(nil, matchingRequest) + router.ServeHTTP(recorder, notMatchingRequest) + } +} diff --git a/vendor/github.com/gorilla/mux/context.go b/vendor/github.com/gorilla/mux/context.go new file mode 100644 index 000000000..665940a26 --- /dev/null +++ b/vendor/github.com/gorilla/mux/context.go @@ -0,0 +1,18 @@ +package mux + +import ( + "context" + "net/http" +) + +func contextGet(r *http.Request, key interface{}) interface{} { + return r.Context().Value(key) +} + +func contextSet(r *http.Request, key, val interface{}) *http.Request { + if val == nil { + return r + } + + return r.WithContext(context.WithValue(r.Context(), key, val)) +} diff --git a/vendor/github.com/gorilla/mux/context_test.go b/vendor/github.com/gorilla/mux/context_test.go new file mode 100644 index 000000000..d8a56b422 --- /dev/null +++ b/vendor/github.com/gorilla/mux/context_test.go @@ -0,0 +1,30 @@ +package mux + +import ( + "context" + "net/http" + "testing" + "time" +) + +func TestNativeContextMiddleware(t *testing.T) { + withTimeout := func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), time.Minute) + defer cancel() + h.ServeHTTP(w, r.WithContext(ctx)) + }) + } + + r := NewRouter() + r.Handle("/path/{foo}", withTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := Vars(r) + if vars["foo"] != "bar" { + t.Fatal("Expected foo var to be set") + } + }))) + + rec := NewRecorder() + req := newRequest("GET", "/path/bar") + r.ServeHTTP(rec, req) +} diff --git a/vendor/github.com/gorilla/mux/doc.go b/vendor/github.com/gorilla/mux/doc.go new file mode 100644 index 000000000..38957deea --- /dev/null +++ b/vendor/github.com/gorilla/mux/doc.go @@ -0,0 +1,306 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package mux implements a request router and dispatcher. + +The name mux stands for "HTTP request multiplexer". Like the standard +http.ServeMux, mux.Router matches incoming requests against a list of +registered routes and calls a handler for the route that matches the URL +or other conditions. The main features are: + + * Requests can be matched based on URL host, path, path prefix, schemes, + header and query values, HTTP methods or using custom matchers. + * URL hosts, paths and query values can have variables with an optional + regular expression. + * Registered URLs can be built, or "reversed", which helps maintaining + references to resources. + * Routes can be used as subrouters: nested routes are only tested if the + parent route matches. This is useful to define groups of routes that + share common conditions like a host, a path prefix or other repeated + attributes. As a bonus, this optimizes request matching. + * It implements the http.Handler interface so it is compatible with the + standard http.ServeMux. + +Let's start registering a couple of URL paths and handlers: + + func main() { + r := mux.NewRouter() + r.HandleFunc("/", HomeHandler) + r.HandleFunc("/products", ProductsHandler) + r.HandleFunc("/articles", ArticlesHandler) + http.Handle("/", r) + } + +Here we register three routes mapping URL paths to handlers. This is +equivalent to how http.HandleFunc() works: if an incoming request URL matches +one of the paths, the corresponding handler is called passing +(http.ResponseWriter, *http.Request) as parameters. + +Paths can have variables. They are defined using the format {name} or +{name:pattern}. If a regular expression pattern is not defined, the matched +variable will be anything until the next slash. For example: + + r := mux.NewRouter() + r.HandleFunc("/products/{key}", ProductHandler) + r.HandleFunc("/articles/{category}/", ArticlesCategoryHandler) + r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler) + +Groups can be used inside patterns, as long as they are non-capturing (?:re). For example: + + r.HandleFunc("/articles/{category}/{sort:(?:asc|desc|new)}", ArticlesCategoryHandler) + +The names are used to create a map of route variables which can be retrieved +calling mux.Vars(): + + vars := mux.Vars(request) + category := vars["category"] + +Note that if any capturing groups are present, mux will panic() during parsing. To prevent +this, convert any capturing groups to non-capturing, e.g. change "/{sort:(asc|desc)}" to +"/{sort:(?:asc|desc)}". This is a change from prior versions which behaved unpredictably +when capturing groups were present. + +And this is all you need to know about the basic usage. More advanced options +are explained below. + +Routes can also be restricted to a domain or subdomain. Just define a host +pattern to be matched. They can also have variables: + + r := mux.NewRouter() + // Only matches if domain is "www.example.com". + r.Host("www.example.com") + // Matches a dynamic subdomain. + r.Host("{subdomain:[a-z]+}.domain.com") + +There are several other matchers that can be added. To match path prefixes: + + r.PathPrefix("/products/") + +...or HTTP methods: + + r.Methods("GET", "POST") + +...or URL schemes: + + r.Schemes("https") + +...or header values: + + r.Headers("X-Requested-With", "XMLHttpRequest") + +...or query values: + + r.Queries("key", "value") + +...or to use a custom matcher function: + + r.MatcherFunc(func(r *http.Request, rm *RouteMatch) bool { + return r.ProtoMajor == 0 + }) + +...and finally, it is possible to combine several matchers in a single route: + + r.HandleFunc("/products", ProductsHandler). + Host("www.example.com"). + Methods("GET"). + Schemes("http") + +Setting the same matching conditions again and again can be boring, so we have +a way to group several routes that share the same requirements. +We call it "subrouting". + +For example, let's say we have several URLs that should only match when the +host is "www.example.com". Create a route for that host and get a "subrouter" +from it: + + r := mux.NewRouter() + s := r.Host("www.example.com").Subrouter() + +Then register routes in the subrouter: + + s.HandleFunc("/products/", ProductsHandler) + s.HandleFunc("/products/{key}", ProductHandler) + s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler) + +The three URL paths we registered above will only be tested if the domain is +"www.example.com", because the subrouter is tested first. This is not +only convenient, but also optimizes request matching. You can create +subrouters combining any attribute matchers accepted by a route. + +Subrouters can be used to create domain or path "namespaces": you define +subrouters in a central place and then parts of the app can register its +paths relatively to a given subrouter. + +There's one more thing about subroutes. When a subrouter has a path prefix, +the inner routes use it as base for their paths: + + r := mux.NewRouter() + s := r.PathPrefix("/products").Subrouter() + // "/products/" + s.HandleFunc("/", ProductsHandler) + // "/products/{key}/" + s.HandleFunc("/{key}/", ProductHandler) + // "/products/{key}/details" + s.HandleFunc("/{key}/details", ProductDetailsHandler) + +Note that the path provided to PathPrefix() represents a "wildcard": calling +PathPrefix("/static/").Handler(...) means that the handler will be passed any +request that matches "/static/*". This makes it easy to serve static files with mux: + + func main() { + var dir string + + flag.StringVar(&dir, "dir", ".", "the directory to serve files from. Defaults to the current dir") + flag.Parse() + r := mux.NewRouter() + + // This will serve files under http://localhost:8000/static/ + r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir)))) + + srv := &http.Server{ + Handler: r, + Addr: "127.0.0.1:8000", + // Good practice: enforce timeouts for servers you create! + WriteTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + } + + log.Fatal(srv.ListenAndServe()) + } + +Now let's see how to build registered URLs. + +Routes can be named. All routes that define a name can have their URLs built, +or "reversed". We define a name calling Name() on a route. For example: + + r := mux.NewRouter() + r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). + Name("article") + +To build a URL, get the route and call the URL() method, passing a sequence of +key/value pairs for the route variables. For the previous route, we would do: + + url, err := r.Get("article").URL("category", "technology", "id", "42") + +...and the result will be a url.URL with the following path: + + "/articles/technology/42" + +This also works for host and query value variables: + + r := mux.NewRouter() + r.Host("{subdomain}.domain.com"). + Path("/articles/{category}/{id:[0-9]+}"). + Queries("filter", "{filter}"). + HandlerFunc(ArticleHandler). + Name("article") + + // url.String() will be "http://news.domain.com/articles/technology/42?filter=gorilla" + url, err := r.Get("article").URL("subdomain", "news", + "category", "technology", + "id", "42", + "filter", "gorilla") + +All variables defined in the route are required, and their values must +conform to the corresponding patterns. These requirements guarantee that a +generated URL will always match a registered route -- the only exception is +for explicitly defined "build-only" routes which never match. + +Regex support also exists for matching Headers within a route. For example, we could do: + + r.HeadersRegexp("Content-Type", "application/(text|json)") + +...and the route will match both requests with a Content-Type of `application/json` as well as +`application/text` + +There's also a way to build only the URL host or path for a route: +use the methods URLHost() or URLPath() instead. For the previous route, +we would do: + + // "http://news.domain.com/" + host, err := r.Get("article").URLHost("subdomain", "news") + + // "/articles/technology/42" + path, err := r.Get("article").URLPath("category", "technology", "id", "42") + +And if you use subrouters, host and path defined separately can be built +as well: + + r := mux.NewRouter() + s := r.Host("{subdomain}.domain.com").Subrouter() + s.Path("/articles/{category}/{id:[0-9]+}"). + HandlerFunc(ArticleHandler). + Name("article") + + // "http://news.domain.com/articles/technology/42" + url, err := r.Get("article").URL("subdomain", "news", + "category", "technology", + "id", "42") + +Mux supports the addition of middlewares to a Router, which are executed in the order they are added if a match is found, including its subrouters. Middlewares are (typically) small pieces of code which take one request, do something with it, and pass it down to another middleware or the final handler. Some common use cases for middleware are request logging, header manipulation, or ResponseWriter hijacking. + + type MiddlewareFunc func(http.Handler) http.Handler + +Typically, the returned handler is a closure which does something with the http.ResponseWriter and http.Request passed to it, and then calls the handler passed as parameter to the MiddlewareFunc (closures can access variables from the context where they are created). + +A very basic middleware which logs the URI of the request being handled could be written as: + + func simpleMw(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Do stuff here + log.Println(r.RequestURI) + // Call the next handler, which can be another middleware in the chain, or the final handler. + next.ServeHTTP(w, r) + }) + } + +Middlewares can be added to a router using `Router.Use()`: + + r := mux.NewRouter() + r.HandleFunc("/", handler) + r.Use(simpleMw) + +A more complex authentication middleware, which maps session token to users, could be written as: + + // Define our struct + type authenticationMiddleware struct { + tokenUsers map[string]string + } + + // Initialize it somewhere + func (amw *authenticationMiddleware) Populate() { + amw.tokenUsers["00000000"] = "user0" + amw.tokenUsers["aaaaaaaa"] = "userA" + amw.tokenUsers["05f717e5"] = "randomUser" + amw.tokenUsers["deadbeef"] = "user0" + } + + // Middleware function, which will be called for each request + func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("X-Session-Token") + + if user, found := amw.tokenUsers[token]; found { + // We found the token in our map + log.Printf("Authenticated user %s\n", user) + next.ServeHTTP(w, r) + } else { + http.Error(w, "Forbidden", http.StatusForbidden) + } + }) + } + + r := mux.NewRouter() + r.HandleFunc("/", handler) + + amw := authenticationMiddleware{} + amw.Populate() + + r.Use(amw.Middleware) + +Note: The handler chain will be stopped if your middleware doesn't call `next.ServeHTTP()` with the corresponding parameters. This can be used to abort a request if the middleware writer wants to. + +*/ +package mux diff --git a/vendor/github.com/gorilla/mux/example_authentication_middleware_test.go b/vendor/github.com/gorilla/mux/example_authentication_middleware_test.go new file mode 100644 index 000000000..6f2ea86ca --- /dev/null +++ b/vendor/github.com/gorilla/mux/example_authentication_middleware_test.go @@ -0,0 +1,46 @@ +package mux_test + +import ( + "log" + "net/http" + + "github.com/gorilla/mux" +) + +// Define our struct +type authenticationMiddleware struct { + tokenUsers map[string]string +} + +// Initialize it somewhere +func (amw *authenticationMiddleware) Populate() { + amw.tokenUsers["00000000"] = "user0" + amw.tokenUsers["aaaaaaaa"] = "userA" + amw.tokenUsers["05f717e5"] = "randomUser" + amw.tokenUsers["deadbeef"] = "user0" +} + +// Middleware function, which will be called for each request +func (amw *authenticationMiddleware) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("X-Session-Token") + + if user, found := amw.tokenUsers[token]; found { + // We found the token in our map + log.Printf("Authenticated user %s\n", user) + next.ServeHTTP(w, r) + } else { + http.Error(w, "Forbidden", http.StatusForbidden) + } + }) +} + +func Example_authenticationMiddleware() { + r := mux.NewRouter() + r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + // Do something here + }) + amw := authenticationMiddleware{make(map[string]string)} + amw.Populate() + r.Use(amw.Middleware) +} diff --git a/vendor/github.com/gorilla/mux/example_route_test.go b/vendor/github.com/gorilla/mux/example_route_test.go new file mode 100644 index 000000000..112557071 --- /dev/null +++ b/vendor/github.com/gorilla/mux/example_route_test.go @@ -0,0 +1,51 @@ +package mux_test + +import ( + "fmt" + "net/http" + + "github.com/gorilla/mux" +) + +// This example demonstrates setting a regular expression matcher for +// the header value. A plain word will match any value that contains a +// matching substring as if the pattern was wrapped with `.*`. +func ExampleRoute_HeadersRegexp() { + r := mux.NewRouter() + route := r.NewRoute().HeadersRegexp("Accept", "html") + + req1, _ := http.NewRequest("GET", "example.com", nil) + req1.Header.Add("Accept", "text/plain") + req1.Header.Add("Accept", "text/html") + + req2, _ := http.NewRequest("GET", "example.com", nil) + req2.Header.Set("Accept", "application/xhtml+xml") + + matchInfo := &mux.RouteMatch{} + fmt.Printf("Match: %v %q\n", route.Match(req1, matchInfo), req1.Header["Accept"]) + fmt.Printf("Match: %v %q\n", route.Match(req2, matchInfo), req2.Header["Accept"]) + // Output: + // Match: true ["text/plain" "text/html"] + // Match: true ["application/xhtml+xml"] +} + +// This example demonstrates setting a strict regular expression matcher +// for the header value. Using the start and end of string anchors, the +// value must be an exact match. +func ExampleRoute_HeadersRegexp_exactMatch() { + r := mux.NewRouter() + route := r.NewRoute().HeadersRegexp("Origin", "^https://example.co$") + + yes, _ := http.NewRequest("GET", "example.co", nil) + yes.Header.Set("Origin", "https://example.co") + + no, _ := http.NewRequest("GET", "example.co.uk", nil) + no.Header.Set("Origin", "https://example.co.uk") + + matchInfo := &mux.RouteMatch{} + fmt.Printf("Match: %v %q\n", route.Match(yes, matchInfo), yes.Header["Origin"]) + fmt.Printf("Match: %v %q\n", route.Match(no, matchInfo), no.Header["Origin"]) + // Output: + // Match: true ["https://example.co"] + // Match: false ["https://example.co.uk"] +} diff --git a/vendor/github.com/gorilla/mux/go.mod b/vendor/github.com/gorilla/mux/go.mod new file mode 100644 index 000000000..cfc8ede58 --- /dev/null +++ b/vendor/github.com/gorilla/mux/go.mod @@ -0,0 +1 @@ +module github.com/gorilla/mux diff --git a/vendor/github.com/gorilla/mux/middleware.go b/vendor/github.com/gorilla/mux/middleware.go new file mode 100644 index 000000000..ceb812cee --- /dev/null +++ b/vendor/github.com/gorilla/mux/middleware.go @@ -0,0 +1,72 @@ +package mux + +import ( + "net/http" + "strings" +) + +// MiddlewareFunc is a function which receives an http.Handler and returns another http.Handler. +// Typically, the returned handler is a closure which does something with the http.ResponseWriter and http.Request passed +// to it, and then calls the handler passed as parameter to the MiddlewareFunc. +type MiddlewareFunc func(http.Handler) http.Handler + +// middleware interface is anything which implements a MiddlewareFunc named Middleware. +type middleware interface { + Middleware(handler http.Handler) http.Handler +} + +// Middleware allows MiddlewareFunc to implement the middleware interface. +func (mw MiddlewareFunc) Middleware(handler http.Handler) http.Handler { + return mw(handler) +} + +// Use appends a MiddlewareFunc to the chain. Middleware can be used to intercept or otherwise modify requests and/or responses, and are executed in the order that they are applied to the Router. +func (r *Router) Use(mwf ...MiddlewareFunc) { + for _, fn := range mwf { + r.middlewares = append(r.middlewares, fn) + } +} + +// useInterface appends a middleware to the chain. Middleware can be used to intercept or otherwise modify requests and/or responses, and are executed in the order that they are applied to the Router. +func (r *Router) useInterface(mw middleware) { + r.middlewares = append(r.middlewares, mw) +} + +// CORSMethodMiddleware sets the Access-Control-Allow-Methods response header +// on a request, by matching routes based only on paths. It also handles +// OPTIONS requests, by settings Access-Control-Allow-Methods, and then +// returning without calling the next http handler. +func CORSMethodMiddleware(r *Router) MiddlewareFunc { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + var allMethods []string + + err := r.Walk(func(route *Route, _ *Router, _ []*Route) error { + for _, m := range route.matchers { + if _, ok := m.(*routeRegexp); ok { + if m.Match(req, &RouteMatch{}) { + methods, err := route.GetMethods() + if err != nil { + return err + } + + allMethods = append(allMethods, methods...) + } + break + } + } + return nil + }) + + if err == nil { + w.Header().Set("Access-Control-Allow-Methods", strings.Join(append(allMethods, "OPTIONS"), ",")) + + if req.Method == "OPTIONS" { + return + } + } + + next.ServeHTTP(w, req) + }) + } +} diff --git a/vendor/github.com/gorilla/mux/middleware_test.go b/vendor/github.com/gorilla/mux/middleware_test.go new file mode 100644 index 000000000..24016cbb7 --- /dev/null +++ b/vendor/github.com/gorilla/mux/middleware_test.go @@ -0,0 +1,437 @@ +package mux + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" +) + +type testMiddleware struct { + timesCalled uint +} + +func (tm *testMiddleware) Middleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tm.timesCalled++ + h.ServeHTTP(w, r) + }) +} + +func dummyHandler(w http.ResponseWriter, r *http.Request) {} + +func TestMiddlewareAdd(t *testing.T) { + router := NewRouter() + router.HandleFunc("/", dummyHandler).Methods("GET") + + mw := &testMiddleware{} + + router.useInterface(mw) + if len(router.middlewares) != 1 || router.middlewares[0] != mw { + t.Fatal("Middleware was not added correctly") + } + + router.Use(mw.Middleware) + if len(router.middlewares) != 2 { + t.Fatal("MiddlewareFunc method was not added correctly") + } + + banalMw := func(handler http.Handler) http.Handler { + return handler + } + router.Use(banalMw) + if len(router.middlewares) != 3 { + t.Fatal("MiddlewareFunc method was not added correctly") + } +} + +func TestMiddleware(t *testing.T) { + router := NewRouter() + router.HandleFunc("/", dummyHandler).Methods("GET") + + mw := &testMiddleware{} + router.useInterface(mw) + + rw := NewRecorder() + req := newRequest("GET", "/") + + // Test regular middleware call + router.ServeHTTP(rw, req) + if mw.timesCalled != 1 { + t.Fatalf("Expected %d calls, but got only %d", 1, mw.timesCalled) + } + + // Middleware should not be called for 404 + req = newRequest("GET", "/not/found") + router.ServeHTTP(rw, req) + if mw.timesCalled != 1 { + t.Fatalf("Expected %d calls, but got only %d", 1, mw.timesCalled) + } + + // Middleware should not be called if there is a method mismatch + req = newRequest("POST", "/") + router.ServeHTTP(rw, req) + if mw.timesCalled != 1 { + t.Fatalf("Expected %d calls, but got only %d", 1, mw.timesCalled) + } + + // Add the middleware again as function + router.Use(mw.Middleware) + req = newRequest("GET", "/") + router.ServeHTTP(rw, req) + if mw.timesCalled != 3 { + t.Fatalf("Expected %d calls, but got only %d", 3, mw.timesCalled) + } + +} + +func TestMiddlewareSubrouter(t *testing.T) { + router := NewRouter() + router.HandleFunc("/", dummyHandler).Methods("GET") + + subrouter := router.PathPrefix("/sub").Subrouter() + subrouter.HandleFunc("/x", dummyHandler).Methods("GET") + + mw := &testMiddleware{} + subrouter.useInterface(mw) + + rw := NewRecorder() + req := newRequest("GET", "/") + + router.ServeHTTP(rw, req) + if mw.timesCalled != 0 { + t.Fatalf("Expected %d calls, but got only %d", 0, mw.timesCalled) + } + + req = newRequest("GET", "/sub/") + router.ServeHTTP(rw, req) + if mw.timesCalled != 0 { + t.Fatalf("Expected %d calls, but got only %d", 0, mw.timesCalled) + } + + req = newRequest("GET", "/sub/x") + router.ServeHTTP(rw, req) + if mw.timesCalled != 1 { + t.Fatalf("Expected %d calls, but got only %d", 1, mw.timesCalled) + } + + req = newRequest("GET", "/sub/not/found") + router.ServeHTTP(rw, req) + if mw.timesCalled != 1 { + t.Fatalf("Expected %d calls, but got only %d", 1, mw.timesCalled) + } + + router.useInterface(mw) + + req = newRequest("GET", "/") + router.ServeHTTP(rw, req) + if mw.timesCalled != 2 { + t.Fatalf("Expected %d calls, but got only %d", 2, mw.timesCalled) + } + + req = newRequest("GET", "/sub/x") + router.ServeHTTP(rw, req) + if mw.timesCalled != 4 { + t.Fatalf("Expected %d calls, but got only %d", 4, mw.timesCalled) + } +} + +func TestMiddlewareExecution(t *testing.T) { + mwStr := []byte("Middleware\n") + handlerStr := []byte("Logic\n") + + router := NewRouter() + router.HandleFunc("/", func(w http.ResponseWriter, e *http.Request) { + w.Write(handlerStr) + }) + + rw := NewRecorder() + req := newRequest("GET", "/") + + // Test handler-only call + router.ServeHTTP(rw, req) + + if !bytes.Equal(rw.Body.Bytes(), handlerStr) { + t.Fatal("Handler response is not what it should be") + } + + // Test middleware call + rw = NewRecorder() + + router.Use(func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(mwStr) + h.ServeHTTP(w, r) + }) + }) + + router.ServeHTTP(rw, req) + if !bytes.Equal(rw.Body.Bytes(), append(mwStr, handlerStr...)) { + t.Fatal("Middleware + handler response is not what it should be") + } +} + +func TestMiddlewareNotFound(t *testing.T) { + mwStr := []byte("Middleware\n") + handlerStr := []byte("Logic\n") + + router := NewRouter() + router.HandleFunc("/", func(w http.ResponseWriter, e *http.Request) { + w.Write(handlerStr) + }) + router.Use(func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(mwStr) + h.ServeHTTP(w, r) + }) + }) + + // Test not found call with default handler + rw := NewRecorder() + req := newRequest("GET", "/notfound") + + router.ServeHTTP(rw, req) + if bytes.Contains(rw.Body.Bytes(), mwStr) { + t.Fatal("Middleware was called for a 404") + } + + // Test not found call with custom handler + rw = NewRecorder() + req = newRequest("GET", "/notfound") + + router.NotFoundHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + rw.Write([]byte("Custom 404 handler")) + }) + router.ServeHTTP(rw, req) + + if bytes.Contains(rw.Body.Bytes(), mwStr) { + t.Fatal("Middleware was called for a custom 404") + } +} + +func TestMiddlewareMethodMismatch(t *testing.T) { + mwStr := []byte("Middleware\n") + handlerStr := []byte("Logic\n") + + router := NewRouter() + router.HandleFunc("/", func(w http.ResponseWriter, e *http.Request) { + w.Write(handlerStr) + }).Methods("GET") + + router.Use(func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(mwStr) + h.ServeHTTP(w, r) + }) + }) + + // Test method mismatch + rw := NewRecorder() + req := newRequest("POST", "/") + + router.ServeHTTP(rw, req) + if bytes.Contains(rw.Body.Bytes(), mwStr) { + t.Fatal("Middleware was called for a method mismatch") + } + + // Test not found call + rw = NewRecorder() + req = newRequest("POST", "/") + + router.MethodNotAllowedHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + rw.Write([]byte("Method not allowed")) + }) + router.ServeHTTP(rw, req) + + if bytes.Contains(rw.Body.Bytes(), mwStr) { + t.Fatal("Middleware was called for a method mismatch") + } +} + +func TestMiddlewareNotFoundSubrouter(t *testing.T) { + mwStr := []byte("Middleware\n") + handlerStr := []byte("Logic\n") + + router := NewRouter() + router.HandleFunc("/", func(w http.ResponseWriter, e *http.Request) { + w.Write(handlerStr) + }) + + subrouter := router.PathPrefix("/sub/").Subrouter() + subrouter.HandleFunc("/", func(w http.ResponseWriter, e *http.Request) { + w.Write(handlerStr) + }) + + router.Use(func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(mwStr) + h.ServeHTTP(w, r) + }) + }) + + // Test not found call for default handler + rw := NewRecorder() + req := newRequest("GET", "/sub/notfound") + + router.ServeHTTP(rw, req) + if bytes.Contains(rw.Body.Bytes(), mwStr) { + t.Fatal("Middleware was called for a 404") + } + + // Test not found call with custom handler + rw = NewRecorder() + req = newRequest("GET", "/sub/notfound") + + subrouter.NotFoundHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + rw.Write([]byte("Custom 404 handler")) + }) + router.ServeHTTP(rw, req) + + if bytes.Contains(rw.Body.Bytes(), mwStr) { + t.Fatal("Middleware was called for a custom 404") + } +} + +func TestMiddlewareMethodMismatchSubrouter(t *testing.T) { + mwStr := []byte("Middleware\n") + handlerStr := []byte("Logic\n") + + router := NewRouter() + router.HandleFunc("/", func(w http.ResponseWriter, e *http.Request) { + w.Write(handlerStr) + }) + + subrouter := router.PathPrefix("/sub/").Subrouter() + subrouter.HandleFunc("/", func(w http.ResponseWriter, e *http.Request) { + w.Write(handlerStr) + }).Methods("GET") + + router.Use(func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(mwStr) + h.ServeHTTP(w, r) + }) + }) + + // Test method mismatch without custom handler + rw := NewRecorder() + req := newRequest("POST", "/sub/") + + router.ServeHTTP(rw, req) + if bytes.Contains(rw.Body.Bytes(), mwStr) { + t.Fatal("Middleware was called for a method mismatch") + } + + // Test method mismatch with custom handler + rw = NewRecorder() + req = newRequest("POST", "/sub/") + + router.MethodNotAllowedHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + rw.Write([]byte("Method not allowed")) + }) + router.ServeHTTP(rw, req) + + if bytes.Contains(rw.Body.Bytes(), mwStr) { + t.Fatal("Middleware was called for a method mismatch") + } +} + +func TestCORSMethodMiddleware(t *testing.T) { + router := NewRouter() + + cases := []struct { + path string + response string + method string + testURL string + expectedAllowedMethods string + }{ + {"/g/{o}", "a", "POST", "/g/asdf", "POST,PUT,GET,OPTIONS"}, + {"/g/{o}", "b", "PUT", "/g/bla", "POST,PUT,GET,OPTIONS"}, + {"/g/{o}", "c", "GET", "/g/orilla", "POST,PUT,GET,OPTIONS"}, + {"/g", "d", "POST", "/g", "POST,OPTIONS"}, + } + + for _, tt := range cases { + router.HandleFunc(tt.path, stringHandler(tt.response)).Methods(tt.method) + } + + router.Use(CORSMethodMiddleware(router)) + + for _, tt := range cases { + rr := httptest.NewRecorder() + req := newRequest(tt.method, tt.testURL) + + router.ServeHTTP(rr, req) + + if rr.Body.String() != tt.response { + t.Errorf("Expected body '%s', found '%s'", tt.response, rr.Body.String()) + } + + allowedMethods := rr.Header().Get("Access-Control-Allow-Methods") + + if allowedMethods != tt.expectedAllowedMethods { + t.Errorf("Expected Access-Control-Allow-Methods '%s', found '%s'", tt.expectedAllowedMethods, allowedMethods) + } + } +} + +func TestMiddlewareOnMultiSubrouter(t *testing.T) { + first := "first" + second := "second" + notFound := "404 not found" + + router := NewRouter() + firstSubRouter := router.PathPrefix("/").Subrouter() + secondSubRouter := router.PathPrefix("/").Subrouter() + + router.NotFoundHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + rw.Write([]byte(notFound)) + }) + + firstSubRouter.HandleFunc("/first", func(w http.ResponseWriter, r *http.Request) { + + }) + + secondSubRouter.HandleFunc("/second", func(w http.ResponseWriter, r *http.Request) { + + }) + + firstSubRouter.Use(func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(first)) + h.ServeHTTP(w, r) + }) + }) + + secondSubRouter.Use(func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(second)) + h.ServeHTTP(w, r) + }) + }) + + rw := NewRecorder() + req := newRequest("GET", "/first") + + router.ServeHTTP(rw, req) + if rw.Body.String() != first { + t.Fatalf("Middleware did not run: expected %s middleware to write a response (got %s)", first, rw.Body.String()) + } + + rw = NewRecorder() + req = newRequest("GET", "/second") + + router.ServeHTTP(rw, req) + if rw.Body.String() != second { + t.Fatalf("Middleware did not run: expected %s middleware to write a response (got %s)", second, rw.Body.String()) + } + + rw = NewRecorder() + req = newRequest("GET", "/second/not-exist") + + router.ServeHTTP(rw, req) + if rw.Body.String() != notFound { + t.Fatalf("Notfound handler did not run: expected %s for not-exist, (got %s)", notFound, rw.Body.String()) + } +} diff --git a/vendor/github.com/gorilla/mux/mux.go b/vendor/github.com/gorilla/mux/mux.go new file mode 100644 index 000000000..a2cd193e4 --- /dev/null +++ b/vendor/github.com/gorilla/mux/mux.go @@ -0,0 +1,607 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import ( + "errors" + "fmt" + "net/http" + "path" + "regexp" +) + +var ( + // ErrMethodMismatch is returned when the method in the request does not match + // the method defined against the route. + ErrMethodMismatch = errors.New("method is not allowed") + // ErrNotFound is returned when no route match is found. + ErrNotFound = errors.New("no matching route was found") +) + +// NewRouter returns a new router instance. +func NewRouter() *Router { + return &Router{namedRoutes: make(map[string]*Route)} +} + +// Router registers routes to be matched and dispatches a handler. +// +// It implements the http.Handler interface, so it can be registered to serve +// requests: +// +// var router = mux.NewRouter() +// +// func main() { +// http.Handle("/", router) +// } +// +// Or, for Google App Engine, register it in a init() function: +// +// func init() { +// http.Handle("/", router) +// } +// +// This will send all incoming requests to the router. +type Router struct { + // Configurable Handler to be used when no route matches. + NotFoundHandler http.Handler + + // Configurable Handler to be used when the request method does not match the route. + MethodNotAllowedHandler http.Handler + + // Routes to be matched, in order. + routes []*Route + + // Routes by name for URL building. + namedRoutes map[string]*Route + + // If true, do not clear the request context after handling the request. + // + // Deprecated: No effect when go1.7+ is used, since the context is stored + // on the request itself. + KeepContext bool + + // Slice of middlewares to be called after a match is found + middlewares []middleware + + // configuration shared with `Route` + routeConf +} + +// common route configuration shared between `Router` and `Route` +type routeConf struct { + // If true, "/path/foo%2Fbar/to" will match the path "/path/{var}/to" + useEncodedPath bool + + // If true, when the path pattern is "/path/", accessing "/path" will + // redirect to the former and vice versa. + strictSlash bool + + // If true, when the path pattern is "/path//to", accessing "/path//to" + // will not redirect + skipClean bool + + // Manager for the variables from host and path. + regexp routeRegexpGroup + + // List of matchers. + matchers []matcher + + // The scheme used when building URLs. + buildScheme string + + buildVarsFunc BuildVarsFunc +} + +// returns an effective deep copy of `routeConf` +func copyRouteConf(r routeConf) routeConf { + c := r + + if r.regexp.path != nil { + c.regexp.path = copyRouteRegexp(r.regexp.path) + } + + if r.regexp.host != nil { + c.regexp.host = copyRouteRegexp(r.regexp.host) + } + + c.regexp.queries = make([]*routeRegexp, 0, len(r.regexp.queries)) + for _, q := range r.regexp.queries { + c.regexp.queries = append(c.regexp.queries, copyRouteRegexp(q)) + } + + c.matchers = make([]matcher, 0, len(r.matchers)) + for _, m := range r.matchers { + c.matchers = append(c.matchers, m) + } + + return c +} + +func copyRouteRegexp(r *routeRegexp) *routeRegexp { + c := *r + return &c +} + +// Match attempts to match the given request against the router's registered routes. +// +// If the request matches a route of this router or one of its subrouters the Route, +// Handler, and Vars fields of the the match argument are filled and this function +// returns true. +// +// If the request does not match any of this router's or its subrouters' routes +// then this function returns false. If available, a reason for the match failure +// will be filled in the match argument's MatchErr field. If the match failure type +// (eg: not found) has a registered handler, the handler is assigned to the Handler +// field of the match argument. +func (r *Router) Match(req *http.Request, match *RouteMatch) bool { + for _, route := range r.routes { + if route.Match(req, match) { + // Build middleware chain if no error was found + if match.MatchErr == nil { + for i := len(r.middlewares) - 1; i >= 0; i-- { + match.Handler = r.middlewares[i].Middleware(match.Handler) + } + } + return true + } + } + + if match.MatchErr == ErrMethodMismatch { + if r.MethodNotAllowedHandler != nil { + match.Handler = r.MethodNotAllowedHandler + return true + } + + return false + } + + // Closest match for a router (includes sub-routers) + if r.NotFoundHandler != nil { + match.Handler = r.NotFoundHandler + match.MatchErr = ErrNotFound + return true + } + + match.MatchErr = ErrNotFound + return false +} + +// ServeHTTP dispatches the handler registered in the matched route. +// +// When there is a match, the route variables can be retrieved calling +// mux.Vars(request). +func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if !r.skipClean { + path := req.URL.Path + if r.useEncodedPath { + path = req.URL.EscapedPath() + } + // Clean path to canonical form and redirect. + if p := cleanPath(path); p != path { + + // Added 3 lines (Philip Schlump) - It was dropping the query string and #whatever from query. + // This matches with fix in go 1.2 r.c. 4 for same problem. Go Issue: + // http://code.google.com/p/go/issues/detail?id=5252 + url := *req.URL + url.Path = p + p = url.String() + + w.Header().Set("Location", p) + w.WriteHeader(http.StatusMovedPermanently) + return + } + } + var match RouteMatch + var handler http.Handler + if r.Match(req, &match) { + handler = match.Handler + req = setVars(req, match.Vars) + req = setCurrentRoute(req, match.Route) + } + + if handler == nil && match.MatchErr == ErrMethodMismatch { + handler = methodNotAllowedHandler() + } + + if handler == nil { + handler = http.NotFoundHandler() + } + + handler.ServeHTTP(w, req) +} + +// Get returns a route registered with the given name. +func (r *Router) Get(name string) *Route { + return r.namedRoutes[name] +} + +// GetRoute returns a route registered with the given name. This method +// was renamed to Get() and remains here for backwards compatibility. +func (r *Router) GetRoute(name string) *Route { + return r.namedRoutes[name] +} + +// StrictSlash defines the trailing slash behavior for new routes. The initial +// value is false. +// +// When true, if the route path is "/path/", accessing "/path" will perform a redirect +// to the former and vice versa. In other words, your application will always +// see the path as specified in the route. +// +// When false, if the route path is "/path", accessing "/path/" will not match +// this route and vice versa. +// +// The re-direct is a HTTP 301 (Moved Permanently). Note that when this is set for +// routes with a non-idempotent method (e.g. POST, PUT), the subsequent re-directed +// request will be made as a GET by most clients. Use middleware or client settings +// to modify this behaviour as needed. +// +// Special case: when a route sets a path prefix using the PathPrefix() method, +// strict slash is ignored for that route because the redirect behavior can't +// be determined from a prefix alone. However, any subrouters created from that +// route inherit the original StrictSlash setting. +func (r *Router) StrictSlash(value bool) *Router { + r.strictSlash = value + return r +} + +// SkipClean defines the path cleaning behaviour for new routes. The initial +// value is false. Users should be careful about which routes are not cleaned +// +// When true, if the route path is "/path//to", it will remain with the double +// slash. This is helpful if you have a route like: /fetch/http://xkcd.com/534/ +// +// When false, the path will be cleaned, so /fetch/http://xkcd.com/534/ will +// become /fetch/http/xkcd.com/534 +func (r *Router) SkipClean(value bool) *Router { + r.skipClean = value + return r +} + +// UseEncodedPath tells the router to match the encoded original path +// to the routes. +// For eg. "/path/foo%2Fbar/to" will match the path "/path/{var}/to". +// +// If not called, the router will match the unencoded path to the routes. +// For eg. "/path/foo%2Fbar/to" will match the path "/path/foo/bar/to" +func (r *Router) UseEncodedPath() *Router { + r.useEncodedPath = true + return r +} + +// ---------------------------------------------------------------------------- +// Route factories +// ---------------------------------------------------------------------------- + +// NewRoute registers an empty route. +func (r *Router) NewRoute() *Route { + // initialize a route with a copy of the parent router's configuration + route := &Route{routeConf: copyRouteConf(r.routeConf), namedRoutes: r.namedRoutes} + r.routes = append(r.routes, route) + return route +} + +// Name registers a new route with a name. +// See Route.Name(). +func (r *Router) Name(name string) *Route { + return r.NewRoute().Name(name) +} + +// Handle registers a new route with a matcher for the URL path. +// See Route.Path() and Route.Handler(). +func (r *Router) Handle(path string, handler http.Handler) *Route { + return r.NewRoute().Path(path).Handler(handler) +} + +// HandleFunc registers a new route with a matcher for the URL path. +// See Route.Path() and Route.HandlerFunc(). +func (r *Router) HandleFunc(path string, f func(http.ResponseWriter, + *http.Request)) *Route { + return r.NewRoute().Path(path).HandlerFunc(f) +} + +// Headers registers a new route with a matcher for request header values. +// See Route.Headers(). +func (r *Router) Headers(pairs ...string) *Route { + return r.NewRoute().Headers(pairs...) +} + +// Host registers a new route with a matcher for the URL host. +// See Route.Host(). +func (r *Router) Host(tpl string) *Route { + return r.NewRoute().Host(tpl) +} + +// MatcherFunc registers a new route with a custom matcher function. +// See Route.MatcherFunc(). +func (r *Router) MatcherFunc(f MatcherFunc) *Route { + return r.NewRoute().MatcherFunc(f) +} + +// Methods registers a new route with a matcher for HTTP methods. +// See Route.Methods(). +func (r *Router) Methods(methods ...string) *Route { + return r.NewRoute().Methods(methods...) +} + +// Path registers a new route with a matcher for the URL path. +// See Route.Path(). +func (r *Router) Path(tpl string) *Route { + return r.NewRoute().Path(tpl) +} + +// PathPrefix registers a new route with a matcher for the URL path prefix. +// See Route.PathPrefix(). +func (r *Router) PathPrefix(tpl string) *Route { + return r.NewRoute().PathPrefix(tpl) +} + +// Queries registers a new route with a matcher for URL query values. +// See Route.Queries(). +func (r *Router) Queries(pairs ...string) *Route { + return r.NewRoute().Queries(pairs...) +} + +// Schemes registers a new route with a matcher for URL schemes. +// See Route.Schemes(). +func (r *Router) Schemes(schemes ...string) *Route { + return r.NewRoute().Schemes(schemes...) +} + +// BuildVarsFunc registers a new route with a custom function for modifying +// route variables before building a URL. +func (r *Router) BuildVarsFunc(f BuildVarsFunc) *Route { + return r.NewRoute().BuildVarsFunc(f) +} + +// Walk walks the router and all its sub-routers, calling walkFn for each route +// in the tree. The routes are walked in the order they were added. Sub-routers +// are explored depth-first. +func (r *Router) Walk(walkFn WalkFunc) error { + return r.walk(walkFn, []*Route{}) +} + +// SkipRouter is used as a return value from WalkFuncs to indicate that the +// router that walk is about to descend down to should be skipped. +var SkipRouter = errors.New("skip this router") + +// WalkFunc is the type of the function called for each route visited by Walk. +// At every invocation, it is given the current route, and the current router, +// and a list of ancestor routes that lead to the current route. +type WalkFunc func(route *Route, router *Router, ancestors []*Route) error + +func (r *Router) walk(walkFn WalkFunc, ancestors []*Route) error { + for _, t := range r.routes { + err := walkFn(t, r, ancestors) + if err == SkipRouter { + continue + } + if err != nil { + return err + } + for _, sr := range t.matchers { + if h, ok := sr.(*Router); ok { + ancestors = append(ancestors, t) + err := h.walk(walkFn, ancestors) + if err != nil { + return err + } + ancestors = ancestors[:len(ancestors)-1] + } + } + if h, ok := t.handler.(*Router); ok { + ancestors = append(ancestors, t) + err := h.walk(walkFn, ancestors) + if err != nil { + return err + } + ancestors = ancestors[:len(ancestors)-1] + } + } + return nil +} + +// ---------------------------------------------------------------------------- +// Context +// ---------------------------------------------------------------------------- + +// RouteMatch stores information about a matched route. +type RouteMatch struct { + Route *Route + Handler http.Handler + Vars map[string]string + + // MatchErr is set to appropriate matching error + // It is set to ErrMethodMismatch if there is a mismatch in + // the request method and route method + MatchErr error +} + +type contextKey int + +const ( + varsKey contextKey = iota + routeKey +) + +// Vars returns the route variables for the current request, if any. +func Vars(r *http.Request) map[string]string { + if rv := contextGet(r, varsKey); rv != nil { + return rv.(map[string]string) + } + return nil +} + +// CurrentRoute returns the matched route for the current request, if any. +// This only works when called inside the handler of the matched route +// because the matched route is stored in the request context which is cleared +// after the handler returns, unless the KeepContext option is set on the +// Router. +func CurrentRoute(r *http.Request) *Route { + if rv := contextGet(r, routeKey); rv != nil { + return rv.(*Route) + } + return nil +} + +func setVars(r *http.Request, val interface{}) *http.Request { + return contextSet(r, varsKey, val) +} + +func setCurrentRoute(r *http.Request, val interface{}) *http.Request { + return contextSet(r, routeKey, val) +} + +// ---------------------------------------------------------------------------- +// Helpers +// ---------------------------------------------------------------------------- + +// cleanPath returns the canonical path for p, eliminating . and .. elements. +// Borrowed from the net/http package. +func cleanPath(p string) string { + if p == "" { + return "/" + } + if p[0] != '/' { + p = "/" + p + } + np := path.Clean(p) + // path.Clean removes trailing slash except for root; + // put the trailing slash back if necessary. + if p[len(p)-1] == '/' && np != "/" { + np += "/" + } + + return np +} + +// uniqueVars returns an error if two slices contain duplicated strings. +func uniqueVars(s1, s2 []string) error { + for _, v1 := range s1 { + for _, v2 := range s2 { + if v1 == v2 { + return fmt.Errorf("mux: duplicated route variable %q", v2) + } + } + } + return nil +} + +// checkPairs returns the count of strings passed in, and an error if +// the count is not an even number. +func checkPairs(pairs ...string) (int, error) { + length := len(pairs) + if length%2 != 0 { + return length, fmt.Errorf( + "mux: number of parameters must be multiple of 2, got %v", pairs) + } + return length, nil +} + +// mapFromPairsToString converts variadic string parameters to a +// string to string map. +func mapFromPairsToString(pairs ...string) (map[string]string, error) { + length, err := checkPairs(pairs...) + if err != nil { + return nil, err + } + m := make(map[string]string, length/2) + for i := 0; i < length; i += 2 { + m[pairs[i]] = pairs[i+1] + } + return m, nil +} + +// mapFromPairsToRegex converts variadic string parameters to a +// string to regex map. +func mapFromPairsToRegex(pairs ...string) (map[string]*regexp.Regexp, error) { + length, err := checkPairs(pairs...) + if err != nil { + return nil, err + } + m := make(map[string]*regexp.Regexp, length/2) + for i := 0; i < length; i += 2 { + regex, err := regexp.Compile(pairs[i+1]) + if err != nil { + return nil, err + } + m[pairs[i]] = regex + } + return m, nil +} + +// matchInArray returns true if the given string value is in the array. +func matchInArray(arr []string, value string) bool { + for _, v := range arr { + if v == value { + return true + } + } + return false +} + +// matchMapWithString returns true if the given key/value pairs exist in a given map. +func matchMapWithString(toCheck map[string]string, toMatch map[string][]string, canonicalKey bool) bool { + for k, v := range toCheck { + // Check if key exists. + if canonicalKey { + k = http.CanonicalHeaderKey(k) + } + if values := toMatch[k]; values == nil { + return false + } else if v != "" { + // If value was defined as an empty string we only check that the + // key exists. Otherwise we also check for equality. + valueExists := false + for _, value := range values { + if v == value { + valueExists = true + break + } + } + if !valueExists { + return false + } + } + } + return true +} + +// matchMapWithRegex returns true if the given key/value pairs exist in a given map compiled against +// the given regex +func matchMapWithRegex(toCheck map[string]*regexp.Regexp, toMatch map[string][]string, canonicalKey bool) bool { + for k, v := range toCheck { + // Check if key exists. + if canonicalKey { + k = http.CanonicalHeaderKey(k) + } + if values := toMatch[k]; values == nil { + return false + } else if v != nil { + // If value was defined as an empty string we only check that the + // key exists. Otherwise we also check for equality. + valueExists := false + for _, value := range values { + if v.MatchString(value) { + valueExists = true + break + } + } + if !valueExists { + return false + } + } + } + return true +} + +// methodNotAllowed replies to the request with an HTTP status code 405. +func methodNotAllowed(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusMethodNotAllowed) +} + +// methodNotAllowedHandler returns a simple request handler +// that replies to each request with a status code 405. +func methodNotAllowedHandler() http.Handler { return http.HandlerFunc(methodNotAllowed) } diff --git a/vendor/github.com/gorilla/mux/mux_test.go b/vendor/github.com/gorilla/mux/mux_test.go new file mode 100644 index 000000000..f5c1e9c5b --- /dev/null +++ b/vendor/github.com/gorilla/mux/mux_test.go @@ -0,0 +1,2846 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "net/http" + "net/url" + "reflect" + "strings" + "testing" +) + +func (r *Route) GoString() string { + matchers := make([]string, len(r.matchers)) + for i, m := range r.matchers { + matchers[i] = fmt.Sprintf("%#v", m) + } + return fmt.Sprintf("&Route{matchers:[]matcher{%s}}", strings.Join(matchers, ", ")) +} + +func (r *routeRegexp) GoString() string { + return fmt.Sprintf("&routeRegexp{template: %q, regexpType: %v, options: %v, regexp: regexp.MustCompile(%q), reverse: %q, varsN: %v, varsR: %v", r.template, r.regexpType, r.options, r.regexp.String(), r.reverse, r.varsN, r.varsR) +} + +type routeTest struct { + title string // title of the test + route *Route // the route being tested + request *http.Request // a request to test the route + vars map[string]string // the expected vars of the match + scheme string // the expected scheme of the built URL + host string // the expected host of the built URL + path string // the expected path of the built URL + query string // the expected query string of the built URL + pathTemplate string // the expected path template of the route + hostTemplate string // the expected host template of the route + queriesTemplate string // the expected query template of the route + methods []string // the expected route methods + pathRegexp string // the expected path regexp + queriesRegexp string // the expected query regexp + shouldMatch bool // whether the request is expected to match the route at all + shouldRedirect bool // whether the request should result in a redirect +} + +func TestHost(t *testing.T) { + + tests := []routeTest{ + { + title: "Host route match", + route: new(Route).Host("aaa.bbb.ccc"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{}, + host: "aaa.bbb.ccc", + path: "", + shouldMatch: true, + }, + { + title: "Host route, wrong host in request URL", + route: new(Route).Host("aaa.bbb.ccc"), + request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), + vars: map[string]string{}, + host: "aaa.bbb.ccc", + path: "", + shouldMatch: false, + }, + { + title: "Host route with port, match", + route: new(Route).Host("aaa.bbb.ccc:1234"), + request: newRequest("GET", "http://aaa.bbb.ccc:1234/111/222/333"), + vars: map[string]string{}, + host: "aaa.bbb.ccc:1234", + path: "", + shouldMatch: true, + }, + { + title: "Host route with port, wrong port in request URL", + route: new(Route).Host("aaa.bbb.ccc:1234"), + request: newRequest("GET", "http://aaa.bbb.ccc:9999/111/222/333"), + vars: map[string]string{}, + host: "aaa.bbb.ccc:1234", + path: "", + shouldMatch: false, + }, + { + title: "Host route, match with host in request header", + route: new(Route).Host("aaa.bbb.ccc"), + request: newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc"), + vars: map[string]string{}, + host: "aaa.bbb.ccc", + path: "", + shouldMatch: true, + }, + { + title: "Host route, wrong host in request header", + route: new(Route).Host("aaa.bbb.ccc"), + request: newRequestHost("GET", "/111/222/333", "aaa.222.ccc"), + vars: map[string]string{}, + host: "aaa.bbb.ccc", + path: "", + shouldMatch: false, + }, + { + title: "Host route with port, match with request header", + route: new(Route).Host("aaa.bbb.ccc:1234"), + request: newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc:1234"), + vars: map[string]string{}, + host: "aaa.bbb.ccc:1234", + path: "", + shouldMatch: true, + }, + { + title: "Host route with port, wrong host in request header", + route: new(Route).Host("aaa.bbb.ccc:1234"), + request: newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc:9999"), + vars: map[string]string{}, + host: "aaa.bbb.ccc:1234", + path: "", + shouldMatch: false, + }, + { + title: "Host route with pattern, match with request header", + route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc:1{v2:(?:23|4)}"), + request: newRequestHost("GET", "/111/222/333", "aaa.bbb.ccc:123"), + vars: map[string]string{"v1": "bbb", "v2": "23"}, + host: "aaa.bbb.ccc:123", + path: "", + hostTemplate: `aaa.{v1:[a-z]{3}}.ccc:1{v2:(?:23|4)}`, + shouldMatch: true, + }, + { + title: "Host route with pattern, match", + route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{"v1": "bbb"}, + host: "aaa.bbb.ccc", + path: "", + hostTemplate: `aaa.{v1:[a-z]{3}}.ccc`, + shouldMatch: true, + }, + { + title: "Host route with pattern, additional capturing group, match", + route: new(Route).Host("aaa.{v1:[a-z]{2}(?:b|c)}.ccc"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{"v1": "bbb"}, + host: "aaa.bbb.ccc", + path: "", + hostTemplate: `aaa.{v1:[a-z]{2}(?:b|c)}.ccc`, + shouldMatch: true, + }, + { + title: "Host route with pattern, wrong host in request URL", + route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc"), + request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), + vars: map[string]string{"v1": "bbb"}, + host: "aaa.bbb.ccc", + path: "", + hostTemplate: `aaa.{v1:[a-z]{3}}.ccc`, + shouldMatch: false, + }, + { + title: "Host route with multiple patterns, match", + route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc"}, + host: "aaa.bbb.ccc", + path: "", + hostTemplate: `{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}`, + shouldMatch: true, + }, + { + title: "Host route with multiple patterns, wrong host in request URL", + route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}"), + request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), + vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc"}, + host: "aaa.bbb.ccc", + path: "", + hostTemplate: `{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}`, + shouldMatch: false, + }, + { + title: "Host route with hyphenated name and pattern, match", + route: new(Route).Host("aaa.{v-1:[a-z]{3}}.ccc"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{"v-1": "bbb"}, + host: "aaa.bbb.ccc", + path: "", + hostTemplate: `aaa.{v-1:[a-z]{3}}.ccc`, + shouldMatch: true, + }, + { + title: "Host route with hyphenated name and pattern, additional capturing group, match", + route: new(Route).Host("aaa.{v-1:[a-z]{2}(?:b|c)}.ccc"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{"v-1": "bbb"}, + host: "aaa.bbb.ccc", + path: "", + hostTemplate: `aaa.{v-1:[a-z]{2}(?:b|c)}.ccc`, + shouldMatch: true, + }, + { + title: "Host route with multiple hyphenated names and patterns, match", + route: new(Route).Host("{v-1:[a-z]{3}}.{v-2:[a-z]{3}}.{v-3:[a-z]{3}}"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{"v-1": "aaa", "v-2": "bbb", "v-3": "ccc"}, + host: "aaa.bbb.ccc", + path: "", + hostTemplate: `{v-1:[a-z]{3}}.{v-2:[a-z]{3}}.{v-3:[a-z]{3}}`, + shouldMatch: true, + }, + } + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + }) + } +} + +func TestPath(t *testing.T) { + tests := []routeTest{ + { + title: "Path route, match", + route: new(Route).Path("/111/222/333"), + request: newRequest("GET", "http://localhost/111/222/333"), + vars: map[string]string{}, + host: "", + path: "/111/222/333", + shouldMatch: true, + }, + { + title: "Path route, match with trailing slash in request and path", + route: new(Route).Path("/111/"), + request: newRequest("GET", "http://localhost/111/"), + vars: map[string]string{}, + host: "", + path: "/111/", + shouldMatch: true, + }, + { + title: "Path route, do not match with trailing slash in path", + route: new(Route).Path("/111/"), + request: newRequest("GET", "http://localhost/111"), + vars: map[string]string{}, + host: "", + path: "/111", + pathTemplate: `/111/`, + pathRegexp: `^/111/$`, + shouldMatch: false, + }, + { + title: "Path route, do not match with trailing slash in request", + route: new(Route).Path("/111"), + request: newRequest("GET", "http://localhost/111/"), + vars: map[string]string{}, + host: "", + path: "/111/", + pathTemplate: `/111`, + shouldMatch: false, + }, + { + title: "Path route, match root with no host", + route: new(Route).Path("/"), + request: newRequest("GET", "/"), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + pathRegexp: `^/$`, + shouldMatch: true, + }, + { + title: "Path route, match root with no host, App Engine format", + route: new(Route).Path("/"), + request: func() *http.Request { + r := newRequest("GET", "http://localhost/") + r.RequestURI = "/" + return r + }(), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + shouldMatch: true, + }, + { + title: "Path route, wrong path in request in request URL", + route: new(Route).Path("/111/222/333"), + request: newRequest("GET", "http://localhost/1/2/3"), + vars: map[string]string{}, + host: "", + path: "/111/222/333", + shouldMatch: false, + }, + { + title: "Path route with pattern, match", + route: new(Route).Path("/111/{v1:[0-9]{3}}/333"), + request: newRequest("GET", "http://localhost/111/222/333"), + vars: map[string]string{"v1": "222"}, + host: "", + path: "/111/222/333", + pathTemplate: `/111/{v1:[0-9]{3}}/333`, + shouldMatch: true, + }, + { + title: "Path route with pattern, URL in request does not match", + route: new(Route).Path("/111/{v1:[0-9]{3}}/333"), + request: newRequest("GET", "http://localhost/111/aaa/333"), + vars: map[string]string{"v1": "222"}, + host: "", + path: "/111/222/333", + pathTemplate: `/111/{v1:[0-9]{3}}/333`, + pathRegexp: `^/111/(?P[0-9]{3})/333$`, + shouldMatch: false, + }, + { + title: "Path route with multiple patterns, match", + route: new(Route).Path("/{v1:[0-9]{3}}/{v2:[0-9]{3}}/{v3:[0-9]{3}}"), + request: newRequest("GET", "http://localhost/111/222/333"), + vars: map[string]string{"v1": "111", "v2": "222", "v3": "333"}, + host: "", + path: "/111/222/333", + pathTemplate: `/{v1:[0-9]{3}}/{v2:[0-9]{3}}/{v3:[0-9]{3}}`, + pathRegexp: `^/(?P[0-9]{3})/(?P[0-9]{3})/(?P[0-9]{3})$`, + shouldMatch: true, + }, + { + title: "Path route with multiple patterns, URL in request does not match", + route: new(Route).Path("/{v1:[0-9]{3}}/{v2:[0-9]{3}}/{v3:[0-9]{3}}"), + request: newRequest("GET", "http://localhost/111/aaa/333"), + vars: map[string]string{"v1": "111", "v2": "222", "v3": "333"}, + host: "", + path: "/111/222/333", + pathTemplate: `/{v1:[0-9]{3}}/{v2:[0-9]{3}}/{v3:[0-9]{3}}`, + pathRegexp: `^/(?P[0-9]{3})/(?P[0-9]{3})/(?P[0-9]{3})$`, + shouldMatch: false, + }, + { + title: "Path route with multiple patterns with pipe, match", + route: new(Route).Path("/{category:a|(?:b/c)}/{product}/{id:[0-9]+}"), + request: newRequest("GET", "http://localhost/a/product_name/1"), + vars: map[string]string{"category": "a", "product": "product_name", "id": "1"}, + host: "", + path: "/a/product_name/1", + pathTemplate: `/{category:a|(?:b/c)}/{product}/{id:[0-9]+}`, + pathRegexp: `^/(?Pa|(?:b/c))/(?P[^/]+)/(?P[0-9]+)$`, + shouldMatch: true, + }, + { + title: "Path route with hyphenated name and pattern, match", + route: new(Route).Path("/111/{v-1:[0-9]{3}}/333"), + request: newRequest("GET", "http://localhost/111/222/333"), + vars: map[string]string{"v-1": "222"}, + host: "", + path: "/111/222/333", + pathTemplate: `/111/{v-1:[0-9]{3}}/333`, + pathRegexp: `^/111/(?P[0-9]{3})/333$`, + shouldMatch: true, + }, + { + title: "Path route with multiple hyphenated names and patterns, match", + route: new(Route).Path("/{v-1:[0-9]{3}}/{v-2:[0-9]{3}}/{v-3:[0-9]{3}}"), + request: newRequest("GET", "http://localhost/111/222/333"), + vars: map[string]string{"v-1": "111", "v-2": "222", "v-3": "333"}, + host: "", + path: "/111/222/333", + pathTemplate: `/{v-1:[0-9]{3}}/{v-2:[0-9]{3}}/{v-3:[0-9]{3}}`, + pathRegexp: `^/(?P[0-9]{3})/(?P[0-9]{3})/(?P[0-9]{3})$`, + shouldMatch: true, + }, + { + title: "Path route with multiple hyphenated names and patterns with pipe, match", + route: new(Route).Path("/{product-category:a|(?:b/c)}/{product-name}/{product-id:[0-9]+}"), + request: newRequest("GET", "http://localhost/a/product_name/1"), + vars: map[string]string{"product-category": "a", "product-name": "product_name", "product-id": "1"}, + host: "", + path: "/a/product_name/1", + pathTemplate: `/{product-category:a|(?:b/c)}/{product-name}/{product-id:[0-9]+}`, + pathRegexp: `^/(?Pa|(?:b/c))/(?P[^/]+)/(?P[0-9]+)$`, + shouldMatch: true, + }, + { + title: "Path route with multiple hyphenated names and patterns with pipe and case insensitive, match", + route: new(Route).Path("/{type:(?i:daily|mini|variety)}-{date:\\d{4,4}-\\d{2,2}-\\d{2,2}}"), + request: newRequest("GET", "http://localhost/daily-2016-01-01"), + vars: map[string]string{"type": "daily", "date": "2016-01-01"}, + host: "", + path: "/daily-2016-01-01", + pathTemplate: `/{type:(?i:daily|mini|variety)}-{date:\d{4,4}-\d{2,2}-\d{2,2}}`, + pathRegexp: `^/(?P(?i:daily|mini|variety))-(?P\d{4,4}-\d{2,2}-\d{2,2})$`, + shouldMatch: true, + }, + { + title: "Path route with empty match right after other match", + route: new(Route).Path(`/{v1:[0-9]*}{v2:[a-z]*}/{v3:[0-9]*}`), + request: newRequest("GET", "http://localhost/111/222"), + vars: map[string]string{"v1": "111", "v2": "", "v3": "222"}, + host: "", + path: "/111/222", + pathTemplate: `/{v1:[0-9]*}{v2:[a-z]*}/{v3:[0-9]*}`, + pathRegexp: `^/(?P[0-9]*)(?P[a-z]*)/(?P[0-9]*)$`, + shouldMatch: true, + }, + { + title: "Path route with single pattern with pipe, match", + route: new(Route).Path("/{category:a|b/c}"), + request: newRequest("GET", "http://localhost/a"), + vars: map[string]string{"category": "a"}, + host: "", + path: "/a", + pathTemplate: `/{category:a|b/c}`, + shouldMatch: true, + }, + { + title: "Path route with single pattern with pipe, match", + route: new(Route).Path("/{category:a|b/c}"), + request: newRequest("GET", "http://localhost/b/c"), + vars: map[string]string{"category": "b/c"}, + host: "", + path: "/b/c", + pathTemplate: `/{category:a|b/c}`, + shouldMatch: true, + }, + { + title: "Path route with multiple patterns with pipe, match", + route: new(Route).Path("/{category:a|b/c}/{product}/{id:[0-9]+}"), + request: newRequest("GET", "http://localhost/a/product_name/1"), + vars: map[string]string{"category": "a", "product": "product_name", "id": "1"}, + host: "", + path: "/a/product_name/1", + pathTemplate: `/{category:a|b/c}/{product}/{id:[0-9]+}`, + shouldMatch: true, + }, + { + title: "Path route with multiple patterns with pipe, match", + route: new(Route).Path("/{category:a|b/c}/{product}/{id:[0-9]+}"), + request: newRequest("GET", "http://localhost/b/c/product_name/1"), + vars: map[string]string{"category": "b/c", "product": "product_name", "id": "1"}, + host: "", + path: "/b/c/product_name/1", + pathTemplate: `/{category:a|b/c}/{product}/{id:[0-9]+}`, + shouldMatch: true, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + testUseEscapedRoute(t, test) + testRegexp(t, test) + }) + } +} + +func TestPathPrefix(t *testing.T) { + tests := []routeTest{ + { + title: "PathPrefix route, match", + route: new(Route).PathPrefix("/111"), + request: newRequest("GET", "http://localhost/111/222/333"), + vars: map[string]string{}, + host: "", + path: "/111", + shouldMatch: true, + }, + { + title: "PathPrefix route, match substring", + route: new(Route).PathPrefix("/1"), + request: newRequest("GET", "http://localhost/111/222/333"), + vars: map[string]string{}, + host: "", + path: "/1", + shouldMatch: true, + }, + { + title: "PathPrefix route, URL prefix in request does not match", + route: new(Route).PathPrefix("/111"), + request: newRequest("GET", "http://localhost/1/2/3"), + vars: map[string]string{}, + host: "", + path: "/111", + shouldMatch: false, + }, + { + title: "PathPrefix route with pattern, match", + route: new(Route).PathPrefix("/111/{v1:[0-9]{3}}"), + request: newRequest("GET", "http://localhost/111/222/333"), + vars: map[string]string{"v1": "222"}, + host: "", + path: "/111/222", + pathTemplate: `/111/{v1:[0-9]{3}}`, + shouldMatch: true, + }, + { + title: "PathPrefix route with pattern, URL prefix in request does not match", + route: new(Route).PathPrefix("/111/{v1:[0-9]{3}}"), + request: newRequest("GET", "http://localhost/111/aaa/333"), + vars: map[string]string{"v1": "222"}, + host: "", + path: "/111/222", + pathTemplate: `/111/{v1:[0-9]{3}}`, + shouldMatch: false, + }, + { + title: "PathPrefix route with multiple patterns, match", + route: new(Route).PathPrefix("/{v1:[0-9]{3}}/{v2:[0-9]{3}}"), + request: newRequest("GET", "http://localhost/111/222/333"), + vars: map[string]string{"v1": "111", "v2": "222"}, + host: "", + path: "/111/222", + pathTemplate: `/{v1:[0-9]{3}}/{v2:[0-9]{3}}`, + shouldMatch: true, + }, + { + title: "PathPrefix route with multiple patterns, URL prefix in request does not match", + route: new(Route).PathPrefix("/{v1:[0-9]{3}}/{v2:[0-9]{3}}"), + request: newRequest("GET", "http://localhost/111/aaa/333"), + vars: map[string]string{"v1": "111", "v2": "222"}, + host: "", + path: "/111/222", + pathTemplate: `/{v1:[0-9]{3}}/{v2:[0-9]{3}}`, + shouldMatch: false, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + testUseEscapedRoute(t, test) + }) + } +} + +func TestSchemeHostPath(t *testing.T) { + tests := []routeTest{ + { + title: "Host and Path route, match", + route: new(Route).Host("aaa.bbb.ccc").Path("/111/222/333"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{}, + scheme: "http", + host: "aaa.bbb.ccc", + path: "/111/222/333", + pathTemplate: `/111/222/333`, + hostTemplate: `aaa.bbb.ccc`, + shouldMatch: true, + }, + { + title: "Scheme, Host, and Path route, match", + route: new(Route).Schemes("https").Host("aaa.bbb.ccc").Path("/111/222/333"), + request: newRequest("GET", "https://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{}, + scheme: "https", + host: "aaa.bbb.ccc", + path: "/111/222/333", + pathTemplate: `/111/222/333`, + hostTemplate: `aaa.bbb.ccc`, + shouldMatch: true, + }, + { + title: "Host and Path route, wrong host in request URL", + route: new(Route).Host("aaa.bbb.ccc").Path("/111/222/333"), + request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), + vars: map[string]string{}, + scheme: "http", + host: "aaa.bbb.ccc", + path: "/111/222/333", + pathTemplate: `/111/222/333`, + hostTemplate: `aaa.bbb.ccc`, + shouldMatch: false, + }, + { + title: "Host and Path route with pattern, match", + route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc").Path("/111/{v2:[0-9]{3}}/333"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{"v1": "bbb", "v2": "222"}, + scheme: "http", + host: "aaa.bbb.ccc", + path: "/111/222/333", + pathTemplate: `/111/{v2:[0-9]{3}}/333`, + hostTemplate: `aaa.{v1:[a-z]{3}}.ccc`, + shouldMatch: true, + }, + { + title: "Scheme, Host, and Path route with host and path patterns, match", + route: new(Route).Schemes("ftp", "ssss").Host("aaa.{v1:[a-z]{3}}.ccc").Path("/111/{v2:[0-9]{3}}/333"), + request: newRequest("GET", "ssss://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{"v1": "bbb", "v2": "222"}, + scheme: "ftp", + host: "aaa.bbb.ccc", + path: "/111/222/333", + pathTemplate: `/111/{v2:[0-9]{3}}/333`, + hostTemplate: `aaa.{v1:[a-z]{3}}.ccc`, + shouldMatch: true, + }, + { + title: "Host and Path route with pattern, URL in request does not match", + route: new(Route).Host("aaa.{v1:[a-z]{3}}.ccc").Path("/111/{v2:[0-9]{3}}/333"), + request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), + vars: map[string]string{"v1": "bbb", "v2": "222"}, + scheme: "http", + host: "aaa.bbb.ccc", + path: "/111/222/333", + pathTemplate: `/111/{v2:[0-9]{3}}/333`, + hostTemplate: `aaa.{v1:[a-z]{3}}.ccc`, + shouldMatch: false, + }, + { + title: "Host and Path route with multiple patterns, match", + route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}").Path("/{v4:[0-9]{3}}/{v5:[0-9]{3}}/{v6:[0-9]{3}}"), + request: newRequest("GET", "http://aaa.bbb.ccc/111/222/333"), + vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc", "v4": "111", "v5": "222", "v6": "333"}, + scheme: "http", + host: "aaa.bbb.ccc", + path: "/111/222/333", + pathTemplate: `/{v4:[0-9]{3}}/{v5:[0-9]{3}}/{v6:[0-9]{3}}`, + hostTemplate: `{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}`, + shouldMatch: true, + }, + { + title: "Host and Path route with multiple patterns, URL in request does not match", + route: new(Route).Host("{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}").Path("/{v4:[0-9]{3}}/{v5:[0-9]{3}}/{v6:[0-9]{3}}"), + request: newRequest("GET", "http://aaa.222.ccc/111/222/333"), + vars: map[string]string{"v1": "aaa", "v2": "bbb", "v3": "ccc", "v4": "111", "v5": "222", "v6": "333"}, + scheme: "http", + host: "aaa.bbb.ccc", + path: "/111/222/333", + pathTemplate: `/{v4:[0-9]{3}}/{v5:[0-9]{3}}/{v6:[0-9]{3}}`, + hostTemplate: `{v1:[a-z]{3}}.{v2:[a-z]{3}}.{v3:[a-z]{3}}`, + shouldMatch: false, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + testUseEscapedRoute(t, test) + }) + } +} + +func TestHeaders(t *testing.T) { + // newRequestHeaders creates a new request with a method, url, and headers + newRequestHeaders := func(method, url string, headers map[string]string) *http.Request { + req, err := http.NewRequest(method, url, nil) + if err != nil { + panic(err) + } + for k, v := range headers { + req.Header.Add(k, v) + } + return req + } + + tests := []routeTest{ + { + title: "Headers route, match", + route: new(Route).Headers("foo", "bar", "baz", "ding"), + request: newRequestHeaders("GET", "http://localhost", map[string]string{"foo": "bar", "baz": "ding"}), + vars: map[string]string{}, + host: "", + path: "", + shouldMatch: true, + }, + { + title: "Headers route, bad header values", + route: new(Route).Headers("foo", "bar", "baz", "ding"), + request: newRequestHeaders("GET", "http://localhost", map[string]string{"foo": "bar", "baz": "dong"}), + vars: map[string]string{}, + host: "", + path: "", + shouldMatch: false, + }, + { + title: "Headers route, regex header values to match", + route: new(Route).Headers("foo", "ba[zr]"), + request: newRequestHeaders("GET", "http://localhost", map[string]string{"foo": "bar"}), + vars: map[string]string{}, + host: "", + path: "", + shouldMatch: false, + }, + { + title: "Headers route, regex header values to match", + route: new(Route).HeadersRegexp("foo", "ba[zr]"), + request: newRequestHeaders("GET", "http://localhost", map[string]string{"foo": "baz"}), + vars: map[string]string{}, + host: "", + path: "", + shouldMatch: true, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + }) + } +} + +func TestMethods(t *testing.T) { + tests := []routeTest{ + { + title: "Methods route, match GET", + route: new(Route).Methods("GET", "POST"), + request: newRequest("GET", "http://localhost"), + vars: map[string]string{}, + host: "", + path: "", + methods: []string{"GET", "POST"}, + shouldMatch: true, + }, + { + title: "Methods route, match POST", + route: new(Route).Methods("GET", "POST"), + request: newRequest("POST", "http://localhost"), + vars: map[string]string{}, + host: "", + path: "", + methods: []string{"GET", "POST"}, + shouldMatch: true, + }, + { + title: "Methods route, bad method", + route: new(Route).Methods("GET", "POST"), + request: newRequest("PUT", "http://localhost"), + vars: map[string]string{}, + host: "", + path: "", + methods: []string{"GET", "POST"}, + shouldMatch: false, + }, + { + title: "Route without methods", + route: new(Route), + request: newRequest("PUT", "http://localhost"), + vars: map[string]string{}, + host: "", + path: "", + methods: []string{}, + shouldMatch: true, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + testMethods(t, test) + }) + } +} + +func TestQueries(t *testing.T) { + tests := []routeTest{ + { + title: "Queries route, match", + route: new(Route).Queries("foo", "bar", "baz", "ding"), + request: newRequest("GET", "http://localhost?foo=bar&baz=ding"), + vars: map[string]string{}, + host: "", + path: "", + query: "foo=bar&baz=ding", + queriesTemplate: "foo=bar,baz=ding", + queriesRegexp: "^foo=bar$,^baz=ding$", + shouldMatch: true, + }, + { + title: "Queries route, match with a query string", + route: new(Route).Host("www.example.com").Path("/api").Queries("foo", "bar", "baz", "ding"), + request: newRequest("GET", "http://www.example.com/api?foo=bar&baz=ding"), + vars: map[string]string{}, + host: "", + path: "", + query: "foo=bar&baz=ding", + pathTemplate: `/api`, + hostTemplate: `www.example.com`, + queriesTemplate: "foo=bar,baz=ding", + queriesRegexp: "^foo=bar$,^baz=ding$", + shouldMatch: true, + }, + { + title: "Queries route, match with a query string out of order", + route: new(Route).Host("www.example.com").Path("/api").Queries("foo", "bar", "baz", "ding"), + request: newRequest("GET", "http://www.example.com/api?baz=ding&foo=bar"), + vars: map[string]string{}, + host: "", + path: "", + query: "foo=bar&baz=ding", + pathTemplate: `/api`, + hostTemplate: `www.example.com`, + queriesTemplate: "foo=bar,baz=ding", + queriesRegexp: "^foo=bar$,^baz=ding$", + shouldMatch: true, + }, + { + title: "Queries route, bad query", + route: new(Route).Queries("foo", "bar", "baz", "ding"), + request: newRequest("GET", "http://localhost?foo=bar&baz=dong"), + vars: map[string]string{}, + host: "", + path: "", + queriesTemplate: "foo=bar,baz=ding", + queriesRegexp: "^foo=bar$,^baz=ding$", + shouldMatch: false, + }, + { + title: "Queries route with pattern, match", + route: new(Route).Queries("foo", "{v1}"), + request: newRequest("GET", "http://localhost?foo=bar"), + vars: map[string]string{"v1": "bar"}, + host: "", + path: "", + query: "foo=bar", + queriesTemplate: "foo={v1}", + queriesRegexp: "^foo=(?P.*)$", + shouldMatch: true, + }, + { + title: "Queries route with multiple patterns, match", + route: new(Route).Queries("foo", "{v1}", "baz", "{v2}"), + request: newRequest("GET", "http://localhost?foo=bar&baz=ding"), + vars: map[string]string{"v1": "bar", "v2": "ding"}, + host: "", + path: "", + query: "foo=bar&baz=ding", + queriesTemplate: "foo={v1},baz={v2}", + queriesRegexp: "^foo=(?P.*)$,^baz=(?P.*)$", + shouldMatch: true, + }, + { + title: "Queries route with regexp pattern, match", + route: new(Route).Queries("foo", "{v1:[0-9]+}"), + request: newRequest("GET", "http://localhost?foo=10"), + vars: map[string]string{"v1": "10"}, + host: "", + path: "", + query: "foo=10", + queriesTemplate: "foo={v1:[0-9]+}", + queriesRegexp: "^foo=(?P[0-9]+)$", + shouldMatch: true, + }, + { + title: "Queries route with regexp pattern, regexp does not match", + route: new(Route).Queries("foo", "{v1:[0-9]+}"), + request: newRequest("GET", "http://localhost?foo=a"), + vars: map[string]string{}, + host: "", + path: "", + queriesTemplate: "foo={v1:[0-9]+}", + queriesRegexp: "^foo=(?P[0-9]+)$", + shouldMatch: false, + }, + { + title: "Queries route with regexp pattern with quantifier, match", + route: new(Route).Queries("foo", "{v1:[0-9]{1}}"), + request: newRequest("GET", "http://localhost?foo=1"), + vars: map[string]string{"v1": "1"}, + host: "", + path: "", + query: "foo=1", + queriesTemplate: "foo={v1:[0-9]{1}}", + queriesRegexp: "^foo=(?P[0-9]{1})$", + shouldMatch: true, + }, + { + title: "Queries route with regexp pattern with quantifier, additional variable in query string, match", + route: new(Route).Queries("foo", "{v1:[0-9]{1}}"), + request: newRequest("GET", "http://localhost?bar=2&foo=1"), + vars: map[string]string{"v1": "1"}, + host: "", + path: "", + query: "foo=1", + queriesTemplate: "foo={v1:[0-9]{1}}", + queriesRegexp: "^foo=(?P[0-9]{1})$", + shouldMatch: true, + }, + { + title: "Queries route with regexp pattern with quantifier, regexp does not match", + route: new(Route).Queries("foo", "{v1:[0-9]{1}}"), + request: newRequest("GET", "http://localhost?foo=12"), + vars: map[string]string{}, + host: "", + path: "", + queriesTemplate: "foo={v1:[0-9]{1}}", + queriesRegexp: "^foo=(?P[0-9]{1})$", + shouldMatch: false, + }, + { + title: "Queries route with regexp pattern with quantifier, additional capturing group", + route: new(Route).Queries("foo", "{v1:[0-9]{1}(?:a|b)}"), + request: newRequest("GET", "http://localhost?foo=1a"), + vars: map[string]string{"v1": "1a"}, + host: "", + path: "", + query: "foo=1a", + queriesTemplate: "foo={v1:[0-9]{1}(?:a|b)}", + queriesRegexp: "^foo=(?P[0-9]{1}(?:a|b))$", + shouldMatch: true, + }, + { + title: "Queries route with regexp pattern with quantifier, additional variable in query string, regexp does not match", + route: new(Route).Queries("foo", "{v1:[0-9]{1}}"), + request: newRequest("GET", "http://localhost?foo=12"), + vars: map[string]string{}, + host: "", + path: "", + queriesTemplate: "foo={v1:[0-9]{1}}", + queriesRegexp: "^foo=(?P[0-9]{1})$", + shouldMatch: false, + }, + { + title: "Queries route with hyphenated name, match", + route: new(Route).Queries("foo", "{v-1}"), + request: newRequest("GET", "http://localhost?foo=bar"), + vars: map[string]string{"v-1": "bar"}, + host: "", + path: "", + query: "foo=bar", + queriesTemplate: "foo={v-1}", + queriesRegexp: "^foo=(?P.*)$", + shouldMatch: true, + }, + { + title: "Queries route with multiple hyphenated names, match", + route: new(Route).Queries("foo", "{v-1}", "baz", "{v-2}"), + request: newRequest("GET", "http://localhost?foo=bar&baz=ding"), + vars: map[string]string{"v-1": "bar", "v-2": "ding"}, + host: "", + path: "", + query: "foo=bar&baz=ding", + queriesTemplate: "foo={v-1},baz={v-2}", + queriesRegexp: "^foo=(?P.*)$,^baz=(?P.*)$", + shouldMatch: true, + }, + { + title: "Queries route with hyphenate name and pattern, match", + route: new(Route).Queries("foo", "{v-1:[0-9]+}"), + request: newRequest("GET", "http://localhost?foo=10"), + vars: map[string]string{"v-1": "10"}, + host: "", + path: "", + query: "foo=10", + queriesTemplate: "foo={v-1:[0-9]+}", + queriesRegexp: "^foo=(?P[0-9]+)$", + shouldMatch: true, + }, + { + title: "Queries route with hyphenated name and pattern with quantifier, additional capturing group", + route: new(Route).Queries("foo", "{v-1:[0-9]{1}(?:a|b)}"), + request: newRequest("GET", "http://localhost?foo=1a"), + vars: map[string]string{"v-1": "1a"}, + host: "", + path: "", + query: "foo=1a", + queriesTemplate: "foo={v-1:[0-9]{1}(?:a|b)}", + queriesRegexp: "^foo=(?P[0-9]{1}(?:a|b))$", + shouldMatch: true, + }, + { + title: "Queries route with empty value, should match", + route: new(Route).Queries("foo", ""), + request: newRequest("GET", "http://localhost?foo=bar"), + vars: map[string]string{}, + host: "", + path: "", + query: "foo=", + queriesTemplate: "foo=", + queriesRegexp: "^foo=.*$", + shouldMatch: true, + }, + { + title: "Queries route with empty value and no parameter in request, should not match", + route: new(Route).Queries("foo", ""), + request: newRequest("GET", "http://localhost"), + vars: map[string]string{}, + host: "", + path: "", + queriesTemplate: "foo=", + queriesRegexp: "^foo=.*$", + shouldMatch: false, + }, + { + title: "Queries route with empty value and empty parameter in request, should match", + route: new(Route).Queries("foo", ""), + request: newRequest("GET", "http://localhost?foo="), + vars: map[string]string{}, + host: "", + path: "", + query: "foo=", + queriesTemplate: "foo=", + queriesRegexp: "^foo=.*$", + shouldMatch: true, + }, + { + title: "Queries route with overlapping value, should not match", + route: new(Route).Queries("foo", "bar"), + request: newRequest("GET", "http://localhost?foo=barfoo"), + vars: map[string]string{}, + host: "", + path: "", + queriesTemplate: "foo=bar", + queriesRegexp: "^foo=bar$", + shouldMatch: false, + }, + { + title: "Queries route with no parameter in request, should not match", + route: new(Route).Queries("foo", "{bar}"), + request: newRequest("GET", "http://localhost"), + vars: map[string]string{}, + host: "", + path: "", + queriesTemplate: "foo={bar}", + queriesRegexp: "^foo=(?P.*)$", + shouldMatch: false, + }, + { + title: "Queries route with empty parameter in request, should match", + route: new(Route).Queries("foo", "{bar}"), + request: newRequest("GET", "http://localhost?foo="), + vars: map[string]string{"foo": ""}, + host: "", + path: "", + query: "foo=", + queriesTemplate: "foo={bar}", + queriesRegexp: "^foo=(?P.*)$", + shouldMatch: true, + }, + { + title: "Queries route, bad submatch", + route: new(Route).Queries("foo", "bar", "baz", "ding"), + request: newRequest("GET", "http://localhost?fffoo=bar&baz=dingggg"), + vars: map[string]string{}, + host: "", + path: "", + queriesTemplate: "foo=bar,baz=ding", + queriesRegexp: "^foo=bar$,^baz=ding$", + shouldMatch: false, + }, + { + title: "Queries route with pattern, match, escaped value", + route: new(Route).Queries("foo", "{v1}"), + request: newRequest("GET", "http://localhost?foo=%25bar%26%20%2F%3D%3F"), + vars: map[string]string{"v1": "%bar& /=?"}, + host: "", + path: "", + query: "foo=%25bar%26+%2F%3D%3F", + queriesTemplate: "foo={v1}", + queriesRegexp: "^foo=(?P.*)$", + shouldMatch: true, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testTemplate(t, test) + testQueriesTemplates(t, test) + testUseEscapedRoute(t, test) + testQueriesRegexp(t, test) + }) + } +} + +func TestSchemes(t *testing.T) { + tests := []routeTest{ + // Schemes + { + title: "Schemes route, default scheme, match http, build http", + route: new(Route).Host("localhost"), + request: newRequest("GET", "http://localhost"), + scheme: "http", + host: "localhost", + shouldMatch: true, + }, + { + title: "Schemes route, match https, build https", + route: new(Route).Schemes("https", "ftp").Host("localhost"), + request: newRequest("GET", "https://localhost"), + scheme: "https", + host: "localhost", + shouldMatch: true, + }, + { + title: "Schemes route, match ftp, build https", + route: new(Route).Schemes("https", "ftp").Host("localhost"), + request: newRequest("GET", "ftp://localhost"), + scheme: "https", + host: "localhost", + shouldMatch: true, + }, + { + title: "Schemes route, match ftp, build ftp", + route: new(Route).Schemes("ftp", "https").Host("localhost"), + request: newRequest("GET", "ftp://localhost"), + scheme: "ftp", + host: "localhost", + shouldMatch: true, + }, + { + title: "Schemes route, bad scheme", + route: new(Route).Schemes("https", "ftp").Host("localhost"), + request: newRequest("GET", "http://localhost"), + scheme: "https", + host: "localhost", + shouldMatch: false, + }, + } + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + }) + } +} + +func TestMatcherFunc(t *testing.T) { + m := func(r *http.Request, m *RouteMatch) bool { + return r.URL.Host == "aaa.bbb.ccc" + } + + tests := []routeTest{ + { + title: "MatchFunc route, match", + route: new(Route).MatcherFunc(m), + request: newRequest("GET", "http://aaa.bbb.ccc"), + vars: map[string]string{}, + host: "", + path: "", + shouldMatch: true, + }, + { + title: "MatchFunc route, non-match", + route: new(Route).MatcherFunc(m), + request: newRequest("GET", "http://aaa.222.ccc"), + vars: map[string]string{}, + host: "", + path: "", + shouldMatch: false, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + }) + } +} + +func TestBuildVarsFunc(t *testing.T) { + tests := []routeTest{ + { + title: "BuildVarsFunc set on route", + route: new(Route).Path(`/111/{v1:\d}{v2:.*}`).BuildVarsFunc(func(vars map[string]string) map[string]string { + vars["v1"] = "3" + vars["v2"] = "a" + return vars + }), + request: newRequest("GET", "http://localhost/111/2"), + path: "/111/3a", + pathTemplate: `/111/{v1:\d}{v2:.*}`, + shouldMatch: true, + }, + { + title: "BuildVarsFunc set on route and parent route", + route: new(Route).PathPrefix(`/{v1:\d}`).BuildVarsFunc(func(vars map[string]string) map[string]string { + vars["v1"] = "2" + return vars + }).Subrouter().Path(`/{v2:\w}`).BuildVarsFunc(func(vars map[string]string) map[string]string { + vars["v2"] = "b" + return vars + }), + request: newRequest("GET", "http://localhost/1/a"), + path: "/2/b", + pathTemplate: `/{v1:\d}/{v2:\w}`, + shouldMatch: true, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + }) + } +} + +func TestSubRouter(t *testing.T) { + subrouter1 := new(Route).Host("{v1:[a-z]+}.google.com").Subrouter() + subrouter2 := new(Route).PathPrefix("/foo/{v1}").Subrouter() + subrouter3 := new(Route).PathPrefix("/foo").Subrouter() + subrouter4 := new(Route).PathPrefix("/foo/bar").Subrouter() + subrouter5 := new(Route).PathPrefix("/{category}").Subrouter() + tests := []routeTest{ + { + route: subrouter1.Path("/{v2:[a-z]+}"), + request: newRequest("GET", "http://aaa.google.com/bbb"), + vars: map[string]string{"v1": "aaa", "v2": "bbb"}, + host: "aaa.google.com", + path: "/bbb", + pathTemplate: `/{v2:[a-z]+}`, + hostTemplate: `{v1:[a-z]+}.google.com`, + shouldMatch: true, + }, + { + route: subrouter1.Path("/{v2:[a-z]+}"), + request: newRequest("GET", "http://111.google.com/111"), + vars: map[string]string{"v1": "aaa", "v2": "bbb"}, + host: "aaa.google.com", + path: "/bbb", + pathTemplate: `/{v2:[a-z]+}`, + hostTemplate: `{v1:[a-z]+}.google.com`, + shouldMatch: false, + }, + { + route: subrouter2.Path("/baz/{v2}"), + request: newRequest("GET", "http://localhost/foo/bar/baz/ding"), + vars: map[string]string{"v1": "bar", "v2": "ding"}, + host: "", + path: "/foo/bar/baz/ding", + pathTemplate: `/foo/{v1}/baz/{v2}`, + shouldMatch: true, + }, + { + route: subrouter2.Path("/baz/{v2}"), + request: newRequest("GET", "http://localhost/foo/bar"), + vars: map[string]string{"v1": "bar", "v2": "ding"}, + host: "", + path: "/foo/bar/baz/ding", + pathTemplate: `/foo/{v1}/baz/{v2}`, + shouldMatch: false, + }, + { + route: subrouter3.Path("/"), + request: newRequest("GET", "http://localhost/foo/"), + vars: map[string]string{}, + host: "", + path: "/foo/", + pathTemplate: `/foo/`, + shouldMatch: true, + }, + { + route: subrouter3.Path(""), + request: newRequest("GET", "http://localhost/foo"), + vars: map[string]string{}, + host: "", + path: "/foo", + pathTemplate: `/foo`, + shouldMatch: true, + }, + + { + route: subrouter4.Path("/"), + request: newRequest("GET", "http://localhost/foo/bar/"), + vars: map[string]string{}, + host: "", + path: "/foo/bar/", + pathTemplate: `/foo/bar/`, + shouldMatch: true, + }, + { + route: subrouter4.Path(""), + request: newRequest("GET", "http://localhost/foo/bar"), + vars: map[string]string{}, + host: "", + path: "/foo/bar", + pathTemplate: `/foo/bar`, + shouldMatch: true, + }, + { + route: subrouter5.Path("/"), + request: newRequest("GET", "http://localhost/baz/"), + vars: map[string]string{"category": "baz"}, + host: "", + path: "/baz/", + pathTemplate: `/{category}/`, + shouldMatch: true, + }, + { + route: subrouter5.Path(""), + request: newRequest("GET", "http://localhost/baz"), + vars: map[string]string{"category": "baz"}, + host: "", + path: "/baz", + pathTemplate: `/{category}`, + shouldMatch: true, + }, + { + title: "Mismatch method specified on parent route", + route: new(Route).Methods("POST").PathPrefix("/foo").Subrouter().Path("/"), + request: newRequest("GET", "http://localhost/foo/"), + vars: map[string]string{}, + host: "", + path: "/foo/", + pathTemplate: `/foo/`, + shouldMatch: false, + }, + { + title: "Match method specified on parent route", + route: new(Route).Methods("POST").PathPrefix("/foo").Subrouter().Path("/"), + request: newRequest("POST", "http://localhost/foo/"), + vars: map[string]string{}, + host: "", + path: "/foo/", + pathTemplate: `/foo/`, + shouldMatch: true, + }, + { + title: "Mismatch scheme specified on parent route", + route: new(Route).Schemes("https").Subrouter().PathPrefix("/"), + request: newRequest("GET", "http://localhost/"), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + shouldMatch: false, + }, + { + title: "Match scheme specified on parent route", + route: new(Route).Schemes("http").Subrouter().PathPrefix("/"), + request: newRequest("GET", "http://localhost/"), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + shouldMatch: true, + }, + { + title: "No match header specified on parent route", + route: new(Route).Headers("X-Forwarded-Proto", "https").Subrouter().PathPrefix("/"), + request: newRequest("GET", "http://localhost/"), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + shouldMatch: false, + }, + { + title: "Header mismatch value specified on parent route", + route: new(Route).Headers("X-Forwarded-Proto", "https").Subrouter().PathPrefix("/"), + request: newRequestWithHeaders("GET", "http://localhost/", "X-Forwarded-Proto", "http"), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + shouldMatch: false, + }, + { + title: "Header match value specified on parent route", + route: new(Route).Headers("X-Forwarded-Proto", "https").Subrouter().PathPrefix("/"), + request: newRequestWithHeaders("GET", "http://localhost/", "X-Forwarded-Proto", "https"), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + shouldMatch: true, + }, + { + title: "Query specified on parent route not present", + route: new(Route).Headers("key", "foobar").Subrouter().PathPrefix("/"), + request: newRequest("GET", "http://localhost/"), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + shouldMatch: false, + }, + { + title: "Query mismatch value specified on parent route", + route: new(Route).Queries("key", "foobar").Subrouter().PathPrefix("/"), + request: newRequest("GET", "http://localhost/?key=notfoobar"), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + shouldMatch: false, + }, + { + title: "Query match value specified on subroute", + route: new(Route).Queries("key", "foobar").Subrouter().PathPrefix("/"), + request: newRequest("GET", "http://localhost/?key=foobar"), + vars: map[string]string{}, + host: "", + path: "/", + pathTemplate: `/`, + shouldMatch: true, + }, + { + title: "Build with scheme on parent router", + route: new(Route).Schemes("ftp").Host("google.com").Subrouter().Path("/"), + request: newRequest("GET", "ftp://google.com/"), + scheme: "ftp", + host: "google.com", + path: "/", + pathTemplate: `/`, + hostTemplate: `google.com`, + shouldMatch: true, + }, + { + title: "Prefer scheme on child route when building URLs", + route: new(Route).Schemes("https", "ftp").Host("google.com").Subrouter().Schemes("ftp").Path("/"), + request: newRequest("GET", "ftp://google.com/"), + scheme: "ftp", + host: "google.com", + path: "/", + pathTemplate: `/`, + hostTemplate: `google.com`, + shouldMatch: true, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + testUseEscapedRoute(t, test) + }) + } +} + +func TestNamedRoutes(t *testing.T) { + r1 := NewRouter() + r1.NewRoute().Name("a") + r1.NewRoute().Name("b") + r1.NewRoute().Name("c") + + r2 := r1.NewRoute().Subrouter() + r2.NewRoute().Name("d") + r2.NewRoute().Name("e") + r2.NewRoute().Name("f") + + r3 := r2.NewRoute().Subrouter() + r3.NewRoute().Name("g") + r3.NewRoute().Name("h") + r3.NewRoute().Name("i") + r3.Name("j") + + if r1.namedRoutes == nil || len(r1.namedRoutes) != 10 { + t.Errorf("Expected 10 named routes, got %v", r1.namedRoutes) + } else if r1.Get("j") == nil { + t.Errorf("Subroute name not registered") + } +} + +func TestNameMultipleCalls(t *testing.T) { + r1 := NewRouter() + rt := r1.NewRoute().Name("foo").Name("bar") + err := rt.GetError() + if err == nil { + t.Errorf("Expected an error") + } +} + +func TestStrictSlash(t *testing.T) { + r := NewRouter() + r.StrictSlash(true) + + tests := []routeTest{ + { + title: "Redirect path without slash", + route: r.NewRoute().Path("/111/"), + request: newRequest("GET", "http://localhost/111"), + vars: map[string]string{}, + host: "", + path: "/111/", + shouldMatch: true, + shouldRedirect: true, + }, + { + title: "Do not redirect path with slash", + route: r.NewRoute().Path("/111/"), + request: newRequest("GET", "http://localhost/111/"), + vars: map[string]string{}, + host: "", + path: "/111/", + shouldMatch: true, + shouldRedirect: false, + }, + { + title: "Redirect path with slash", + route: r.NewRoute().Path("/111"), + request: newRequest("GET", "http://localhost/111/"), + vars: map[string]string{}, + host: "", + path: "/111", + shouldMatch: true, + shouldRedirect: true, + }, + { + title: "Do not redirect path without slash", + route: r.NewRoute().Path("/111"), + request: newRequest("GET", "http://localhost/111"), + vars: map[string]string{}, + host: "", + path: "/111", + shouldMatch: true, + shouldRedirect: false, + }, + { + title: "Propagate StrictSlash to subrouters", + route: r.NewRoute().PathPrefix("/static/").Subrouter().Path("/images/"), + request: newRequest("GET", "http://localhost/static/images"), + vars: map[string]string{}, + host: "", + path: "/static/images/", + shouldMatch: true, + shouldRedirect: true, + }, + { + title: "Ignore StrictSlash for path prefix", + route: r.NewRoute().PathPrefix("/static/"), + request: newRequest("GET", "http://localhost/static/logo.png"), + vars: map[string]string{}, + host: "", + path: "/static/", + shouldMatch: true, + shouldRedirect: false, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + testUseEscapedRoute(t, test) + }) + } +} + +func TestUseEncodedPath(t *testing.T) { + r := NewRouter() + r.UseEncodedPath() + + tests := []routeTest{ + { + title: "Router with useEncodedPath, URL with encoded slash does match", + route: r.NewRoute().Path("/v1/{v1}/v2"), + request: newRequest("GET", "http://localhost/v1/1%2F2/v2"), + vars: map[string]string{"v1": "1%2F2"}, + host: "", + path: "/v1/1%2F2/v2", + pathTemplate: `/v1/{v1}/v2`, + shouldMatch: true, + }, + { + title: "Router with useEncodedPath, URL with encoded slash doesn't match", + route: r.NewRoute().Path("/v1/1/2/v2"), + request: newRequest("GET", "http://localhost/v1/1%2F2/v2"), + vars: map[string]string{"v1": "1%2F2"}, + host: "", + path: "/v1/1%2F2/v2", + pathTemplate: `/v1/1/2/v2`, + shouldMatch: false, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testRoute(t, test) + testTemplate(t, test) + }) + } +} + +func TestWalkSingleDepth(t *testing.T) { + r0 := NewRouter() + r1 := NewRouter() + r2 := NewRouter() + + r0.Path("/g") + r0.Path("/o") + r0.Path("/d").Handler(r1) + r0.Path("/r").Handler(r2) + r0.Path("/a") + + r1.Path("/z") + r1.Path("/i") + r1.Path("/l") + r1.Path("/l") + + r2.Path("/i") + r2.Path("/l") + r2.Path("/l") + + paths := []string{"g", "o", "r", "i", "l", "l", "a"} + depths := []int{0, 0, 0, 1, 1, 1, 0} + i := 0 + err := r0.Walk(func(route *Route, router *Router, ancestors []*Route) error { + matcher := route.matchers[0].(*routeRegexp) + if matcher.template == "/d" { + return SkipRouter + } + if len(ancestors) != depths[i] { + t.Errorf(`Expected depth of %d at i = %d; got "%d"`, depths[i], i, len(ancestors)) + } + if matcher.template != "/"+paths[i] { + t.Errorf(`Expected "/%s" at i = %d; got "%s"`, paths[i], i, matcher.template) + } + i++ + return nil + }) + if err != nil { + panic(err) + } + if i != len(paths) { + t.Errorf("Expected %d routes, found %d", len(paths), i) + } +} + +func TestWalkNested(t *testing.T) { + router := NewRouter() + + routeSubrouter := func(r *Route) (*Route, *Router) { + return r, r.Subrouter() + } + + gRoute, g := routeSubrouter(router.Path("/g")) + oRoute, o := routeSubrouter(g.PathPrefix("/o")) + rRoute, r := routeSubrouter(o.PathPrefix("/r")) + iRoute, i := routeSubrouter(r.PathPrefix("/i")) + l1Route, l1 := routeSubrouter(i.PathPrefix("/l")) + l2Route, l2 := routeSubrouter(l1.PathPrefix("/l")) + l2.Path("/a") + + testCases := []struct { + path string + ancestors []*Route + }{ + {"/g", []*Route{}}, + {"/g/o", []*Route{gRoute}}, + {"/g/o/r", []*Route{gRoute, oRoute}}, + {"/g/o/r/i", []*Route{gRoute, oRoute, rRoute}}, + {"/g/o/r/i/l", []*Route{gRoute, oRoute, rRoute, iRoute}}, + {"/g/o/r/i/l/l", []*Route{gRoute, oRoute, rRoute, iRoute, l1Route}}, + {"/g/o/r/i/l/l/a", []*Route{gRoute, oRoute, rRoute, iRoute, l1Route, l2Route}}, + } + + idx := 0 + err := router.Walk(func(route *Route, router *Router, ancestors []*Route) error { + path := testCases[idx].path + tpl := route.regexp.path.template + if tpl != path { + t.Errorf(`Expected %s got %s`, path, tpl) + } + currWantAncestors := testCases[idx].ancestors + if !reflect.DeepEqual(currWantAncestors, ancestors) { + t.Errorf(`Expected %+v got %+v`, currWantAncestors, ancestors) + } + idx++ + return nil + }) + if err != nil { + panic(err) + } + if idx != len(testCases) { + t.Errorf("Expected %d routes, found %d", len(testCases), idx) + } +} + +func TestWalkSubrouters(t *testing.T) { + router := NewRouter() + + g := router.Path("/g").Subrouter() + o := g.PathPrefix("/o").Subrouter() + o.Methods("GET") + o.Methods("PUT") + + // all 4 routes should be matched + paths := []string{"/g", "/g/o", "/g/o", "/g/o"} + idx := 0 + err := router.Walk(func(route *Route, router *Router, ancestors []*Route) error { + path := paths[idx] + tpl, _ := route.GetPathTemplate() + if tpl != path { + t.Errorf(`Expected %s got %s`, path, tpl) + } + idx++ + return nil + }) + if err != nil { + panic(err) + } + if idx != len(paths) { + t.Errorf("Expected %d routes, found %d", len(paths), idx) + } +} + +func TestWalkErrorRoute(t *testing.T) { + router := NewRouter() + router.Path("/g") + expectedError := errors.New("error") + err := router.Walk(func(route *Route, router *Router, ancestors []*Route) error { + return expectedError + }) + if err != expectedError { + t.Errorf("Expected %v routes, found %v", expectedError, err) + } +} + +func TestWalkErrorMatcher(t *testing.T) { + router := NewRouter() + expectedError := router.Path("/g").Subrouter().Path("").GetError() + err := router.Walk(func(route *Route, router *Router, ancestors []*Route) error { + return route.GetError() + }) + if err != expectedError { + t.Errorf("Expected %v routes, found %v", expectedError, err) + } +} + +func TestWalkErrorHandler(t *testing.T) { + handler := NewRouter() + expectedError := handler.Path("/path").Subrouter().Path("").GetError() + router := NewRouter() + router.Path("/g").Handler(handler) + err := router.Walk(func(route *Route, router *Router, ancestors []*Route) error { + return route.GetError() + }) + if err != expectedError { + t.Errorf("Expected %v routes, found %v", expectedError, err) + } +} + +func TestSubrouterErrorHandling(t *testing.T) { + superRouterCalled := false + subRouterCalled := false + + router := NewRouter() + router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + superRouterCalled = true + }) + subRouter := router.PathPrefix("/bign8").Subrouter() + subRouter.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + subRouterCalled = true + }) + + req, _ := http.NewRequest("GET", "http://localhost/bign8/was/here", nil) + router.ServeHTTP(NewRecorder(), req) + + if superRouterCalled { + t.Error("Super router 404 handler called when sub-router 404 handler is available.") + } + if !subRouterCalled { + t.Error("Sub-router 404 handler was not called.") + } +} + +// See: https://github.com/gorilla/mux/issues/200 +func TestPanicOnCapturingGroups(t *testing.T) { + defer func() { + if recover() == nil { + t.Errorf("(Test that capturing groups now fail fast) Expected panic, however test completed successfully.\n") + } + }() + NewRouter().NewRoute().Path("/{type:(promo|special)}/{promoId}.json") +} + +// ---------------------------------------------------------------------------- +// Helpers +// ---------------------------------------------------------------------------- + +func getRouteTemplate(route *Route) string { + host, err := route.GetHostTemplate() + if err != nil { + host = "none" + } + path, err := route.GetPathTemplate() + if err != nil { + path = "none" + } + return fmt.Sprintf("Host: %v, Path: %v", host, path) +} + +func testRoute(t *testing.T, test routeTest) { + request := test.request + route := test.route + vars := test.vars + shouldMatch := test.shouldMatch + query := test.query + shouldRedirect := test.shouldRedirect + uri := url.URL{ + Scheme: test.scheme, + Host: test.host, + Path: test.path, + } + if uri.Scheme == "" { + uri.Scheme = "http" + } + + var match RouteMatch + ok := route.Match(request, &match) + if ok != shouldMatch { + msg := "Should match" + if !shouldMatch { + msg = "Should not match" + } + t.Errorf("(%v) %v:\nRoute: %#v\nRequest: %#v\nVars: %v\n", test.title, msg, route, request, vars) + return + } + if shouldMatch { + if vars != nil && !stringMapEqual(vars, match.Vars) { + t.Errorf("(%v) Vars not equal: expected %v, got %v", test.title, vars, match.Vars) + return + } + if test.scheme != "" { + u, err := route.URL(mapToPairs(match.Vars)...) + if err != nil { + t.Fatalf("(%v) URL error: %v -- %v", test.title, err, getRouteTemplate(route)) + } + if uri.Scheme != u.Scheme { + t.Errorf("(%v) URLScheme not equal: expected %v, got %v", test.title, uri.Scheme, u.Scheme) + return + } + } + if test.host != "" { + u, err := test.route.URLHost(mapToPairs(match.Vars)...) + if err != nil { + t.Fatalf("(%v) URLHost error: %v -- %v", test.title, err, getRouteTemplate(route)) + } + if uri.Scheme != u.Scheme { + t.Errorf("(%v) URLHost scheme not equal: expected %v, got %v -- %v", test.title, uri.Scheme, u.Scheme, getRouteTemplate(route)) + return + } + if uri.Host != u.Host { + t.Errorf("(%v) URLHost host not equal: expected %v, got %v -- %v", test.title, uri.Host, u.Host, getRouteTemplate(route)) + return + } + } + if test.path != "" { + u, err := route.URLPath(mapToPairs(match.Vars)...) + if err != nil { + t.Fatalf("(%v) URLPath error: %v -- %v", test.title, err, getRouteTemplate(route)) + } + if uri.Path != u.Path { + t.Errorf("(%v) URLPath not equal: expected %v, got %v -- %v", test.title, uri.Path, u.Path, getRouteTemplate(route)) + return + } + } + if test.host != "" && test.path != "" { + u, err := route.URL(mapToPairs(match.Vars)...) + if err != nil { + t.Fatalf("(%v) URL error: %v -- %v", test.title, err, getRouteTemplate(route)) + } + if expected, got := uri.String(), u.String(); expected != got { + t.Errorf("(%v) URL not equal: expected %v, got %v -- %v", test.title, expected, got, getRouteTemplate(route)) + return + } + } + if query != "" { + u, err := route.URL(mapToPairs(match.Vars)...) + if err != nil { + t.Errorf("(%v) erred while creating url: %v", test.title, err) + return + } + if query != u.RawQuery { + t.Errorf("(%v) URL query not equal: expected %v, got %v", test.title, query, u.RawQuery) + return + } + } + if shouldRedirect && match.Handler == nil { + t.Errorf("(%v) Did not redirect", test.title) + return + } + if !shouldRedirect && match.Handler != nil { + t.Errorf("(%v) Unexpected redirect", test.title) + return + } + } +} + +func testUseEscapedRoute(t *testing.T, test routeTest) { + test.route.useEncodedPath = true + testRoute(t, test) +} + +func testTemplate(t *testing.T, test routeTest) { + route := test.route + pathTemplate := test.pathTemplate + if len(pathTemplate) == 0 { + pathTemplate = test.path + } + hostTemplate := test.hostTemplate + if len(hostTemplate) == 0 { + hostTemplate = test.host + } + + routePathTemplate, pathErr := route.GetPathTemplate() + if pathErr == nil && routePathTemplate != pathTemplate { + t.Errorf("(%v) GetPathTemplate not equal: expected %v, got %v", test.title, pathTemplate, routePathTemplate) + } + + routeHostTemplate, hostErr := route.GetHostTemplate() + if hostErr == nil && routeHostTemplate != hostTemplate { + t.Errorf("(%v) GetHostTemplate not equal: expected %v, got %v", test.title, hostTemplate, routeHostTemplate) + } +} + +func testMethods(t *testing.T, test routeTest) { + route := test.route + methods, _ := route.GetMethods() + if strings.Join(methods, ",") != strings.Join(test.methods, ",") { + t.Errorf("(%v) GetMethods not equal: expected %v, got %v", test.title, test.methods, methods) + } +} + +func testRegexp(t *testing.T, test routeTest) { + route := test.route + routePathRegexp, regexpErr := route.GetPathRegexp() + if test.pathRegexp != "" && regexpErr == nil && routePathRegexp != test.pathRegexp { + t.Errorf("(%v) GetPathRegexp not equal: expected %v, got %v", test.title, test.pathRegexp, routePathRegexp) + } +} + +func testQueriesRegexp(t *testing.T, test routeTest) { + route := test.route + queries, queriesErr := route.GetQueriesRegexp() + gotQueries := strings.Join(queries, ",") + if test.queriesRegexp != "" && queriesErr == nil && gotQueries != test.queriesRegexp { + t.Errorf("(%v) GetQueriesRegexp not equal: expected %v, got %v", test.title, test.queriesRegexp, gotQueries) + } +} + +func testQueriesTemplates(t *testing.T, test routeTest) { + route := test.route + queries, queriesErr := route.GetQueriesTemplates() + gotQueries := strings.Join(queries, ",") + if test.queriesTemplate != "" && queriesErr == nil && gotQueries != test.queriesTemplate { + t.Errorf("(%v) GetQueriesTemplates not equal: expected %v, got %v", test.title, test.queriesTemplate, gotQueries) + } +} + +type TestA301ResponseWriter struct { + hh http.Header + status int +} + +func (ho *TestA301ResponseWriter) Header() http.Header { + return http.Header(ho.hh) +} + +func (ho *TestA301ResponseWriter) Write(b []byte) (int, error) { + return 0, nil +} + +func (ho *TestA301ResponseWriter) WriteHeader(code int) { + ho.status = code +} + +func Test301Redirect(t *testing.T) { + m := make(http.Header) + + func1 := func(w http.ResponseWriter, r *http.Request) {} + func2 := func(w http.ResponseWriter, r *http.Request) {} + + r := NewRouter() + r.HandleFunc("/api/", func2).Name("func2") + r.HandleFunc("/", func1).Name("func1") + + req, _ := http.NewRequest("GET", "http://localhost//api/?abc=def", nil) + + res := TestA301ResponseWriter{ + hh: m, + status: 0, + } + r.ServeHTTP(&res, req) + + if "http://localhost/api/?abc=def" != res.hh["Location"][0] { + t.Errorf("Should have complete URL with query string") + } +} + +func TestSkipClean(t *testing.T) { + func1 := func(w http.ResponseWriter, r *http.Request) {} + func2 := func(w http.ResponseWriter, r *http.Request) {} + + r := NewRouter() + r.SkipClean(true) + r.HandleFunc("/api/", func2).Name("func2") + r.HandleFunc("/", func1).Name("func1") + + req, _ := http.NewRequest("GET", "http://localhost//api/?abc=def", nil) + res := NewRecorder() + r.ServeHTTP(res, req) + + if len(res.HeaderMap["Location"]) != 0 { + t.Errorf("Shouldn't redirect since skip clean is disabled") + } +} + +// https://plus.google.com/101022900381697718949/posts/eWy6DjFJ6uW +func TestSubrouterHeader(t *testing.T) { + expected := "func1 response" + func1 := func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, expected) + } + func2 := func(http.ResponseWriter, *http.Request) {} + + r := NewRouter() + s := r.Headers("SomeSpecialHeader", "").Subrouter() + s.HandleFunc("/", func1).Name("func1") + r.HandleFunc("/", func2).Name("func2") + + req, _ := http.NewRequest("GET", "http://localhost/", nil) + req.Header.Add("SomeSpecialHeader", "foo") + match := new(RouteMatch) + matched := r.Match(req, match) + if !matched { + t.Errorf("Should match request") + } + if match.Route.GetName() != "func1" { + t.Errorf("Expecting func1 handler, got %s", match.Route.GetName()) + } + resp := NewRecorder() + match.Handler.ServeHTTP(resp, req) + if resp.Body.String() != expected { + t.Errorf("Expecting %q", expected) + } +} + +func TestNoMatchMethodErrorHandler(t *testing.T) { + func1 := func(w http.ResponseWriter, r *http.Request) {} + + r := NewRouter() + r.HandleFunc("/", func1).Methods("GET", "POST") + + req, _ := http.NewRequest("PUT", "http://localhost/", nil) + match := new(RouteMatch) + matched := r.Match(req, match) + + if matched { + t.Error("Should not have matched route for methods") + } + + if match.MatchErr != ErrMethodMismatch { + t.Error("Should get ErrMethodMismatch error") + } + + resp := NewRecorder() + r.ServeHTTP(resp, req) + if resp.Code != 405 { + t.Errorf("Expecting code %v", 405) + } + + // Add matching route + r.HandleFunc("/", func1).Methods("PUT") + + match = new(RouteMatch) + matched = r.Match(req, match) + + if !matched { + t.Error("Should have matched route for methods") + } + + if match.MatchErr != nil { + t.Error("Should not have any matching error. Found:", match.MatchErr) + } +} + +func TestErrMatchNotFound(t *testing.T) { + emptyHandler := func(w http.ResponseWriter, r *http.Request) {} + + r := NewRouter() + r.HandleFunc("/", emptyHandler) + s := r.PathPrefix("/sub/").Subrouter() + s.HandleFunc("/", emptyHandler) + + // Regular 404 not found + req, _ := http.NewRequest("GET", "/sub/whatever", nil) + match := new(RouteMatch) + matched := r.Match(req, match) + + if matched { + t.Errorf("Subrouter should not have matched that, got %v", match.Route) + } + // Even without a custom handler, MatchErr is set to ErrNotFound + if match.MatchErr != ErrNotFound { + t.Errorf("Expected ErrNotFound MatchErr, but was %v", match.MatchErr) + } + + // Now lets add a 404 handler to subrouter + s.NotFoundHandler = http.NotFoundHandler() + req, _ = http.NewRequest("GET", "/sub/whatever", nil) + + // Test the subrouter first + match = new(RouteMatch) + matched = s.Match(req, match) + // Now we should get a match + if !matched { + t.Errorf("Subrouter should have matched %s", req.RequestURI) + } + // But MatchErr should be set to ErrNotFound anyway + if match.MatchErr != ErrNotFound { + t.Errorf("Expected ErrNotFound MatchErr, but was %v", match.MatchErr) + } + + // Now test the parent (MatchErr should propagate) + match = new(RouteMatch) + matched = r.Match(req, match) + + // Now we should get a match + if !matched { + t.Errorf("Router should have matched %s via subrouter", req.RequestURI) + } + // But MatchErr should be set to ErrNotFound anyway + if match.MatchErr != ErrNotFound { + t.Errorf("Expected ErrNotFound MatchErr, but was %v", match.MatchErr) + } +} + +// methodsSubrouterTest models the data necessary for testing handler +// matching for subrouters created after HTTP methods matcher registration. +type methodsSubrouterTest struct { + title string + wantCode int + router *Router + // method is the input into the request and expected response + method string + // input request path + path string + // redirectTo is the expected location path for strict-slash matches + redirectTo string +} + +// methodHandler writes the method string in response. +func methodHandler(method string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(method)) + } +} + +// TestMethodsSubrouterCatchall matches handlers for subrouters where a +// catchall handler is set for a mis-matching method. +func TestMethodsSubrouterCatchall(t *testing.T) { + t.Parallel() + + router := NewRouter() + router.Methods("PATCH").Subrouter().PathPrefix("/").HandlerFunc(methodHandler("PUT")) + router.Methods("GET").Subrouter().HandleFunc("/foo", methodHandler("GET")) + router.Methods("POST").Subrouter().HandleFunc("/foo", methodHandler("POST")) + router.Methods("DELETE").Subrouter().HandleFunc("/foo", methodHandler("DELETE")) + + tests := []methodsSubrouterTest{ + { + title: "match GET handler", + router: router, + path: "http://localhost/foo", + method: "GET", + wantCode: http.StatusOK, + }, + { + title: "match POST handler", + router: router, + method: "POST", + path: "http://localhost/foo", + wantCode: http.StatusOK, + }, + { + title: "match DELETE handler", + router: router, + method: "DELETE", + path: "http://localhost/foo", + wantCode: http.StatusOK, + }, + { + title: "disallow PUT method", + router: router, + method: "PUT", + path: "http://localhost/foo", + wantCode: http.StatusMethodNotAllowed, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testMethodsSubrouter(t, test) + }) + } +} + +// TestMethodsSubrouterStrictSlash matches handlers on subrouters with +// strict-slash matchers. +func TestMethodsSubrouterStrictSlash(t *testing.T) { + t.Parallel() + + router := NewRouter() + sub := router.PathPrefix("/").Subrouter() + sub.StrictSlash(true).Path("/foo").Methods("GET").Subrouter().HandleFunc("", methodHandler("GET")) + sub.StrictSlash(true).Path("/foo/").Methods("PUT").Subrouter().HandleFunc("/", methodHandler("PUT")) + sub.StrictSlash(true).Path("/foo/").Methods("POST").Subrouter().HandleFunc("/", methodHandler("POST")) + + tests := []methodsSubrouterTest{ + { + title: "match POST handler", + router: router, + method: "POST", + path: "http://localhost/foo/", + wantCode: http.StatusOK, + }, + { + title: "match GET handler", + router: router, + method: "GET", + path: "http://localhost/foo", + wantCode: http.StatusOK, + }, + { + title: "match POST handler, redirect strict-slash", + router: router, + method: "POST", + path: "http://localhost/foo", + redirectTo: "http://localhost/foo/", + wantCode: http.StatusMovedPermanently, + }, + { + title: "match GET handler, redirect strict-slash", + router: router, + method: "GET", + path: "http://localhost/foo/", + redirectTo: "http://localhost/foo", + wantCode: http.StatusMovedPermanently, + }, + { + title: "disallow DELETE method", + router: router, + method: "DELETE", + path: "http://localhost/foo", + wantCode: http.StatusMethodNotAllowed, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testMethodsSubrouter(t, test) + }) + } +} + +// TestMethodsSubrouterPathPrefix matches handlers on subrouters created +// on a router with a path prefix matcher and method matcher. +func TestMethodsSubrouterPathPrefix(t *testing.T) { + t.Parallel() + + router := NewRouter() + router.PathPrefix("/1").Methods("POST").Subrouter().HandleFunc("/2", methodHandler("POST")) + router.PathPrefix("/1").Methods("DELETE").Subrouter().HandleFunc("/2", methodHandler("DELETE")) + router.PathPrefix("/1").Methods("PUT").Subrouter().HandleFunc("/2", methodHandler("PUT")) + router.PathPrefix("/1").Methods("POST").Subrouter().HandleFunc("/2", methodHandler("POST2")) + + tests := []methodsSubrouterTest{ + { + title: "match first POST handler", + router: router, + method: "POST", + path: "http://localhost/1/2", + wantCode: http.StatusOK, + }, + { + title: "match DELETE handler", + router: router, + method: "DELETE", + path: "http://localhost/1/2", + wantCode: http.StatusOK, + }, + { + title: "match PUT handler", + router: router, + method: "PUT", + path: "http://localhost/1/2", + wantCode: http.StatusOK, + }, + { + title: "disallow PATCH method", + router: router, + method: "PATCH", + path: "http://localhost/1/2", + wantCode: http.StatusMethodNotAllowed, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testMethodsSubrouter(t, test) + }) + } +} + +// TestMethodsSubrouterSubrouter matches handlers on subrouters produced +// from method matchers registered on a root subrouter. +func TestMethodsSubrouterSubrouter(t *testing.T) { + t.Parallel() + + router := NewRouter() + sub := router.PathPrefix("/1").Subrouter() + sub.Methods("POST").Subrouter().HandleFunc("/2", methodHandler("POST")) + sub.Methods("GET").Subrouter().HandleFunc("/2", methodHandler("GET")) + sub.Methods("PATCH").Subrouter().HandleFunc("/2", methodHandler("PATCH")) + sub.HandleFunc("/2", methodHandler("PUT")).Subrouter().Methods("PUT") + sub.HandleFunc("/2", methodHandler("POST2")).Subrouter().Methods("POST") + + tests := []methodsSubrouterTest{ + { + title: "match first POST handler", + router: router, + method: "POST", + path: "http://localhost/1/2", + wantCode: http.StatusOK, + }, + { + title: "match GET handler", + router: router, + method: "GET", + path: "http://localhost/1/2", + wantCode: http.StatusOK, + }, + { + title: "match PATCH handler", + router: router, + method: "PATCH", + path: "http://localhost/1/2", + wantCode: http.StatusOK, + }, + { + title: "match PUT handler", + router: router, + method: "PUT", + path: "http://localhost/1/2", + wantCode: http.StatusOK, + }, + { + title: "disallow DELETE method", + router: router, + method: "DELETE", + path: "http://localhost/1/2", + wantCode: http.StatusMethodNotAllowed, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testMethodsSubrouter(t, test) + }) + } +} + +// TestMethodsSubrouterPathVariable matches handlers on matching paths +// with path variables in them. +func TestMethodsSubrouterPathVariable(t *testing.T) { + t.Parallel() + + router := NewRouter() + router.Methods("GET").Subrouter().HandleFunc("/foo", methodHandler("GET")) + router.Methods("POST").Subrouter().HandleFunc("/{any}", methodHandler("POST")) + router.Methods("DELETE").Subrouter().HandleFunc("/1/{any}", methodHandler("DELETE")) + router.Methods("PUT").Subrouter().HandleFunc("/1/{any}", methodHandler("PUT")) + + tests := []methodsSubrouterTest{ + { + title: "match GET handler", + router: router, + method: "GET", + path: "http://localhost/foo", + wantCode: http.StatusOK, + }, + { + title: "match POST handler", + router: router, + method: "POST", + path: "http://localhost/foo", + wantCode: http.StatusOK, + }, + { + title: "match DELETE handler", + router: router, + method: "DELETE", + path: "http://localhost/1/foo", + wantCode: http.StatusOK, + }, + { + title: "match PUT handler", + router: router, + method: "PUT", + path: "http://localhost/1/foo", + wantCode: http.StatusOK, + }, + { + title: "disallow PATCH method", + router: router, + method: "PATCH", + path: "http://localhost/1/foo", + wantCode: http.StatusMethodNotAllowed, + }, + } + + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + testMethodsSubrouter(t, test) + }) + } +} + +func ExampleSetURLVars() { + req, _ := http.NewRequest("GET", "/foo", nil) + req = SetURLVars(req, map[string]string{"foo": "bar"}) + + fmt.Println(Vars(req)["foo"]) + + // Output: bar +} + +// testMethodsSubrouter runs an individual methodsSubrouterTest. +func testMethodsSubrouter(t *testing.T, test methodsSubrouterTest) { + // Execute request + req, _ := http.NewRequest(test.method, test.path, nil) + resp := NewRecorder() + test.router.ServeHTTP(resp, req) + + switch test.wantCode { + case http.StatusMethodNotAllowed: + if resp.Code != http.StatusMethodNotAllowed { + t.Errorf(`(%s) Expected "405 Method Not Allowed", but got %d code`, test.title, resp.Code) + } else if matchedMethod := resp.Body.String(); matchedMethod != "" { + t.Errorf(`(%s) Expected "405 Method Not Allowed", but %q handler was called`, test.title, matchedMethod) + } + + case http.StatusMovedPermanently: + if gotLocation := resp.HeaderMap.Get("Location"); gotLocation != test.redirectTo { + t.Errorf("(%s) Expected %q route-match to redirect to %q, but got %q", test.title, test.method, test.redirectTo, gotLocation) + } + + case http.StatusOK: + if matchedMethod := resp.Body.String(); matchedMethod != test.method { + t.Errorf("(%s) Expected %q handler to be called, but %q handler was called", test.title, test.method, matchedMethod) + } + + default: + expectedCodes := []int{http.StatusMethodNotAllowed, http.StatusMovedPermanently, http.StatusOK} + t.Errorf("(%s) Expected wantCode to be one of: %v, but got %d", test.title, expectedCodes, test.wantCode) + } +} + +func TestSubrouterMatching(t *testing.T) { + const ( + none, stdOnly, subOnly uint8 = 0, 1 << 0, 1 << 1 + both = subOnly | stdOnly + ) + + type request struct { + Name string + Request *http.Request + Flags uint8 + } + + cases := []struct { + Name string + Standard, Subrouter func(*Router) + Requests []request + }{ + { + "pathPrefix", + func(r *Router) { + r.PathPrefix("/before").PathPrefix("/after") + }, + func(r *Router) { + r.PathPrefix("/before").Subrouter().PathPrefix("/after") + }, + []request{ + {"no match final path prefix", newRequest("GET", "/after"), none}, + {"no match parent path prefix", newRequest("GET", "/before"), none}, + {"matches append", newRequest("GET", "/before/after"), both}, + {"matches as prefix", newRequest("GET", "/before/after/1234"), both}, + }, + }, + { + "path", + func(r *Router) { + r.Path("/before").Path("/after") + }, + func(r *Router) { + r.Path("/before").Subrouter().Path("/after") + }, + []request{ + {"no match subroute path", newRequest("GET", "/after"), none}, + {"no match parent path", newRequest("GET", "/before"), none}, + {"no match as prefix", newRequest("GET", "/before/after/1234"), none}, + {"no match append", newRequest("GET", "/before/after"), none}, + }, + }, + { + "host", + func(r *Router) { + r.Host("before.com").Host("after.com") + }, + func(r *Router) { + r.Host("before.com").Subrouter().Host("after.com") + }, + []request{ + {"no match before", newRequestHost("GET", "/", "before.com"), none}, + {"no match other", newRequestHost("GET", "/", "other.com"), none}, + {"matches after", newRequestHost("GET", "/", "after.com"), none}, + }, + }, + { + "queries variant keys", + func(r *Router) { + r.Queries("foo", "bar").Queries("cricket", "baseball") + }, + func(r *Router) { + r.Queries("foo", "bar").Subrouter().Queries("cricket", "baseball") + }, + []request{ + {"matches with all", newRequest("GET", "/?foo=bar&cricket=baseball"), both}, + {"matches with more", newRequest("GET", "/?foo=bar&cricket=baseball&something=else"), both}, + {"no match with none", newRequest("GET", "/"), none}, + {"no match with some", newRequest("GET", "/?cricket=baseball"), none}, + }, + }, + { + "queries overlapping keys", + func(r *Router) { + r.Queries("foo", "bar").Queries("foo", "baz") + }, + func(r *Router) { + r.Queries("foo", "bar").Subrouter().Queries("foo", "baz") + }, + []request{ + {"no match old value", newRequest("GET", "/?foo=bar"), none}, + {"no match diff value", newRequest("GET", "/?foo=bak"), none}, + {"no match with none", newRequest("GET", "/"), none}, + {"matches override", newRequest("GET", "/?foo=baz"), none}, + }, + }, + { + "header variant keys", + func(r *Router) { + r.Headers("foo", "bar").Headers("cricket", "baseball") + }, + func(r *Router) { + r.Headers("foo", "bar").Subrouter().Headers("cricket", "baseball") + }, + []request{ + { + "matches with all", + newRequestWithHeaders("GET", "/", "foo", "bar", "cricket", "baseball"), + both, + }, + { + "matches with more", + newRequestWithHeaders("GET", "/", "foo", "bar", "cricket", "baseball", "something", "else"), + both, + }, + {"no match with none", newRequest("GET", "/"), none}, + {"no match with some", newRequestWithHeaders("GET", "/", "cricket", "baseball"), none}, + }, + }, + { + "header overlapping keys", + func(r *Router) { + r.Headers("foo", "bar").Headers("foo", "baz") + }, + func(r *Router) { + r.Headers("foo", "bar").Subrouter().Headers("foo", "baz") + }, + []request{ + {"no match old value", newRequestWithHeaders("GET", "/", "foo", "bar"), none}, + {"no match diff value", newRequestWithHeaders("GET", "/", "foo", "bak"), none}, + {"no match with none", newRequest("GET", "/"), none}, + {"matches override", newRequestWithHeaders("GET", "/", "foo", "baz"), none}, + }, + }, + { + "method", + func(r *Router) { + r.Methods("POST").Methods("GET") + }, + func(r *Router) { + r.Methods("POST").Subrouter().Methods("GET") + }, + []request{ + {"matches before", newRequest("POST", "/"), none}, + {"no match other", newRequest("HEAD", "/"), none}, + {"matches override", newRequest("GET", "/"), none}, + }, + }, + { + "schemes", + func(r *Router) { + r.Schemes("http").Schemes("https") + }, + func(r *Router) { + r.Schemes("http").Subrouter().Schemes("https") + }, + []request{ + {"matches overrides", newRequest("GET", "https://www.example.com/"), none}, + {"matches original", newRequest("GET", "http://www.example.com/"), none}, + {"no match other", newRequest("GET", "ftp://www.example.com/"), none}, + }, + }, + } + + // case -> request -> router + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + for _, req := range c.Requests { + t.Run(req.Name, func(t *testing.T) { + for _, v := range []struct { + Name string + Config func(*Router) + Expected bool + }{ + {"subrouter", c.Subrouter, (req.Flags & subOnly) != 0}, + {"standard", c.Standard, (req.Flags & stdOnly) != 0}, + } { + r := NewRouter() + v.Config(r) + if r.Match(req.Request, &RouteMatch{}) != v.Expected { + if v.Expected { + t.Errorf("expected %v match", v.Name) + } else { + t.Errorf("expected %v no match", v.Name) + } + } + } + }) + } + }) + } +} + +// verify that copyRouteConf copies fields as expected. +func Test_copyRouteConf(t *testing.T) { + var ( + m MatcherFunc = func(*http.Request, *RouteMatch) bool { + return true + } + b BuildVarsFunc = func(i map[string]string) map[string]string { + return i + } + r, _ = newRouteRegexp("hi", regexpTypeHost, routeRegexpOptions{}) + ) + + tests := []struct { + name string + args routeConf + want routeConf + }{ + { + "empty", + routeConf{}, + routeConf{}, + }, + { + "full", + routeConf{ + useEncodedPath: true, + strictSlash: true, + skipClean: true, + regexp: routeRegexpGroup{host: r, path: r, queries: []*routeRegexp{r}}, + matchers: []matcher{m}, + buildScheme: "https", + buildVarsFunc: b, + }, + routeConf{ + useEncodedPath: true, + strictSlash: true, + skipClean: true, + regexp: routeRegexpGroup{host: r, path: r, queries: []*routeRegexp{r}}, + matchers: []matcher{m}, + buildScheme: "https", + buildVarsFunc: b, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // special case some incomparable fields of routeConf before delegating to reflect.DeepEqual + got := copyRouteConf(tt.args) + + // funcs not comparable, just compare length of slices + if len(got.matchers) != len(tt.want.matchers) { + t.Errorf("matchers different lengths: %v %v", len(got.matchers), len(tt.want.matchers)) + } + got.matchers, tt.want.matchers = nil, nil + + // deep equal treats nil slice differently to empty slice so check for zero len first + { + bothZero := len(got.regexp.queries) == 0 && len(tt.want.regexp.queries) == 0 + if !bothZero && !reflect.DeepEqual(got.regexp.queries, tt.want.regexp.queries) { + t.Errorf("queries unequal: %v %v", got.regexp.queries, tt.want.regexp.queries) + } + got.regexp.queries, tt.want.regexp.queries = nil, nil + } + + // funcs not comparable, just compare nullity + if (got.buildVarsFunc == nil) != (tt.want.buildVarsFunc == nil) { + t.Errorf("build vars funcs unequal: %v %v", got.buildVarsFunc == nil, tt.want.buildVarsFunc == nil) + } + got.buildVarsFunc, tt.want.buildVarsFunc = nil, nil + + // finish the deal + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("route confs unequal: %v %v", got, tt.want) + } + }) + } +} + +func TestMethodNotAllowed(t *testing.T) { + handler := func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } + router := NewRouter() + router.HandleFunc("/thing", handler).Methods(http.MethodGet) + router.HandleFunc("/something", handler).Methods(http.MethodGet) + + w := NewRecorder() + req := newRequest(http.MethodPut, "/thing") + + router.ServeHTTP(w, req) + + if w.Code != 405 { + t.Fatalf("Expected status code 405 (got %d)", w.Code) + } +} + +func TestSubrouterNotFound(t *testing.T) { + handler := func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } + router := NewRouter() + router.Path("/a").Subrouter().HandleFunc("/thing", handler).Methods(http.MethodGet) + router.Path("/b").Subrouter().HandleFunc("/something", handler).Methods(http.MethodGet) + + w := NewRecorder() + req := newRequest(http.MethodPut, "/not-present") + + router.ServeHTTP(w, req) + + if w.Code != 404 { + t.Fatalf("Expected status code 404 (got %d)", w.Code) + } +} + +// mapToPairs converts a string map to a slice of string pairs +func mapToPairs(m map[string]string) []string { + var i int + p := make([]string, len(m)*2) + for k, v := range m { + p[i] = k + p[i+1] = v + i += 2 + } + return p +} + +// stringMapEqual checks the equality of two string maps +func stringMapEqual(m1, m2 map[string]string) bool { + nil1 := m1 == nil + nil2 := m2 == nil + if nil1 != nil2 || len(m1) != len(m2) { + return false + } + for k, v := range m1 { + if v != m2[k] { + return false + } + } + return true +} + +// stringHandler returns a handler func that writes a message 's' to the +// http.ResponseWriter. +func stringHandler(s string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(s)) + } +} + +// newRequest is a helper function to create a new request with a method and url. +// The request returned is a 'server' request as opposed to a 'client' one through +// simulated write onto the wire and read off of the wire. +// The differences between requests are detailed in the net/http package. +func newRequest(method, url string) *http.Request { + req, err := http.NewRequest(method, url, nil) + if err != nil { + panic(err) + } + // extract the escaped original host+path from url + // http://localhost/path/here?v=1#frag -> //localhost/path/here + opaque := "" + if i := len(req.URL.Scheme); i > 0 { + opaque = url[i+1:] + } + + if i := strings.LastIndex(opaque, "?"); i > -1 { + opaque = opaque[:i] + } + if i := strings.LastIndex(opaque, "#"); i > -1 { + opaque = opaque[:i] + } + + // Escaped host+path workaround as detailed in https://golang.org/pkg/net/url/#URL + // for < 1.5 client side workaround + req.URL.Opaque = opaque + + // Simulate writing to wire + var buff bytes.Buffer + req.Write(&buff) + ioreader := bufio.NewReader(&buff) + + // Parse request off of 'wire' + req, err = http.ReadRequest(ioreader) + if err != nil { + panic(err) + } + return req +} + +// create a new request with the provided headers +func newRequestWithHeaders(method, url string, headers ...string) *http.Request { + req := newRequest(method, url) + + if len(headers)%2 != 0 { + panic(fmt.Sprintf("Expected headers length divisible by 2 but got %v", len(headers))) + } + + for i := 0; i < len(headers); i += 2 { + req.Header.Set(headers[i], headers[i+1]) + } + + return req +} + +// newRequestHost a new request with a method, url, and host header +func newRequestHost(method, url, host string) *http.Request { + req, err := http.NewRequest(method, url, nil) + if err != nil { + panic(err) + } + req.Host = host + return req +} diff --git a/vendor/github.com/gorilla/mux/old_test.go b/vendor/github.com/gorilla/mux/old_test.go new file mode 100644 index 000000000..b228983c4 --- /dev/null +++ b/vendor/github.com/gorilla/mux/old_test.go @@ -0,0 +1,704 @@ +// Old tests ported to Go1. This is a mess. Want to drop it one day. + +// Copyright 2011 Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import ( + "bytes" + "net/http" + "testing" +) + +// ---------------------------------------------------------------------------- +// ResponseRecorder +// ---------------------------------------------------------------------------- +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// ResponseRecorder is an implementation of http.ResponseWriter that +// records its mutations for later inspection in tests. +type ResponseRecorder struct { + Code int // the HTTP response code from WriteHeader + HeaderMap http.Header // the HTTP response headers + Body *bytes.Buffer // if non-nil, the bytes.Buffer to append written data to + Flushed bool +} + +// NewRecorder returns an initialized ResponseRecorder. +func NewRecorder() *ResponseRecorder { + return &ResponseRecorder{ + HeaderMap: make(http.Header), + Body: new(bytes.Buffer), + } +} + +// Header returns the response headers. +func (rw *ResponseRecorder) Header() http.Header { + return rw.HeaderMap +} + +// Write always succeeds and writes to rw.Body, if not nil. +func (rw *ResponseRecorder) Write(buf []byte) (int, error) { + if rw.Body != nil { + rw.Body.Write(buf) + } + if rw.Code == 0 { + rw.Code = http.StatusOK + } + return len(buf), nil +} + +// WriteHeader sets rw.Code. +func (rw *ResponseRecorder) WriteHeader(code int) { + rw.Code = code +} + +// Flush sets rw.Flushed to true. +func (rw *ResponseRecorder) Flush() { + rw.Flushed = true +} + +// ---------------------------------------------------------------------------- + +func TestRouteMatchers(t *testing.T) { + var scheme, host, path, query, method string + var headers map[string]string + var resultVars map[bool]map[string]string + + router := NewRouter() + router.NewRoute().Host("{var1}.google.com"). + Path("/{var2:[a-z]+}/{var3:[0-9]+}"). + Queries("foo", "bar"). + Methods("GET"). + Schemes("https"). + Headers("x-requested-with", "XMLHttpRequest") + router.NewRoute().Host("www.{var4}.com"). + PathPrefix("/foo/{var5:[a-z]+}/{var6:[0-9]+}"). + Queries("baz", "ding"). + Methods("POST"). + Schemes("http"). + Headers("Content-Type", "application/json") + + reset := func() { + // Everything match. + scheme = "https" + host = "www.google.com" + path = "/product/42" + query = "?foo=bar" + method = "GET" + headers = map[string]string{"X-Requested-With": "XMLHttpRequest"} + resultVars = map[bool]map[string]string{ + true: {"var1": "www", "var2": "product", "var3": "42"}, + false: {}, + } + } + + reset2 := func() { + // Everything match. + scheme = "http" + host = "www.google.com" + path = "/foo/product/42/path/that/is/ignored" + query = "?baz=ding" + method = "POST" + headers = map[string]string{"Content-Type": "application/json"} + resultVars = map[bool]map[string]string{ + true: {"var4": "google", "var5": "product", "var6": "42"}, + false: {}, + } + } + + match := func(shouldMatch bool) { + url := scheme + "://" + host + path + query + request, _ := http.NewRequest(method, url, nil) + for key, value := range headers { + request.Header.Add(key, value) + } + + var routeMatch RouteMatch + matched := router.Match(request, &routeMatch) + if matched != shouldMatch { + t.Errorf("Expected: %v\nGot: %v\nRequest: %v %v", shouldMatch, matched, request.Method, url) + } + + if matched { + currentRoute := routeMatch.Route + if currentRoute == nil { + t.Errorf("Expected a current route.") + } + vars := routeMatch.Vars + expectedVars := resultVars[shouldMatch] + if len(vars) != len(expectedVars) { + t.Errorf("Expected vars: %v Got: %v.", expectedVars, vars) + } + for name, value := range vars { + if expectedVars[name] != value { + t.Errorf("Expected vars: %v Got: %v.", expectedVars, vars) + } + } + } + } + + // 1st route -------------------------------------------------------------- + + // Everything match. + reset() + match(true) + + // Scheme doesn't match. + reset() + scheme = "http" + match(false) + + // Host doesn't match. + reset() + host = "www.mygoogle.com" + match(false) + + // Path doesn't match. + reset() + path = "/product/notdigits" + match(false) + + // Query doesn't match. + reset() + query = "?foo=baz" + match(false) + + // Method doesn't match. + reset() + method = "POST" + match(false) + + // Header doesn't match. + reset() + headers = map[string]string{} + match(false) + + // Everything match, again. + reset() + match(true) + + // 2nd route -------------------------------------------------------------- + // Everything match. + reset2() + match(true) + + // Scheme doesn't match. + reset2() + scheme = "https" + match(false) + + // Host doesn't match. + reset2() + host = "sub.google.com" + match(false) + + // Path doesn't match. + reset2() + path = "/bar/product/42" + match(false) + + // Query doesn't match. + reset2() + query = "?foo=baz" + match(false) + + // Method doesn't match. + reset2() + method = "GET" + match(false) + + // Header doesn't match. + reset2() + headers = map[string]string{} + match(false) + + // Everything match, again. + reset2() + match(true) +} + +type headerMatcherTest struct { + matcher headerMatcher + headers map[string]string + result bool +} + +var headerMatcherTests = []headerMatcherTest{ + { + matcher: headerMatcher(map[string]string{"x-requested-with": "XMLHttpRequest"}), + headers: map[string]string{"X-Requested-With": "XMLHttpRequest"}, + result: true, + }, + { + matcher: headerMatcher(map[string]string{"x-requested-with": ""}), + headers: map[string]string{"X-Requested-With": "anything"}, + result: true, + }, + { + matcher: headerMatcher(map[string]string{"x-requested-with": "XMLHttpRequest"}), + headers: map[string]string{}, + result: false, + }, +} + +type hostMatcherTest struct { + matcher *Route + url string + vars map[string]string + result bool +} + +var hostMatcherTests = []hostMatcherTest{ + { + matcher: NewRouter().NewRoute().Host("{foo:[a-z][a-z][a-z]}.{bar:[a-z][a-z][a-z]}.{baz:[a-z][a-z][a-z]}"), + url: "http://abc.def.ghi/", + vars: map[string]string{"foo": "abc", "bar": "def", "baz": "ghi"}, + result: true, + }, + { + matcher: NewRouter().NewRoute().Host("{foo:[a-z][a-z][a-z]}.{bar:[a-z][a-z][a-z]}.{baz:[a-z][a-z][a-z]}"), + url: "http://a.b.c/", + vars: map[string]string{"foo": "abc", "bar": "def", "baz": "ghi"}, + result: false, + }, +} + +type methodMatcherTest struct { + matcher methodMatcher + method string + result bool +} + +var methodMatcherTests = []methodMatcherTest{ + { + matcher: methodMatcher([]string{"GET", "POST", "PUT"}), + method: "GET", + result: true, + }, + { + matcher: methodMatcher([]string{"GET", "POST", "PUT"}), + method: "POST", + result: true, + }, + { + matcher: methodMatcher([]string{"GET", "POST", "PUT"}), + method: "PUT", + result: true, + }, + { + matcher: methodMatcher([]string{"GET", "POST", "PUT"}), + method: "DELETE", + result: false, + }, +} + +type pathMatcherTest struct { + matcher *Route + url string + vars map[string]string + result bool +} + +var pathMatcherTests = []pathMatcherTest{ + { + matcher: NewRouter().NewRoute().Path("/{foo:[0-9][0-9][0-9]}/{bar:[0-9][0-9][0-9]}/{baz:[0-9][0-9][0-9]}"), + url: "http://localhost:8080/123/456/789", + vars: map[string]string{"foo": "123", "bar": "456", "baz": "789"}, + result: true, + }, + { + matcher: NewRouter().NewRoute().Path("/{foo:[0-9][0-9][0-9]}/{bar:[0-9][0-9][0-9]}/{baz:[0-9][0-9][0-9]}"), + url: "http://localhost:8080/1/2/3", + vars: map[string]string{"foo": "123", "bar": "456", "baz": "789"}, + result: false, + }, +} + +type schemeMatcherTest struct { + matcher schemeMatcher + url string + result bool +} + +var schemeMatcherTests = []schemeMatcherTest{ + { + matcher: schemeMatcher([]string{"http", "https"}), + url: "http://localhost:8080/", + result: true, + }, + { + matcher: schemeMatcher([]string{"http", "https"}), + url: "https://localhost:8080/", + result: true, + }, + { + matcher: schemeMatcher([]string{"https"}), + url: "http://localhost:8080/", + result: false, + }, + { + matcher: schemeMatcher([]string{"http"}), + url: "https://localhost:8080/", + result: false, + }, +} + +type urlBuildingTest struct { + route *Route + vars []string + url string +} + +var urlBuildingTests = []urlBuildingTest{ + { + route: new(Route).Host("foo.domain.com"), + vars: []string{}, + url: "http://foo.domain.com", + }, + { + route: new(Route).Host("{subdomain}.domain.com"), + vars: []string{"subdomain", "bar"}, + url: "http://bar.domain.com", + }, + { + route: new(Route).Host("foo.domain.com").Path("/articles"), + vars: []string{}, + url: "http://foo.domain.com/articles", + }, + { + route: new(Route).Path("/articles"), + vars: []string{}, + url: "/articles", + }, + { + route: new(Route).Path("/articles/{category}/{id:[0-9]+}"), + vars: []string{"category", "technology", "id", "42"}, + url: "/articles/technology/42", + }, + { + route: new(Route).Host("{subdomain}.domain.com").Path("/articles/{category}/{id:[0-9]+}"), + vars: []string{"subdomain", "foo", "category", "technology", "id", "42"}, + url: "http://foo.domain.com/articles/technology/42", + }, +} + +func TestHeaderMatcher(t *testing.T) { + for _, v := range headerMatcherTests { + request, _ := http.NewRequest("GET", "http://localhost:8080/", nil) + for key, value := range v.headers { + request.Header.Add(key, value) + } + var routeMatch RouteMatch + result := v.matcher.Match(request, &routeMatch) + if result != v.result { + if v.result { + t.Errorf("%#v: should match %v.", v.matcher, request.Header) + } else { + t.Errorf("%#v: should not match %v.", v.matcher, request.Header) + } + } + } +} + +func TestHostMatcher(t *testing.T) { + for _, v := range hostMatcherTests { + request, _ := http.NewRequest("GET", v.url, nil) + var routeMatch RouteMatch + result := v.matcher.Match(request, &routeMatch) + vars := routeMatch.Vars + if result != v.result { + if v.result { + t.Errorf("%#v: should match %v.", v.matcher, v.url) + } else { + t.Errorf("%#v: should not match %v.", v.matcher, v.url) + } + } + if result { + if len(vars) != len(v.vars) { + t.Errorf("%#v: vars length should be %v, got %v.", v.matcher, len(v.vars), len(vars)) + } + for name, value := range vars { + if v.vars[name] != value { + t.Errorf("%#v: expected value %v for key %v, got %v.", v.matcher, v.vars[name], name, value) + } + } + } else { + if len(vars) != 0 { + t.Errorf("%#v: vars length should be 0, got %v.", v.matcher, len(vars)) + } + } + } +} + +func TestMethodMatcher(t *testing.T) { + for _, v := range methodMatcherTests { + request, _ := http.NewRequest(v.method, "http://localhost:8080/", nil) + var routeMatch RouteMatch + result := v.matcher.Match(request, &routeMatch) + if result != v.result { + if v.result { + t.Errorf("%#v: should match %v.", v.matcher, v.method) + } else { + t.Errorf("%#v: should not match %v.", v.matcher, v.method) + } + } + } +} + +func TestPathMatcher(t *testing.T) { + for _, v := range pathMatcherTests { + request, _ := http.NewRequest("GET", v.url, nil) + var routeMatch RouteMatch + result := v.matcher.Match(request, &routeMatch) + vars := routeMatch.Vars + if result != v.result { + if v.result { + t.Errorf("%#v: should match %v.", v.matcher, v.url) + } else { + t.Errorf("%#v: should not match %v.", v.matcher, v.url) + } + } + if result { + if len(vars) != len(v.vars) { + t.Errorf("%#v: vars length should be %v, got %v.", v.matcher, len(v.vars), len(vars)) + } + for name, value := range vars { + if v.vars[name] != value { + t.Errorf("%#v: expected value %v for key %v, got %v.", v.matcher, v.vars[name], name, value) + } + } + } else { + if len(vars) != 0 { + t.Errorf("%#v: vars length should be 0, got %v.", v.matcher, len(vars)) + } + } + } +} + +func TestSchemeMatcher(t *testing.T) { + for _, v := range schemeMatcherTests { + request, _ := http.NewRequest("GET", v.url, nil) + var routeMatch RouteMatch + result := v.matcher.Match(request, &routeMatch) + if result != v.result { + if v.result { + t.Errorf("%#v: should match %v.", v.matcher, v.url) + } else { + t.Errorf("%#v: should not match %v.", v.matcher, v.url) + } + } + } +} + +func TestUrlBuilding(t *testing.T) { + + for _, v := range urlBuildingTests { + u, _ := v.route.URL(v.vars...) + url := u.String() + if url != v.url { + t.Errorf("expected %v, got %v", v.url, url) + /* + reversePath := "" + reverseHost := "" + if v.route.pathTemplate != nil { + reversePath = v.route.pathTemplate.Reverse + } + if v.route.hostTemplate != nil { + reverseHost = v.route.hostTemplate.Reverse + } + + t.Errorf("%#v:\nexpected: %q\ngot: %q\nreverse path: %q\nreverse host: %q", v.route, v.url, url, reversePath, reverseHost) + */ + } + } + + ArticleHandler := func(w http.ResponseWriter, r *http.Request) { + } + + router := NewRouter() + router.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler).Name("article") + + url, _ := router.Get("article").URL("category", "technology", "id", "42") + expected := "/articles/technology/42" + if url.String() != expected { + t.Errorf("Expected %v, got %v", expected, url.String()) + } +} + +func TestMatchedRouteName(t *testing.T) { + routeName := "stock" + router := NewRouter() + route := router.NewRoute().Path("/products/").Name(routeName) + + url := "http://www.example.com/products/" + request, _ := http.NewRequest("GET", url, nil) + var rv RouteMatch + ok := router.Match(request, &rv) + + if !ok || rv.Route != route { + t.Errorf("Expected same route, got %+v.", rv.Route) + } + + retName := rv.Route.GetName() + if retName != routeName { + t.Errorf("Expected %q, got %q.", routeName, retName) + } +} + +func TestSubRouting(t *testing.T) { + // Example from docs. + router := NewRouter() + subrouter := router.NewRoute().Host("www.example.com").Subrouter() + route := subrouter.NewRoute().Path("/products/").Name("products") + + url := "http://www.example.com/products/" + request, _ := http.NewRequest("GET", url, nil) + var rv RouteMatch + ok := router.Match(request, &rv) + + if !ok || rv.Route != route { + t.Errorf("Expected same route, got %+v.", rv.Route) + } + + u, _ := router.Get("products").URL() + builtURL := u.String() + // Yay, subroute aware of the domain when building! + if builtURL != url { + t.Errorf("Expected %q, got %q.", url, builtURL) + } +} + +func TestVariableNames(t *testing.T) { + route := new(Route).Host("{arg1}.domain.com").Path("/{arg1}/{arg2:[0-9]+}") + if route.err == nil { + t.Errorf("Expected error for duplicated variable names") + } +} + +func TestRedirectSlash(t *testing.T) { + var route *Route + var routeMatch RouteMatch + r := NewRouter() + + r.StrictSlash(false) + route = r.NewRoute() + if route.strictSlash != false { + t.Errorf("Expected false redirectSlash.") + } + + r.StrictSlash(true) + route = r.NewRoute() + if route.strictSlash != true { + t.Errorf("Expected true redirectSlash.") + } + + route = new(Route) + route.strictSlash = true + route.Path("/{arg1}/{arg2:[0-9]+}/") + request, _ := http.NewRequest("GET", "http://localhost/foo/123", nil) + routeMatch = RouteMatch{} + _ = route.Match(request, &routeMatch) + vars := routeMatch.Vars + if vars["arg1"] != "foo" { + t.Errorf("Expected foo.") + } + if vars["arg2"] != "123" { + t.Errorf("Expected 123.") + } + rsp := NewRecorder() + routeMatch.Handler.ServeHTTP(rsp, request) + if rsp.HeaderMap.Get("Location") != "http://localhost/foo/123/" { + t.Errorf("Expected redirect header.") + } + + route = new(Route) + route.strictSlash = true + route.Path("/{arg1}/{arg2:[0-9]+}") + request, _ = http.NewRequest("GET", "http://localhost/foo/123/", nil) + routeMatch = RouteMatch{} + _ = route.Match(request, &routeMatch) + vars = routeMatch.Vars + if vars["arg1"] != "foo" { + t.Errorf("Expected foo.") + } + if vars["arg2"] != "123" { + t.Errorf("Expected 123.") + } + rsp = NewRecorder() + routeMatch.Handler.ServeHTTP(rsp, request) + if rsp.HeaderMap.Get("Location") != "http://localhost/foo/123" { + t.Errorf("Expected redirect header.") + } +} + +// Test for the new regexp library, still not available in stable Go. +func TestNewRegexp(t *testing.T) { + var p *routeRegexp + var matches []string + + tests := map[string]map[string][]string{ + "/{foo:a{2}}": { + "/a": nil, + "/aa": {"aa"}, + "/aaa": nil, + "/aaaa": nil, + }, + "/{foo:a{2,}}": { + "/a": nil, + "/aa": {"aa"}, + "/aaa": {"aaa"}, + "/aaaa": {"aaaa"}, + }, + "/{foo:a{2,3}}": { + "/a": nil, + "/aa": {"aa"}, + "/aaa": {"aaa"}, + "/aaaa": nil, + }, + "/{foo:[a-z]{3}}/{bar:[a-z]{2}}": { + "/a": nil, + "/ab": nil, + "/abc": nil, + "/abcd": nil, + "/abc/ab": {"abc", "ab"}, + "/abc/abc": nil, + "/abcd/ab": nil, + }, + `/{foo:\w{3,}}/{bar:\d{2,}}`: { + "/a": nil, + "/ab": nil, + "/abc": nil, + "/abc/1": nil, + "/abc/12": {"abc", "12"}, + "/abcd/12": {"abcd", "12"}, + "/abcd/123": {"abcd", "123"}, + }, + } + + for pattern, paths := range tests { + p, _ = newRouteRegexp(pattern, regexpTypePath, routeRegexpOptions{}) + for path, result := range paths { + matches = p.regexp.FindStringSubmatch(path) + if result == nil { + if matches != nil { + t.Errorf("%v should not match %v.", pattern, path) + } + } else { + if len(matches) != len(result)+1 { + t.Errorf("Expected %v matches, got %v.", len(result)+1, len(matches)) + } else { + for k, v := range result { + if matches[k+1] != v { + t.Errorf("Expected %v, got %v.", v, matches[k+1]) + } + } + } + } + } + } +} diff --git a/vendor/github.com/gorilla/mux/regexp.go b/vendor/github.com/gorilla/mux/regexp.go new file mode 100644 index 000000000..f25288675 --- /dev/null +++ b/vendor/github.com/gorilla/mux/regexp.go @@ -0,0 +1,328 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import ( + "bytes" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" +) + +type routeRegexpOptions struct { + strictSlash bool + useEncodedPath bool +} + +type regexpType int + +const ( + regexpTypePath regexpType = 0 + regexpTypeHost regexpType = 1 + regexpTypePrefix regexpType = 2 + regexpTypeQuery regexpType = 3 +) + +// newRouteRegexp parses a route template and returns a routeRegexp, +// used to match a host, a path or a query string. +// +// It will extract named variables, assemble a regexp to be matched, create +// a "reverse" template to build URLs and compile regexps to validate variable +// values used in URL building. +// +// Previously we accepted only Python-like identifiers for variable +// names ([a-zA-Z_][a-zA-Z0-9_]*), but currently the only restriction is that +// name and pattern can't be empty, and names can't contain a colon. +func newRouteRegexp(tpl string, typ regexpType, options routeRegexpOptions) (*routeRegexp, error) { + // Check if it is well-formed. + idxs, errBraces := braceIndices(tpl) + if errBraces != nil { + return nil, errBraces + } + // Backup the original. + template := tpl + // Now let's parse it. + defaultPattern := "[^/]+" + if typ == regexpTypeQuery { + defaultPattern = ".*" + } else if typ == regexpTypeHost { + defaultPattern = "[^.]+" + } + // Only match strict slash if not matching + if typ != regexpTypePath { + options.strictSlash = false + } + // Set a flag for strictSlash. + endSlash := false + if options.strictSlash && strings.HasSuffix(tpl, "/") { + tpl = tpl[:len(tpl)-1] + endSlash = true + } + varsN := make([]string, len(idxs)/2) + varsR := make([]*regexp.Regexp, len(idxs)/2) + pattern := bytes.NewBufferString("") + pattern.WriteByte('^') + reverse := bytes.NewBufferString("") + var end int + var err error + for i := 0; i < len(idxs); i += 2 { + // Set all values we are interested in. + raw := tpl[end:idxs[i]] + end = idxs[i+1] + parts := strings.SplitN(tpl[idxs[i]+1:end-1], ":", 2) + name := parts[0] + patt := defaultPattern + if len(parts) == 2 { + patt = parts[1] + } + // Name or pattern can't be empty. + if name == "" || patt == "" { + return nil, fmt.Errorf("mux: missing name or pattern in %q", + tpl[idxs[i]:end]) + } + // Build the regexp pattern. + fmt.Fprintf(pattern, "%s(?P<%s>%s)", regexp.QuoteMeta(raw), varGroupName(i/2), patt) + + // Build the reverse template. + fmt.Fprintf(reverse, "%s%%s", raw) + + // Append variable name and compiled pattern. + varsN[i/2] = name + varsR[i/2], err = regexp.Compile(fmt.Sprintf("^%s$", patt)) + if err != nil { + return nil, err + } + } + // Add the remaining. + raw := tpl[end:] + pattern.WriteString(regexp.QuoteMeta(raw)) + if options.strictSlash { + pattern.WriteString("[/]?") + } + if typ == regexpTypeQuery { + // Add the default pattern if the query value is empty + if queryVal := strings.SplitN(template, "=", 2)[1]; queryVal == "" { + pattern.WriteString(defaultPattern) + } + } + if typ != regexpTypePrefix { + pattern.WriteByte('$') + } + reverse.WriteString(raw) + if endSlash { + reverse.WriteByte('/') + } + // Compile full regexp. + reg, errCompile := regexp.Compile(pattern.String()) + if errCompile != nil { + return nil, errCompile + } + + // Check for capturing groups which used to work in older versions + if reg.NumSubexp() != len(idxs)/2 { + panic(fmt.Sprintf("route %s contains capture groups in its regexp. ", template) + + "Only non-capturing groups are accepted: e.g. (?:pattern) instead of (pattern)") + } + + // Done! + return &routeRegexp{ + template: template, + regexpType: typ, + options: options, + regexp: reg, + reverse: reverse.String(), + varsN: varsN, + varsR: varsR, + }, nil +} + +// routeRegexp stores a regexp to match a host or path and information to +// collect and validate route variables. +type routeRegexp struct { + // The unmodified template. + template string + // The type of match + regexpType regexpType + // Options for matching + options routeRegexpOptions + // Expanded regexp. + regexp *regexp.Regexp + // Reverse template. + reverse string + // Variable names. + varsN []string + // Variable regexps (validators). + varsR []*regexp.Regexp +} + +// Match matches the regexp against the URL host or path. +func (r *routeRegexp) Match(req *http.Request, match *RouteMatch) bool { + if r.regexpType != regexpTypeHost { + if r.regexpType == regexpTypeQuery { + return r.matchQueryString(req) + } + path := req.URL.Path + if r.options.useEncodedPath { + path = req.URL.EscapedPath() + } + return r.regexp.MatchString(path) + } + + return r.regexp.MatchString(getHost(req)) +} + +// url builds a URL part using the given values. +func (r *routeRegexp) url(values map[string]string) (string, error) { + urlValues := make([]interface{}, len(r.varsN)) + for k, v := range r.varsN { + value, ok := values[v] + if !ok { + return "", fmt.Errorf("mux: missing route variable %q", v) + } + if r.regexpType == regexpTypeQuery { + value = url.QueryEscape(value) + } + urlValues[k] = value + } + rv := fmt.Sprintf(r.reverse, urlValues...) + if !r.regexp.MatchString(rv) { + // The URL is checked against the full regexp, instead of checking + // individual variables. This is faster but to provide a good error + // message, we check individual regexps if the URL doesn't match. + for k, v := range r.varsN { + if !r.varsR[k].MatchString(values[v]) { + return "", fmt.Errorf( + "mux: variable %q doesn't match, expected %q", values[v], + r.varsR[k].String()) + } + } + } + return rv, nil +} + +// getURLQuery returns a single query parameter from a request URL. +// For a URL with foo=bar&baz=ding, we return only the relevant key +// value pair for the routeRegexp. +func (r *routeRegexp) getURLQuery(req *http.Request) string { + if r.regexpType != regexpTypeQuery { + return "" + } + templateKey := strings.SplitN(r.template, "=", 2)[0] + for key, vals := range req.URL.Query() { + if key == templateKey && len(vals) > 0 { + return key + "=" + vals[0] + } + } + return "" +} + +func (r *routeRegexp) matchQueryString(req *http.Request) bool { + return r.regexp.MatchString(r.getURLQuery(req)) +} + +// braceIndices returns the first level curly brace indices from a string. +// It returns an error in case of unbalanced braces. +func braceIndices(s string) ([]int, error) { + var level, idx int + var idxs []int + for i := 0; i < len(s); i++ { + switch s[i] { + case '{': + if level++; level == 1 { + idx = i + } + case '}': + if level--; level == 0 { + idxs = append(idxs, idx, i+1) + } else if level < 0 { + return nil, fmt.Errorf("mux: unbalanced braces in %q", s) + } + } + } + if level != 0 { + return nil, fmt.Errorf("mux: unbalanced braces in %q", s) + } + return idxs, nil +} + +// varGroupName builds a capturing group name for the indexed variable. +func varGroupName(idx int) string { + return "v" + strconv.Itoa(idx) +} + +// ---------------------------------------------------------------------------- +// routeRegexpGroup +// ---------------------------------------------------------------------------- + +// routeRegexpGroup groups the route matchers that carry variables. +type routeRegexpGroup struct { + host *routeRegexp + path *routeRegexp + queries []*routeRegexp +} + +// setMatch extracts the variables from the URL once a route matches. +func (v routeRegexpGroup) setMatch(req *http.Request, m *RouteMatch, r *Route) { + // Store host variables. + if v.host != nil { + host := getHost(req) + matches := v.host.regexp.FindStringSubmatchIndex(host) + if len(matches) > 0 { + extractVars(host, matches, v.host.varsN, m.Vars) + } + } + path := req.URL.Path + if r.useEncodedPath { + path = req.URL.EscapedPath() + } + // Store path variables. + if v.path != nil { + matches := v.path.regexp.FindStringSubmatchIndex(path) + if len(matches) > 0 { + extractVars(path, matches, v.path.varsN, m.Vars) + // Check if we should redirect. + if v.path.options.strictSlash { + p1 := strings.HasSuffix(path, "/") + p2 := strings.HasSuffix(v.path.template, "/") + if p1 != p2 { + u, _ := url.Parse(req.URL.String()) + if p1 { + u.Path = u.Path[:len(u.Path)-1] + } else { + u.Path += "/" + } + m.Handler = http.RedirectHandler(u.String(), http.StatusMovedPermanently) + } + } + } + } + // Store query string variables. + for _, q := range v.queries { + queryURL := q.getURLQuery(req) + matches := q.regexp.FindStringSubmatchIndex(queryURL) + if len(matches) > 0 { + extractVars(queryURL, matches, q.varsN, m.Vars) + } + } +} + +// getHost tries its best to return the request host. +// According to section 14.23 of RFC 2616 the Host header +// can include the port number if the default value of 80 is not used. +func getHost(r *http.Request) string { + if r.URL.IsAbs() { + return r.URL.Host + } + return r.Host +} + +func extractVars(input string, matches []int, names []string, output map[string]string) { + for i, name := range names { + output[name] = input[matches[2*i+2]:matches[2*i+3]] + } +} diff --git a/vendor/github.com/gorilla/mux/route.go b/vendor/github.com/gorilla/mux/route.go new file mode 100644 index 000000000..8479c68c1 --- /dev/null +++ b/vendor/github.com/gorilla/mux/route.go @@ -0,0 +1,710 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" +) + +// Route stores information to match a request and build URLs. +type Route struct { + // Request handler for the route. + handler http.Handler + // If true, this route never matches: it is only used to build URLs. + buildOnly bool + // The name used to build URLs. + name string + // Error resulted from building a route. + err error + + // "global" reference to all named routes + namedRoutes map[string]*Route + + // config possibly passed in from `Router` + routeConf +} + +// SkipClean reports whether path cleaning is enabled for this route via +// Router.SkipClean. +func (r *Route) SkipClean() bool { + return r.skipClean +} + +// Match matches the route against the request. +func (r *Route) Match(req *http.Request, match *RouteMatch) bool { + if r.buildOnly || r.err != nil { + return false + } + + var matchErr error + + // Match everything. + for _, m := range r.matchers { + if matched := m.Match(req, match); !matched { + if _, ok := m.(methodMatcher); ok { + matchErr = ErrMethodMismatch + continue + } + + // Ignore ErrNotFound errors. These errors arise from match call + // to Subrouters. + // + // This prevents subsequent matching subrouters from failing to + // run middleware. If not ignored, the middleware would see a + // non-nil MatchErr and be skipped, even when there was a + // matching route. + if match.MatchErr == ErrNotFound { + match.MatchErr = nil + } + + matchErr = nil + return false + } + } + + if matchErr != nil { + match.MatchErr = matchErr + return false + } + + if match.MatchErr == ErrMethodMismatch { + // We found a route which matches request method, clear MatchErr + match.MatchErr = nil + // Then override the mis-matched handler + match.Handler = r.handler + } + + // Yay, we have a match. Let's collect some info about it. + if match.Route == nil { + match.Route = r + } + if match.Handler == nil { + match.Handler = r.handler + } + if match.Vars == nil { + match.Vars = make(map[string]string) + } + + // Set variables. + r.regexp.setMatch(req, match, r) + return true +} + +// ---------------------------------------------------------------------------- +// Route attributes +// ---------------------------------------------------------------------------- + +// GetError returns an error resulted from building the route, if any. +func (r *Route) GetError() error { + return r.err +} + +// BuildOnly sets the route to never match: it is only used to build URLs. +func (r *Route) BuildOnly() *Route { + r.buildOnly = true + return r +} + +// Handler -------------------------------------------------------------------- + +// Handler sets a handler for the route. +func (r *Route) Handler(handler http.Handler) *Route { + if r.err == nil { + r.handler = handler + } + return r +} + +// HandlerFunc sets a handler function for the route. +func (r *Route) HandlerFunc(f func(http.ResponseWriter, *http.Request)) *Route { + return r.Handler(http.HandlerFunc(f)) +} + +// GetHandler returns the handler for the route, if any. +func (r *Route) GetHandler() http.Handler { + return r.handler +} + +// Name ----------------------------------------------------------------------- + +// Name sets the name for the route, used to build URLs. +// It is an error to call Name more than once on a route. +func (r *Route) Name(name string) *Route { + if r.name != "" { + r.err = fmt.Errorf("mux: route already has name %q, can't set %q", + r.name, name) + } + if r.err == nil { + r.name = name + r.namedRoutes[name] = r + } + return r +} + +// GetName returns the name for the route, if any. +func (r *Route) GetName() string { + return r.name +} + +// ---------------------------------------------------------------------------- +// Matchers +// ---------------------------------------------------------------------------- + +// matcher types try to match a request. +type matcher interface { + Match(*http.Request, *RouteMatch) bool +} + +// addMatcher adds a matcher to the route. +func (r *Route) addMatcher(m matcher) *Route { + if r.err == nil { + r.matchers = append(r.matchers, m) + } + return r +} + +// addRegexpMatcher adds a host or path matcher and builder to a route. +func (r *Route) addRegexpMatcher(tpl string, typ regexpType) error { + if r.err != nil { + return r.err + } + if typ == regexpTypePath || typ == regexpTypePrefix { + if len(tpl) > 0 && tpl[0] != '/' { + return fmt.Errorf("mux: path must start with a slash, got %q", tpl) + } + if r.regexp.path != nil { + tpl = strings.TrimRight(r.regexp.path.template, "/") + tpl + } + } + rr, err := newRouteRegexp(tpl, typ, routeRegexpOptions{ + strictSlash: r.strictSlash, + useEncodedPath: r.useEncodedPath, + }) + if err != nil { + return err + } + for _, q := range r.regexp.queries { + if err = uniqueVars(rr.varsN, q.varsN); err != nil { + return err + } + } + if typ == regexpTypeHost { + if r.regexp.path != nil { + if err = uniqueVars(rr.varsN, r.regexp.path.varsN); err != nil { + return err + } + } + r.regexp.host = rr + } else { + if r.regexp.host != nil { + if err = uniqueVars(rr.varsN, r.regexp.host.varsN); err != nil { + return err + } + } + if typ == regexpTypeQuery { + r.regexp.queries = append(r.regexp.queries, rr) + } else { + r.regexp.path = rr + } + } + r.addMatcher(rr) + return nil +} + +// Headers -------------------------------------------------------------------- + +// headerMatcher matches the request against header values. +type headerMatcher map[string]string + +func (m headerMatcher) Match(r *http.Request, match *RouteMatch) bool { + return matchMapWithString(m, r.Header, true) +} + +// Headers adds a matcher for request header values. +// It accepts a sequence of key/value pairs to be matched. For example: +// +// r := mux.NewRouter() +// r.Headers("Content-Type", "application/json", +// "X-Requested-With", "XMLHttpRequest") +// +// The above route will only match if both request header values match. +// If the value is an empty string, it will match any value if the key is set. +func (r *Route) Headers(pairs ...string) *Route { + if r.err == nil { + var headers map[string]string + headers, r.err = mapFromPairsToString(pairs...) + return r.addMatcher(headerMatcher(headers)) + } + return r +} + +// headerRegexMatcher matches the request against the route given a regex for the header +type headerRegexMatcher map[string]*regexp.Regexp + +func (m headerRegexMatcher) Match(r *http.Request, match *RouteMatch) bool { + return matchMapWithRegex(m, r.Header, true) +} + +// HeadersRegexp accepts a sequence of key/value pairs, where the value has regex +// support. For example: +// +// r := mux.NewRouter() +// r.HeadersRegexp("Content-Type", "application/(text|json)", +// "X-Requested-With", "XMLHttpRequest") +// +// The above route will only match if both the request header matches both regular expressions. +// If the value is an empty string, it will match any value if the key is set. +// Use the start and end of string anchors (^ and $) to match an exact value. +func (r *Route) HeadersRegexp(pairs ...string) *Route { + if r.err == nil { + var headers map[string]*regexp.Regexp + headers, r.err = mapFromPairsToRegex(pairs...) + return r.addMatcher(headerRegexMatcher(headers)) + } + return r +} + +// Host ----------------------------------------------------------------------- + +// Host adds a matcher for the URL host. +// It accepts a template with zero or more URL variables enclosed by {}. +// Variables can define an optional regexp pattern to be matched: +// +// - {name} matches anything until the next dot. +// +// - {name:pattern} matches the given regexp pattern. +// +// For example: +// +// r := mux.NewRouter() +// r.Host("www.example.com") +// r.Host("{subdomain}.domain.com") +// r.Host("{subdomain:[a-z]+}.domain.com") +// +// Variable names must be unique in a given route. They can be retrieved +// calling mux.Vars(request). +func (r *Route) Host(tpl string) *Route { + r.err = r.addRegexpMatcher(tpl, regexpTypeHost) + return r +} + +// MatcherFunc ---------------------------------------------------------------- + +// MatcherFunc is the function signature used by custom matchers. +type MatcherFunc func(*http.Request, *RouteMatch) bool + +// Match returns the match for a given request. +func (m MatcherFunc) Match(r *http.Request, match *RouteMatch) bool { + return m(r, match) +} + +// MatcherFunc adds a custom function to be used as request matcher. +func (r *Route) MatcherFunc(f MatcherFunc) *Route { + return r.addMatcher(f) +} + +// Methods -------------------------------------------------------------------- + +// methodMatcher matches the request against HTTP methods. +type methodMatcher []string + +func (m methodMatcher) Match(r *http.Request, match *RouteMatch) bool { + return matchInArray(m, r.Method) +} + +// Methods adds a matcher for HTTP methods. +// It accepts a sequence of one or more methods to be matched, e.g.: +// "GET", "POST", "PUT". +func (r *Route) Methods(methods ...string) *Route { + for k, v := range methods { + methods[k] = strings.ToUpper(v) + } + return r.addMatcher(methodMatcher(methods)) +} + +// Path ----------------------------------------------------------------------- + +// Path adds a matcher for the URL path. +// It accepts a template with zero or more URL variables enclosed by {}. The +// template must start with a "/". +// Variables can define an optional regexp pattern to be matched: +// +// - {name} matches anything until the next slash. +// +// - {name:pattern} matches the given regexp pattern. +// +// For example: +// +// r := mux.NewRouter() +// r.Path("/products/").Handler(ProductsHandler) +// r.Path("/products/{key}").Handler(ProductsHandler) +// r.Path("/articles/{category}/{id:[0-9]+}"). +// Handler(ArticleHandler) +// +// Variable names must be unique in a given route. They can be retrieved +// calling mux.Vars(request). +func (r *Route) Path(tpl string) *Route { + r.err = r.addRegexpMatcher(tpl, regexpTypePath) + return r +} + +// PathPrefix ----------------------------------------------------------------- + +// PathPrefix adds a matcher for the URL path prefix. This matches if the given +// template is a prefix of the full URL path. See Route.Path() for details on +// the tpl argument. +// +// Note that it does not treat slashes specially ("/foobar/" will be matched by +// the prefix "/foo") so you may want to use a trailing slash here. +// +// Also note that the setting of Router.StrictSlash() has no effect on routes +// with a PathPrefix matcher. +func (r *Route) PathPrefix(tpl string) *Route { + r.err = r.addRegexpMatcher(tpl, regexpTypePrefix) + return r +} + +// Query ---------------------------------------------------------------------- + +// Queries adds a matcher for URL query values. +// It accepts a sequence of key/value pairs. Values may define variables. +// For example: +// +// r := mux.NewRouter() +// r.Queries("foo", "bar", "id", "{id:[0-9]+}") +// +// The above route will only match if the URL contains the defined queries +// values, e.g.: ?foo=bar&id=42. +// +// If the value is an empty string, it will match any value if the key is set. +// +// Variables can define an optional regexp pattern to be matched: +// +// - {name} matches anything until the next slash. +// +// - {name:pattern} matches the given regexp pattern. +func (r *Route) Queries(pairs ...string) *Route { + length := len(pairs) + if length%2 != 0 { + r.err = fmt.Errorf( + "mux: number of parameters must be multiple of 2, got %v", pairs) + return nil + } + for i := 0; i < length; i += 2 { + if r.err = r.addRegexpMatcher(pairs[i]+"="+pairs[i+1], regexpTypeQuery); r.err != nil { + return r + } + } + + return r +} + +// Schemes -------------------------------------------------------------------- + +// schemeMatcher matches the request against URL schemes. +type schemeMatcher []string + +func (m schemeMatcher) Match(r *http.Request, match *RouteMatch) bool { + return matchInArray(m, r.URL.Scheme) +} + +// Schemes adds a matcher for URL schemes. +// It accepts a sequence of schemes to be matched, e.g.: "http", "https". +func (r *Route) Schemes(schemes ...string) *Route { + for k, v := range schemes { + schemes[k] = strings.ToLower(v) + } + if len(schemes) > 0 { + r.buildScheme = schemes[0] + } + return r.addMatcher(schemeMatcher(schemes)) +} + +// BuildVarsFunc -------------------------------------------------------------- + +// BuildVarsFunc is the function signature used by custom build variable +// functions (which can modify route variables before a route's URL is built). +type BuildVarsFunc func(map[string]string) map[string]string + +// BuildVarsFunc adds a custom function to be used to modify build variables +// before a route's URL is built. +func (r *Route) BuildVarsFunc(f BuildVarsFunc) *Route { + if r.buildVarsFunc != nil { + // compose the old and new functions + old := r.buildVarsFunc + r.buildVarsFunc = func(m map[string]string) map[string]string { + return f(old(m)) + } + } else { + r.buildVarsFunc = f + } + return r +} + +// Subrouter ------------------------------------------------------------------ + +// Subrouter creates a subrouter for the route. +// +// It will test the inner routes only if the parent route matched. For example: +// +// r := mux.NewRouter() +// s := r.Host("www.example.com").Subrouter() +// s.HandleFunc("/products/", ProductsHandler) +// s.HandleFunc("/products/{key}", ProductHandler) +// s.HandleFunc("/articles/{category}/{id:[0-9]+}"), ArticleHandler) +// +// Here, the routes registered in the subrouter won't be tested if the host +// doesn't match. +func (r *Route) Subrouter() *Router { + // initialize a subrouter with a copy of the parent route's configuration + router := &Router{routeConf: copyRouteConf(r.routeConf), namedRoutes: r.namedRoutes} + r.addMatcher(router) + return router +} + +// ---------------------------------------------------------------------------- +// URL building +// ---------------------------------------------------------------------------- + +// URL builds a URL for the route. +// +// It accepts a sequence of key/value pairs for the route variables. For +// example, given this route: +// +// r := mux.NewRouter() +// r.HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). +// Name("article") +// +// ...a URL for it can be built using: +// +// url, err := r.Get("article").URL("category", "technology", "id", "42") +// +// ...which will return an url.URL with the following path: +// +// "/articles/technology/42" +// +// This also works for host variables: +// +// r := mux.NewRouter() +// r.Host("{subdomain}.domain.com"). +// HandleFunc("/articles/{category}/{id:[0-9]+}", ArticleHandler). +// Name("article") +// +// // url.String() will be "http://news.domain.com/articles/technology/42" +// url, err := r.Get("article").URL("subdomain", "news", +// "category", "technology", +// "id", "42") +// +// All variables defined in the route are required, and their values must +// conform to the corresponding patterns. +func (r *Route) URL(pairs ...string) (*url.URL, error) { + if r.err != nil { + return nil, r.err + } + values, err := r.prepareVars(pairs...) + if err != nil { + return nil, err + } + var scheme, host, path string + queries := make([]string, 0, len(r.regexp.queries)) + if r.regexp.host != nil { + if host, err = r.regexp.host.url(values); err != nil { + return nil, err + } + scheme = "http" + if r.buildScheme != "" { + scheme = r.buildScheme + } + } + if r.regexp.path != nil { + if path, err = r.regexp.path.url(values); err != nil { + return nil, err + } + } + for _, q := range r.regexp.queries { + var query string + if query, err = q.url(values); err != nil { + return nil, err + } + queries = append(queries, query) + } + return &url.URL{ + Scheme: scheme, + Host: host, + Path: path, + RawQuery: strings.Join(queries, "&"), + }, nil +} + +// URLHost builds the host part of the URL for a route. See Route.URL(). +// +// The route must have a host defined. +func (r *Route) URLHost(pairs ...string) (*url.URL, error) { + if r.err != nil { + return nil, r.err + } + if r.regexp.host == nil { + return nil, errors.New("mux: route doesn't have a host") + } + values, err := r.prepareVars(pairs...) + if err != nil { + return nil, err + } + host, err := r.regexp.host.url(values) + if err != nil { + return nil, err + } + u := &url.URL{ + Scheme: "http", + Host: host, + } + if r.buildScheme != "" { + u.Scheme = r.buildScheme + } + return u, nil +} + +// URLPath builds the path part of the URL for a route. See Route.URL(). +// +// The route must have a path defined. +func (r *Route) URLPath(pairs ...string) (*url.URL, error) { + if r.err != nil { + return nil, r.err + } + if r.regexp.path == nil { + return nil, errors.New("mux: route doesn't have a path") + } + values, err := r.prepareVars(pairs...) + if err != nil { + return nil, err + } + path, err := r.regexp.path.url(values) + if err != nil { + return nil, err + } + return &url.URL{ + Path: path, + }, nil +} + +// GetPathTemplate returns the template used to build the +// route match. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not define a path. +func (r *Route) GetPathTemplate() (string, error) { + if r.err != nil { + return "", r.err + } + if r.regexp.path == nil { + return "", errors.New("mux: route doesn't have a path") + } + return r.regexp.path.template, nil +} + +// GetPathRegexp returns the expanded regular expression used to match route path. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not define a path. +func (r *Route) GetPathRegexp() (string, error) { + if r.err != nil { + return "", r.err + } + if r.regexp.path == nil { + return "", errors.New("mux: route does not have a path") + } + return r.regexp.path.regexp.String(), nil +} + +// GetQueriesRegexp returns the expanded regular expressions used to match the +// route queries. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not have queries. +func (r *Route) GetQueriesRegexp() ([]string, error) { + if r.err != nil { + return nil, r.err + } + if r.regexp.queries == nil { + return nil, errors.New("mux: route doesn't have queries") + } + var queries []string + for _, query := range r.regexp.queries { + queries = append(queries, query.regexp.String()) + } + return queries, nil +} + +// GetQueriesTemplates returns the templates used to build the +// query matching. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not define queries. +func (r *Route) GetQueriesTemplates() ([]string, error) { + if r.err != nil { + return nil, r.err + } + if r.regexp.queries == nil { + return nil, errors.New("mux: route doesn't have queries") + } + var queries []string + for _, query := range r.regexp.queries { + queries = append(queries, query.template) + } + return queries, nil +} + +// GetMethods returns the methods the route matches against +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if route does not have methods. +func (r *Route) GetMethods() ([]string, error) { + if r.err != nil { + return nil, r.err + } + for _, m := range r.matchers { + if methods, ok := m.(methodMatcher); ok { + return []string(methods), nil + } + } + return nil, errors.New("mux: route doesn't have methods") +} + +// GetHostTemplate returns the template used to build the +// route match. +// This is useful for building simple REST API documentation and for instrumentation +// against third-party services. +// An error will be returned if the route does not define a host. +func (r *Route) GetHostTemplate() (string, error) { + if r.err != nil { + return "", r.err + } + if r.regexp.host == nil { + return "", errors.New("mux: route doesn't have a host") + } + return r.regexp.host.template, nil +} + +// prepareVars converts the route variable pairs into a map. If the route has a +// BuildVarsFunc, it is invoked. +func (r *Route) prepareVars(pairs ...string) (map[string]string, error) { + m, err := mapFromPairsToString(pairs...) + if err != nil { + return nil, err + } + return r.buildVars(m), nil +} + +func (r *Route) buildVars(m map[string]string) map[string]string { + if r.buildVarsFunc != nil { + m = r.buildVarsFunc(m) + } + return m +} diff --git a/vendor/github.com/gorilla/mux/test_helpers.go b/vendor/github.com/gorilla/mux/test_helpers.go new file mode 100644 index 000000000..32ecffde4 --- /dev/null +++ b/vendor/github.com/gorilla/mux/test_helpers.go @@ -0,0 +1,19 @@ +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mux + +import "net/http" + +// SetURLVars sets the URL variables for the given request, to be accessed via +// mux.Vars for testing route behaviour. Arguments are not modified, a shallow +// copy is returned. +// +// This API should only be used for testing purposes; it provides a way to +// inject variables into the request context. Alternatively, URL variables +// can be set by making a route that captures the required variables, +// starting a server and sending the request to that server. +func SetURLVars(r *http.Request, val map[string]string) *http.Request { + return setVars(r, val) +} diff --git a/vendor/github.com/gorilla/websocket/.gitignore b/vendor/github.com/gorilla/websocket/.gitignore new file mode 100644 index 000000000..cd3fcd1ef --- /dev/null +++ b/vendor/github.com/gorilla/websocket/.gitignore @@ -0,0 +1,25 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +.idea/ +*.iml diff --git a/vendor/github.com/gorilla/websocket/.travis.yml b/vendor/github.com/gorilla/websocket/.travis.yml new file mode 100644 index 000000000..a49db51c4 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/.travis.yml @@ -0,0 +1,19 @@ +language: go +sudo: false + +matrix: + include: + - go: 1.7.x + - go: 1.8.x + - go: 1.9.x + - go: 1.10.x + - go: 1.11.x + - go: tip + allow_failures: + - go: tip + +script: + - go get -t -v ./... + - diff -u <(echo -n) <(gofmt -d .) + - go vet $(go list ./... | grep -v /vendor/) + - go test -v -race ./... diff --git a/vendor/github.com/gorilla/websocket/AUTHORS b/vendor/github.com/gorilla/websocket/AUTHORS new file mode 100644 index 000000000..1931f4006 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/AUTHORS @@ -0,0 +1,9 @@ +# This is the official list of Gorilla WebSocket authors for copyright +# purposes. +# +# Please keep the list sorted. + +Gary Burd +Google LLC (https://opensource.google.com/) +Joachim Bauch + diff --git a/vendor/github.com/gorilla/websocket/LICENSE b/vendor/github.com/gorilla/websocket/LICENSE new file mode 100644 index 000000000..9171c9722 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/gorilla/websocket/README.md b/vendor/github.com/gorilla/websocket/README.md new file mode 100644 index 000000000..24696947e --- /dev/null +++ b/vendor/github.com/gorilla/websocket/README.md @@ -0,0 +1,64 @@ +# Gorilla WebSocket + +Gorilla WebSocket is a [Go](http://golang.org/) implementation of the +[WebSocket](http://www.rfc-editor.org/rfc/rfc6455.txt) protocol. + +[![Build Status](https://travis-ci.org/gorilla/websocket.svg?branch=master)](https://travis-ci.org/gorilla/websocket) +[![GoDoc](https://godoc.org/github.com/gorilla/websocket?status.svg)](https://godoc.org/github.com/gorilla/websocket) + +### Documentation + +* [API Reference](http://godoc.org/github.com/gorilla/websocket) +* [Chat example](https://github.com/gorilla/websocket/tree/master/examples/chat) +* [Command example](https://github.com/gorilla/websocket/tree/master/examples/command) +* [Client and server example](https://github.com/gorilla/websocket/tree/master/examples/echo) +* [File watch example](https://github.com/gorilla/websocket/tree/master/examples/filewatch) + +### Status + +The Gorilla WebSocket package provides a complete and tested implementation of +the [WebSocket](http://www.rfc-editor.org/rfc/rfc6455.txt) protocol. The +package API is stable. + +### Installation + + go get github.com/gorilla/websocket + +### Protocol Compliance + +The Gorilla WebSocket package passes the server tests in the [Autobahn Test +Suite](https://github.com/crossbario/autobahn-testsuite) using the application in the [examples/autobahn +subdirectory](https://github.com/gorilla/websocket/tree/master/examples/autobahn). + +### Gorilla WebSocket compared with other packages + + + + + + + + + + + + + + + + + + +
github.com/gorillagolang.org/x/net
RFC 6455 Features
Passes Autobahn Test SuiteYesNo
Receive fragmented messageYesNo, see note 1
Send close messageYesNo
Send pings and receive pongsYesNo
Get the type of a received data messageYesYes, see note 2
Other Features
Compression ExtensionsExperimentalNo
Read message using io.ReaderYesNo, see note 3
Write message using io.WriteCloserYesNo, see note 3
+ +Notes: + +1. Large messages are fragmented in [Chrome's new WebSocket implementation](http://www.ietf.org/mail-archive/web/hybi/current/msg10503.html). +2. The application can get the type of a received data message by implementing + a [Codec marshal](http://godoc.org/golang.org/x/net/websocket#Codec.Marshal) + function. +3. The go.net io.Reader and io.Writer operate across WebSocket frame boundaries. + Read returns when the input buffer is full or a frame boundary is + encountered. Each call to Write sends a single frame message. The Gorilla + io.Reader and io.WriteCloser operate on a single WebSocket message. + diff --git a/vendor/github.com/gorilla/websocket/client.go b/vendor/github.com/gorilla/websocket/client.go new file mode 100644 index 000000000..962c06a39 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client.go @@ -0,0 +1,395 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httptrace" + "net/url" + "strings" + "time" +) + +// ErrBadHandshake is returned when the server response to opening handshake is +// invalid. +var ErrBadHandshake = errors.New("websocket: bad handshake") + +var errInvalidCompression = errors.New("websocket: invalid compression negotiation") + +// NewClient creates a new client connection using the given net connection. +// The URL u specifies the host and request URI. Use requestHeader to specify +// the origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies +// (Cookie). Use the response.Header to get the selected subprotocol +// (Sec-WebSocket-Protocol) and cookies (Set-Cookie). +// +// If the WebSocket handshake fails, ErrBadHandshake is returned along with a +// non-nil *http.Response so that callers can handle redirects, authentication, +// etc. +// +// Deprecated: Use Dialer instead. +func NewClient(netConn net.Conn, u *url.URL, requestHeader http.Header, readBufSize, writeBufSize int) (c *Conn, response *http.Response, err error) { + d := Dialer{ + ReadBufferSize: readBufSize, + WriteBufferSize: writeBufSize, + NetDial: func(net, addr string) (net.Conn, error) { + return netConn, nil + }, + } + return d.Dial(u.String(), requestHeader) +} + +// A Dialer contains options for connecting to WebSocket server. +type Dialer struct { + // NetDial specifies the dial function for creating TCP connections. If + // NetDial is nil, net.Dial is used. + NetDial func(network, addr string) (net.Conn, error) + + // NetDialContext specifies the dial function for creating TCP connections. If + // NetDialContext is nil, net.DialContext is used. + NetDialContext func(ctx context.Context, network, addr string) (net.Conn, error) + + // Proxy specifies a function to return a proxy for a given + // Request. If the function returns a non-nil error, the + // request is aborted with the provided error. + // If Proxy is nil or returns a nil *URL, no proxy is used. + Proxy func(*http.Request) (*url.URL, error) + + // TLSClientConfig specifies the TLS configuration to use with tls.Client. + // If nil, the default configuration is used. + TLSClientConfig *tls.Config + + // HandshakeTimeout specifies the duration for the handshake to complete. + HandshakeTimeout time.Duration + + // ReadBufferSize and WriteBufferSize specify I/O buffer sizes in bytes. If a buffer + // size is zero, then a useful default size is used. The I/O buffer sizes + // do not limit the size of the messages that can be sent or received. + ReadBufferSize, WriteBufferSize int + + // WriteBufferPool is a pool of buffers for write operations. If the value + // is not set, then write buffers are allocated to the connection for the + // lifetime of the connection. + // + // A pool is most useful when the application has a modest volume of writes + // across a large number of connections. + // + // Applications should use a single pool for each unique value of + // WriteBufferSize. + WriteBufferPool BufferPool + + // Subprotocols specifies the client's requested subprotocols. + Subprotocols []string + + // EnableCompression specifies if the client should attempt to negotiate + // per message compression (RFC 7692). Setting this value to true does not + // guarantee that compression will be supported. Currently only "no context + // takeover" modes are supported. + EnableCompression bool + + // Jar specifies the cookie jar. + // If Jar is nil, cookies are not sent in requests and ignored + // in responses. + Jar http.CookieJar +} + +// Dial creates a new client connection by calling DialContext with a background context. +func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Response, error) { + return d.DialContext(context.Background(), urlStr, requestHeader) +} + +var errMalformedURL = errors.New("malformed ws or wss URL") + +func hostPortNoPort(u *url.URL) (hostPort, hostNoPort string) { + hostPort = u.Host + hostNoPort = u.Host + if i := strings.LastIndex(u.Host, ":"); i > strings.LastIndex(u.Host, "]") { + hostNoPort = hostNoPort[:i] + } else { + switch u.Scheme { + case "wss": + hostPort += ":443" + case "https": + hostPort += ":443" + default: + hostPort += ":80" + } + } + return hostPort, hostNoPort +} + +// DefaultDialer is a dialer with all fields set to the default values. +var DefaultDialer = &Dialer{ + Proxy: http.ProxyFromEnvironment, + HandshakeTimeout: 45 * time.Second, +} + +// nilDialer is dialer to use when receiver is nil. +var nilDialer = *DefaultDialer + +// DialContext creates a new client connection. Use requestHeader to specify the +// origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies (Cookie). +// Use the response.Header to get the selected subprotocol +// (Sec-WebSocket-Protocol) and cookies (Set-Cookie). +// +// The context will be used in the request and in the Dialer. +// +// If the WebSocket handshake fails, ErrBadHandshake is returned along with a +// non-nil *http.Response so that callers can handle redirects, authentication, +// etcetera. The response body may not contain the entire response and does not +// need to be closed by the application. +func (d *Dialer) DialContext(ctx context.Context, urlStr string, requestHeader http.Header) (*Conn, *http.Response, error) { + if d == nil { + d = &nilDialer + } + + challengeKey, err := generateChallengeKey() + if err != nil { + return nil, nil, err + } + + u, err := url.Parse(urlStr) + if err != nil { + return nil, nil, err + } + + switch u.Scheme { + case "ws": + u.Scheme = "http" + case "wss": + u.Scheme = "https" + default: + return nil, nil, errMalformedURL + } + + if u.User != nil { + // User name and password are not allowed in websocket URIs. + return nil, nil, errMalformedURL + } + + req := &http.Request{ + Method: "GET", + URL: u, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http.Header), + Host: u.Host, + } + req = req.WithContext(ctx) + + // Set the cookies present in the cookie jar of the dialer + if d.Jar != nil { + for _, cookie := range d.Jar.Cookies(u) { + req.AddCookie(cookie) + } + } + + // Set the request headers using the capitalization for names and values in + // RFC examples. Although the capitalization shouldn't matter, there are + // servers that depend on it. The Header.Set method is not used because the + // method canonicalizes the header names. + req.Header["Upgrade"] = []string{"websocket"} + req.Header["Connection"] = []string{"Upgrade"} + req.Header["Sec-WebSocket-Key"] = []string{challengeKey} + req.Header["Sec-WebSocket-Version"] = []string{"13"} + if len(d.Subprotocols) > 0 { + req.Header["Sec-WebSocket-Protocol"] = []string{strings.Join(d.Subprotocols, ", ")} + } + for k, vs := range requestHeader { + switch { + case k == "Host": + if len(vs) > 0 { + req.Host = vs[0] + } + case k == "Upgrade" || + k == "Connection" || + k == "Sec-Websocket-Key" || + k == "Sec-Websocket-Version" || + k == "Sec-Websocket-Extensions" || + (k == "Sec-Websocket-Protocol" && len(d.Subprotocols) > 0): + return nil, nil, errors.New("websocket: duplicate header not allowed: " + k) + case k == "Sec-Websocket-Protocol": + req.Header["Sec-WebSocket-Protocol"] = vs + default: + req.Header[k] = vs + } + } + + if d.EnableCompression { + req.Header["Sec-WebSocket-Extensions"] = []string{"permessage-deflate; server_no_context_takeover; client_no_context_takeover"} + } + + if d.HandshakeTimeout != 0 { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, d.HandshakeTimeout) + defer cancel() + } + + // Get network dial function. + var netDial func(network, add string) (net.Conn, error) + + if d.NetDialContext != nil { + netDial = func(network, addr string) (net.Conn, error) { + return d.NetDialContext(ctx, network, addr) + } + } else if d.NetDial != nil { + netDial = d.NetDial + } else { + netDialer := &net.Dialer{} + netDial = func(network, addr string) (net.Conn, error) { + return netDialer.DialContext(ctx, network, addr) + } + } + + // If needed, wrap the dial function to set the connection deadline. + if deadline, ok := ctx.Deadline(); ok { + forwardDial := netDial + netDial = func(network, addr string) (net.Conn, error) { + c, err := forwardDial(network, addr) + if err != nil { + return nil, err + } + err = c.SetDeadline(deadline) + if err != nil { + c.Close() + return nil, err + } + return c, nil + } + } + + // If needed, wrap the dial function to connect through a proxy. + if d.Proxy != nil { + proxyURL, err := d.Proxy(req) + if err != nil { + return nil, nil, err + } + if proxyURL != nil { + dialer, err := proxy_FromURL(proxyURL, netDialerFunc(netDial)) + if err != nil { + return nil, nil, err + } + netDial = dialer.Dial + } + } + + hostPort, hostNoPort := hostPortNoPort(u) + trace := httptrace.ContextClientTrace(ctx) + if trace != nil && trace.GetConn != nil { + trace.GetConn(hostPort) + } + + netConn, err := netDial("tcp", hostPort) + if trace != nil && trace.GotConn != nil { + trace.GotConn(httptrace.GotConnInfo{ + Conn: netConn, + }) + } + if err != nil { + return nil, nil, err + } + + defer func() { + if netConn != nil { + netConn.Close() + } + }() + + if u.Scheme == "https" { + cfg := cloneTLSConfig(d.TLSClientConfig) + if cfg.ServerName == "" { + cfg.ServerName = hostNoPort + } + tlsConn := tls.Client(netConn, cfg) + netConn = tlsConn + + var err error + if trace != nil { + err = doHandshakeWithTrace(trace, tlsConn, cfg) + } else { + err = doHandshake(tlsConn, cfg) + } + + if err != nil { + return nil, nil, err + } + } + + conn := newConn(netConn, false, d.ReadBufferSize, d.WriteBufferSize, d.WriteBufferPool, nil, nil) + + if err := req.Write(netConn); err != nil { + return nil, nil, err + } + + if trace != nil && trace.GotFirstResponseByte != nil { + if peek, err := conn.br.Peek(1); err == nil && len(peek) == 1 { + trace.GotFirstResponseByte() + } + } + + resp, err := http.ReadResponse(conn.br, req) + if err != nil { + return nil, nil, err + } + + if d.Jar != nil { + if rc := resp.Cookies(); len(rc) > 0 { + d.Jar.SetCookies(u, rc) + } + } + + if resp.StatusCode != 101 || + !strings.EqualFold(resp.Header.Get("Upgrade"), "websocket") || + !strings.EqualFold(resp.Header.Get("Connection"), "upgrade") || + resp.Header.Get("Sec-Websocket-Accept") != computeAcceptKey(challengeKey) { + // Before closing the network connection on return from this + // function, slurp up some of the response to aid application + // debugging. + buf := make([]byte, 1024) + n, _ := io.ReadFull(resp.Body, buf) + resp.Body = ioutil.NopCloser(bytes.NewReader(buf[:n])) + return nil, resp, ErrBadHandshake + } + + for _, ext := range parseExtensions(resp.Header) { + if ext[""] != "permessage-deflate" { + continue + } + _, snct := ext["server_no_context_takeover"] + _, cnct := ext["client_no_context_takeover"] + if !snct || !cnct { + return nil, resp, errInvalidCompression + } + conn.newCompressionWriter = compressNoContextTakeover + conn.newDecompressionReader = decompressNoContextTakeover + break + } + + resp.Body = ioutil.NopCloser(bytes.NewReader([]byte{})) + conn.subprotocol = resp.Header.Get("Sec-Websocket-Protocol") + + netConn.SetDeadline(time.Time{}) + netConn = nil // to avoid close in defer. + return conn, resp, nil +} + +func doHandshake(tlsConn *tls.Conn, cfg *tls.Config) error { + if err := tlsConn.Handshake(); err != nil { + return err + } + if !cfg.InsecureSkipVerify { + if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/gorilla/websocket/client_clone.go b/vendor/github.com/gorilla/websocket/client_clone.go new file mode 100644 index 000000000..4f0d94372 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client_clone.go @@ -0,0 +1,16 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.8 + +package websocket + +import "crypto/tls" + +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + return cfg.Clone() +} diff --git a/vendor/github.com/gorilla/websocket/client_clone_legacy.go b/vendor/github.com/gorilla/websocket/client_clone_legacy.go new file mode 100644 index 000000000..babb007fb --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client_clone_legacy.go @@ -0,0 +1,38 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.8 + +package websocket + +import "crypto/tls" + +// cloneTLSConfig clones all public fields except the fields +// SessionTicketsDisabled and SessionTicketKey. This avoids copying the +// sync.Mutex in the sync.Once and makes it safe to call cloneTLSConfig on a +// config in active use. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + return &tls.Config{ + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + } +} diff --git a/vendor/github.com/gorilla/websocket/client_server_test.go b/vendor/github.com/gorilla/websocket/client_server_test.go new file mode 100644 index 000000000..8c6d012df --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client_server_test.go @@ -0,0 +1,905 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/http/cookiejar" + "net/http/httptest" + "net/http/httptrace" + "net/url" + "reflect" + "strings" + "testing" + "time" +) + +var cstUpgrader = Upgrader{ + Subprotocols: []string{"p0", "p1"}, + ReadBufferSize: 1024, + WriteBufferSize: 1024, + EnableCompression: true, + Error: func(w http.ResponseWriter, r *http.Request, status int, reason error) { + http.Error(w, reason.Error(), status) + }, +} + +var cstDialer = Dialer{ + Subprotocols: []string{"p1", "p2"}, + ReadBufferSize: 1024, + WriteBufferSize: 1024, + HandshakeTimeout: 30 * time.Second, +} + +type cstHandler struct{ *testing.T } + +type cstServer struct { + *httptest.Server + URL string + t *testing.T +} + +const ( + cstPath = "/a/b" + cstRawQuery = "x=y" + cstRequestURI = cstPath + "?" + cstRawQuery +) + +func newServer(t *testing.T) *cstServer { + var s cstServer + s.Server = httptest.NewServer(cstHandler{t}) + s.Server.URL += cstRequestURI + s.URL = makeWsProto(s.Server.URL) + return &s +} + +func newTLSServer(t *testing.T) *cstServer { + var s cstServer + s.Server = httptest.NewTLSServer(cstHandler{t}) + s.Server.URL += cstRequestURI + s.URL = makeWsProto(s.Server.URL) + return &s +} + +func (t cstHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != cstPath { + t.Logf("path=%v, want %v", r.URL.Path, cstPath) + http.Error(w, "bad path", http.StatusBadRequest) + return + } + if r.URL.RawQuery != cstRawQuery { + t.Logf("query=%v, want %v", r.URL.RawQuery, cstRawQuery) + http.Error(w, "bad path", http.StatusBadRequest) + return + } + subprotos := Subprotocols(r) + if !reflect.DeepEqual(subprotos, cstDialer.Subprotocols) { + t.Logf("subprotols=%v, want %v", subprotos, cstDialer.Subprotocols) + http.Error(w, "bad protocol", http.StatusBadRequest) + return + } + ws, err := cstUpgrader.Upgrade(w, r, http.Header{"Set-Cookie": {"sessionID=1234"}}) + if err != nil { + t.Logf("Upgrade: %v", err) + return + } + defer ws.Close() + + if ws.Subprotocol() != "p1" { + t.Logf("Subprotocol() = %s, want p1", ws.Subprotocol()) + ws.Close() + return + } + op, rd, err := ws.NextReader() + if err != nil { + t.Logf("NextReader: %v", err) + return + } + wr, err := ws.NextWriter(op) + if err != nil { + t.Logf("NextWriter: %v", err) + return + } + if _, err = io.Copy(wr, rd); err != nil { + t.Logf("NextWriter: %v", err) + return + } + if err := wr.Close(); err != nil { + t.Logf("Close: %v", err) + return + } +} + +func makeWsProto(s string) string { + return "ws" + strings.TrimPrefix(s, "http") +} + +func sendRecv(t *testing.T, ws *Conn) { + const message = "Hello World!" + if err := ws.SetWriteDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatalf("SetWriteDeadline: %v", err) + } + if err := ws.WriteMessage(TextMessage, []byte(message)); err != nil { + t.Fatalf("WriteMessage: %v", err) + } + if err := ws.SetReadDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatalf("SetReadDeadline: %v", err) + } + _, p, err := ws.ReadMessage() + if err != nil { + t.Fatalf("ReadMessage: %v", err) + } + if string(p) != message { + t.Fatalf("message=%s, want %s", p, message) + } +} + +func TestProxyDial(t *testing.T) { + + s := newServer(t) + defer s.Close() + + surl, _ := url.Parse(s.Server.URL) + + cstDialer := cstDialer // make local copy for modification on next line. + cstDialer.Proxy = http.ProxyURL(surl) + + connect := false + origHandler := s.Server.Config.Handler + + // Capture the request Host header. + s.Server.Config.Handler = http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == "CONNECT" { + connect = true + w.WriteHeader(http.StatusOK) + return + } + + if !connect { + t.Log("connect not received") + http.Error(w, "connect not received", http.StatusMethodNotAllowed) + return + } + origHandler.ServeHTTP(w, r) + }) + + ws, _, err := cstDialer.Dial(s.URL, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer ws.Close() + sendRecv(t, ws) +} + +func TestProxyAuthorizationDial(t *testing.T) { + s := newServer(t) + defer s.Close() + + surl, _ := url.Parse(s.Server.URL) + surl.User = url.UserPassword("username", "password") + + cstDialer := cstDialer // make local copy for modification on next line. + cstDialer.Proxy = http.ProxyURL(surl) + + connect := false + origHandler := s.Server.Config.Handler + + // Capture the request Host header. + s.Server.Config.Handler = http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + proxyAuth := r.Header.Get("Proxy-Authorization") + expectedProxyAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte("username:password")) + if r.Method == "CONNECT" && proxyAuth == expectedProxyAuth { + connect = true + w.WriteHeader(http.StatusOK) + return + } + + if !connect { + t.Log("connect with proxy authorization not received") + http.Error(w, "connect with proxy authorization not received", http.StatusMethodNotAllowed) + return + } + origHandler.ServeHTTP(w, r) + }) + + ws, _, err := cstDialer.Dial(s.URL, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer ws.Close() + sendRecv(t, ws) +} + +func TestDial(t *testing.T) { + s := newServer(t) + defer s.Close() + + ws, _, err := cstDialer.Dial(s.URL, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer ws.Close() + sendRecv(t, ws) +} + +func TestDialCookieJar(t *testing.T) { + s := newServer(t) + defer s.Close() + + jar, _ := cookiejar.New(nil) + d := cstDialer + d.Jar = jar + + u, _ := url.Parse(s.URL) + + switch u.Scheme { + case "ws": + u.Scheme = "http" + case "wss": + u.Scheme = "https" + } + + cookies := []*http.Cookie{{Name: "gorilla", Value: "ws", Path: "/"}} + d.Jar.SetCookies(u, cookies) + + ws, _, err := d.Dial(s.URL, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer ws.Close() + + var gorilla string + var sessionID string + for _, c := range d.Jar.Cookies(u) { + if c.Name == "gorilla" { + gorilla = c.Value + } + + if c.Name == "sessionID" { + sessionID = c.Value + } + } + if gorilla != "ws" { + t.Error("Cookie not present in jar.") + } + + if sessionID != "1234" { + t.Error("Set-Cookie not received from the server.") + } + + sendRecv(t, ws) +} + +func rootCAs(t *testing.T, s *httptest.Server) *x509.CertPool { + certs := x509.NewCertPool() + for _, c := range s.TLS.Certificates { + roots, err := x509.ParseCertificates(c.Certificate[len(c.Certificate)-1]) + if err != nil { + t.Fatalf("error parsing server's root cert: %v", err) + } + for _, root := range roots { + certs.AddCert(root) + } + } + return certs +} + +func TestDialTLS(t *testing.T) { + s := newTLSServer(t) + defer s.Close() + + d := cstDialer + d.TLSClientConfig = &tls.Config{RootCAs: rootCAs(t, s.Server)} + ws, _, err := d.Dial(s.URL, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer ws.Close() + sendRecv(t, ws) +} + +func TestDialTimeout(t *testing.T) { + s := newServer(t) + defer s.Close() + + d := cstDialer + d.HandshakeTimeout = -1 + ws, _, err := d.Dial(s.URL, nil) + if err == nil { + ws.Close() + t.Fatalf("Dial: nil") + } +} + +// requireDeadlineNetConn fails the current test when Read or Write are called +// with no deadline. +type requireDeadlineNetConn struct { + t *testing.T + c net.Conn + readDeadlineIsSet bool + writeDeadlineIsSet bool +} + +func (c *requireDeadlineNetConn) SetDeadline(t time.Time) error { + c.writeDeadlineIsSet = !t.Equal(time.Time{}) + c.readDeadlineIsSet = c.writeDeadlineIsSet + return c.c.SetDeadline(t) +} + +func (c *requireDeadlineNetConn) SetReadDeadline(t time.Time) error { + c.readDeadlineIsSet = !t.Equal(time.Time{}) + return c.c.SetDeadline(t) +} + +func (c *requireDeadlineNetConn) SetWriteDeadline(t time.Time) error { + c.writeDeadlineIsSet = !t.Equal(time.Time{}) + return c.c.SetDeadline(t) +} + +func (c *requireDeadlineNetConn) Write(p []byte) (int, error) { + if !c.writeDeadlineIsSet { + c.t.Fatalf("write with no deadline") + } + return c.c.Write(p) +} + +func (c *requireDeadlineNetConn) Read(p []byte) (int, error) { + if !c.readDeadlineIsSet { + c.t.Fatalf("read with no deadline") + } + return c.c.Read(p) +} + +func (c *requireDeadlineNetConn) Close() error { return c.c.Close() } +func (c *requireDeadlineNetConn) LocalAddr() net.Addr { return c.c.LocalAddr() } +func (c *requireDeadlineNetConn) RemoteAddr() net.Addr { return c.c.RemoteAddr() } + +func TestHandshakeTimeout(t *testing.T) { + s := newServer(t) + defer s.Close() + + d := cstDialer + d.NetDial = func(n, a string) (net.Conn, error) { + c, err := net.Dial(n, a) + return &requireDeadlineNetConn{c: c, t: t}, err + } + ws, _, err := d.Dial(s.URL, nil) + if err != nil { + t.Fatal("Dial:", err) + } + ws.Close() +} + +func TestHandshakeTimeoutInContext(t *testing.T) { + s := newServer(t) + defer s.Close() + + d := cstDialer + d.HandshakeTimeout = 0 + d.NetDialContext = func(ctx context.Context, n, a string) (net.Conn, error) { + netDialer := &net.Dialer{} + c, err := netDialer.DialContext(ctx, n, a) + return &requireDeadlineNetConn{c: c, t: t}, err + } + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(30*time.Second)) + defer cancel() + ws, _, err := d.DialContext(ctx, s.URL, nil) + if err != nil { + t.Fatal("Dial:", err) + } + ws.Close() +} + +func TestDialBadScheme(t *testing.T) { + s := newServer(t) + defer s.Close() + + ws, _, err := cstDialer.Dial(s.Server.URL, nil) + if err == nil { + ws.Close() + t.Fatalf("Dial: nil") + } +} + +func TestDialBadOrigin(t *testing.T) { + s := newServer(t) + defer s.Close() + + ws, resp, err := cstDialer.Dial(s.URL, http.Header{"Origin": {"bad"}}) + if err == nil { + ws.Close() + t.Fatalf("Dial: nil") + } + if resp == nil { + t.Fatalf("resp=nil, err=%v", err) + } + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status=%d, want %d", resp.StatusCode, http.StatusForbidden) + } +} + +func TestDialBadHeader(t *testing.T) { + s := newServer(t) + defer s.Close() + + for _, k := range []string{"Upgrade", + "Connection", + "Sec-Websocket-Key", + "Sec-Websocket-Version", + "Sec-Websocket-Protocol"} { + h := http.Header{} + h.Set(k, "bad") + ws, _, err := cstDialer.Dial(s.URL, http.Header{"Origin": {"bad"}}) + if err == nil { + ws.Close() + t.Errorf("Dial with header %s returned nil", k) + } + } +} + +func TestBadMethod(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ws, err := cstUpgrader.Upgrade(w, r, nil) + if err == nil { + t.Errorf("handshake succeeded, expect fail") + ws.Close() + } + })) + defer s.Close() + + req, err := http.NewRequest("POST", s.URL, strings.NewReader("")) + if err != nil { + t.Fatalf("NewRequest returned error %v", err) + } + req.Header.Set("Connection", "upgrade") + req.Header.Set("Upgrade", "websocket") + req.Header.Set("Sec-Websocket-Version", "13") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Do returned error %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Errorf("Status = %d, want %d", resp.StatusCode, http.StatusMethodNotAllowed) + } +} + +func TestHandshake(t *testing.T) { + s := newServer(t) + defer s.Close() + + ws, resp, err := cstDialer.Dial(s.URL, http.Header{"Origin": {s.URL}}) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer ws.Close() + + var sessionID string + for _, c := range resp.Cookies() { + if c.Name == "sessionID" { + sessionID = c.Value + } + } + if sessionID != "1234" { + t.Error("Set-Cookie not received from the server.") + } + + if ws.Subprotocol() != "p1" { + t.Errorf("ws.Subprotocol() = %s, want p1", ws.Subprotocol()) + } + sendRecv(t, ws) +} + +func TestRespOnBadHandshake(t *testing.T) { + const expectedStatus = http.StatusGone + const expectedBody = "This is the response body." + + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(expectedStatus) + io.WriteString(w, expectedBody) + })) + defer s.Close() + + ws, resp, err := cstDialer.Dial(makeWsProto(s.URL), nil) + if err == nil { + ws.Close() + t.Fatalf("Dial: nil") + } + + if resp == nil { + t.Fatalf("resp=nil, err=%v", err) + } + + if resp.StatusCode != expectedStatus { + t.Errorf("resp.StatusCode=%d, want %d", resp.StatusCode, expectedStatus) + } + + p, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("ReadFull(resp.Body) returned error %v", err) + } + + if string(p) != expectedBody { + t.Errorf("resp.Body=%s, want %s", p, expectedBody) + } +} + +type testLogWriter struct { + t *testing.T +} + +func (w testLogWriter) Write(p []byte) (int, error) { + w.t.Logf("%s", p) + return len(p), nil +} + +// TestHost tests handling of host names and confirms that it matches net/http. +func TestHost(t *testing.T) { + + upgrader := Upgrader{} + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if IsWebSocketUpgrade(r) { + c, err := upgrader.Upgrade(w, r, http.Header{"X-Test-Host": {r.Host}}) + if err != nil { + t.Fatal(err) + } + c.Close() + } else { + w.Header().Set("X-Test-Host", r.Host) + } + }) + + server := httptest.NewServer(handler) + defer server.Close() + + tlsServer := httptest.NewTLSServer(handler) + defer tlsServer.Close() + + addrs := map[*httptest.Server]string{server: server.Listener.Addr().String(), tlsServer: tlsServer.Listener.Addr().String()} + wsProtos := map[*httptest.Server]string{server: "ws://", tlsServer: "wss://"} + httpProtos := map[*httptest.Server]string{server: "http://", tlsServer: "https://"} + + // Avoid log noise from net/http server by logging to testing.T + server.Config.ErrorLog = log.New(testLogWriter{t}, "", 0) + tlsServer.Config.ErrorLog = server.Config.ErrorLog + + cas := rootCAs(t, tlsServer) + + tests := []struct { + fail bool // true if dial / get should fail + server *httptest.Server // server to use + url string // host for request URI + header string // optional request host header + tls string // optiona host for tls ServerName + wantAddr string // expected host for dial + wantHeader string // expected request header on server + insecureSkipVerify bool + }{ + { + server: server, + url: addrs[server], + wantAddr: addrs[server], + wantHeader: addrs[server], + }, + { + server: tlsServer, + url: addrs[tlsServer], + wantAddr: addrs[tlsServer], + wantHeader: addrs[tlsServer], + }, + + { + server: server, + url: addrs[server], + header: "badhost.com", + wantAddr: addrs[server], + wantHeader: "badhost.com", + }, + { + server: tlsServer, + url: addrs[tlsServer], + header: "badhost.com", + wantAddr: addrs[tlsServer], + wantHeader: "badhost.com", + }, + + { + server: server, + url: "example.com", + header: "badhost.com", + wantAddr: "example.com:80", + wantHeader: "badhost.com", + }, + { + server: tlsServer, + url: "example.com", + header: "badhost.com", + wantAddr: "example.com:443", + wantHeader: "badhost.com", + }, + + { + server: server, + url: "badhost.com", + header: "example.com", + wantAddr: "badhost.com:80", + wantHeader: "example.com", + }, + { + fail: true, + server: tlsServer, + url: "badhost.com", + header: "example.com", + wantAddr: "badhost.com:443", + }, + { + server: tlsServer, + url: "badhost.com", + insecureSkipVerify: true, + wantAddr: "badhost.com:443", + wantHeader: "badhost.com", + }, + { + server: tlsServer, + url: "badhost.com", + tls: "example.com", + wantAddr: "badhost.com:443", + wantHeader: "badhost.com", + }, + } + + for i, tt := range tests { + + tls := &tls.Config{ + RootCAs: cas, + ServerName: tt.tls, + InsecureSkipVerify: tt.insecureSkipVerify, + } + + var gotAddr string + dialer := Dialer{ + NetDial: func(network, addr string) (net.Conn, error) { + gotAddr = addr + return net.Dial(network, addrs[tt.server]) + }, + TLSClientConfig: tls, + } + + // Test websocket dial + + h := http.Header{} + if tt.header != "" { + h.Set("Host", tt.header) + } + c, resp, err := dialer.Dial(wsProtos[tt.server]+tt.url+"/", h) + if err == nil { + c.Close() + } + + check := func(protos map[*httptest.Server]string) { + name := fmt.Sprintf("%d: %s%s/ header[Host]=%q, tls.ServerName=%q", i+1, protos[tt.server], tt.url, tt.header, tt.tls) + if gotAddr != tt.wantAddr { + t.Errorf("%s: got addr %s, want %s", name, gotAddr, tt.wantAddr) + } + switch { + case tt.fail && err == nil: + t.Errorf("%s: unexpected success", name) + case !tt.fail && err != nil: + t.Errorf("%s: unexpected error %v", name, err) + case !tt.fail && err == nil: + if gotHost := resp.Header.Get("X-Test-Host"); gotHost != tt.wantHeader { + t.Errorf("%s: got host %s, want %s", name, gotHost, tt.wantHeader) + } + } + } + + check(wsProtos) + + // Confirm that net/http has same result + + transport := &http.Transport{ + Dial: dialer.NetDial, + TLSClientConfig: dialer.TLSClientConfig, + } + req, _ := http.NewRequest("GET", httpProtos[tt.server]+tt.url+"/", nil) + if tt.header != "" { + req.Host = tt.header + } + client := &http.Client{Transport: transport} + resp, err = client.Do(req) + if err == nil { + resp.Body.Close() + } + transport.CloseIdleConnections() + check(httpProtos) + } +} + +func TestDialCompression(t *testing.T) { + s := newServer(t) + defer s.Close() + + dialer := cstDialer + dialer.EnableCompression = true + ws, _, err := dialer.Dial(s.URL, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer ws.Close() + sendRecv(t, ws) +} + +func TestSocksProxyDial(t *testing.T) { + s := newServer(t) + defer s.Close() + + proxyListener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen failed: %v", err) + } + defer proxyListener.Close() + go func() { + c1, err := proxyListener.Accept() + if err != nil { + t.Errorf("proxy accept failed: %v", err) + return + } + defer c1.Close() + + c1.SetDeadline(time.Now().Add(30 * time.Second)) + + buf := make([]byte, 32) + if _, err := io.ReadFull(c1, buf[:3]); err != nil { + t.Errorf("read failed: %v", err) + return + } + if want := []byte{5, 1, 0}; !bytes.Equal(want, buf[:len(want)]) { + t.Errorf("read %x, want %x", buf[:len(want)], want) + } + if _, err := c1.Write([]byte{5, 0}); err != nil { + t.Errorf("write failed: %v", err) + return + } + if _, err := io.ReadFull(c1, buf[:10]); err != nil { + t.Errorf("read failed: %v", err) + return + } + if want := []byte{5, 1, 0, 1}; !bytes.Equal(want, buf[:len(want)]) { + t.Errorf("read %x, want %x", buf[:len(want)], want) + return + } + buf[1] = 0 + if _, err := c1.Write(buf[:10]); err != nil { + t.Errorf("write failed: %v", err) + return + } + + ip := net.IP(buf[4:8]) + port := binary.BigEndian.Uint16(buf[8:10]) + + c2, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: ip, Port: int(port)}) + if err != nil { + t.Errorf("dial failed; %v", err) + return + } + defer c2.Close() + done := make(chan struct{}) + go func() { + io.Copy(c1, c2) + close(done) + }() + io.Copy(c2, c1) + <-done + }() + + purl, err := url.Parse("socks5://" + proxyListener.Addr().String()) + if err != nil { + t.Fatalf("parse failed: %v", err) + } + + cstDialer := cstDialer // make local copy for modification on next line. + cstDialer.Proxy = http.ProxyURL(purl) + + ws, _, err := cstDialer.Dial(s.URL, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer ws.Close() + sendRecv(t, ws) +} + +func TestTracingDialWithContext(t *testing.T) { + + var headersWrote, requestWrote, getConn, gotConn, connectDone, gotFirstResponseByte bool + trace := &httptrace.ClientTrace{ + WroteHeaders: func() { + headersWrote = true + }, + WroteRequest: func(httptrace.WroteRequestInfo) { + requestWrote = true + }, + GetConn: func(hostPort string) { + getConn = true + }, + GotConn: func(info httptrace.GotConnInfo) { + gotConn = true + }, + ConnectDone: func(network, addr string, err error) { + connectDone = true + }, + GotFirstResponseByte: func() { + gotFirstResponseByte = true + }, + } + ctx := httptrace.WithClientTrace(context.Background(), trace) + + s := newTLSServer(t) + defer s.Close() + + d := cstDialer + d.TLSClientConfig = &tls.Config{RootCAs: rootCAs(t, s.Server)} + + ws, _, err := d.DialContext(ctx, s.URL, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + + if !headersWrote { + t.Fatal("Headers was not written") + } + if !requestWrote { + t.Fatal("Request was not written") + } + if !getConn { + t.Fatal("getConn was not called") + } + if !gotConn { + t.Fatal("gotConn was not called") + } + if !connectDone { + t.Fatal("connectDone was not called") + } + if !gotFirstResponseByte { + t.Fatal("GotFirstResponseByte was not called") + } + + defer ws.Close() + sendRecv(t, ws) +} + +func TestEmptyTracingDialWithContext(t *testing.T) { + + trace := &httptrace.ClientTrace{} + ctx := httptrace.WithClientTrace(context.Background(), trace) + + s := newTLSServer(t) + defer s.Close() + + d := cstDialer + d.TLSClientConfig = &tls.Config{RootCAs: rootCAs(t, s.Server)} + + ws, _, err := d.DialContext(ctx, s.URL, nil) + if err != nil { + t.Fatalf("Dial: %v", err) + } + + defer ws.Close() + sendRecv(t, ws) +} diff --git a/vendor/github.com/gorilla/websocket/client_test.go b/vendor/github.com/gorilla/websocket/client_test.go new file mode 100644 index 000000000..5aa27b37d --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client_test.go @@ -0,0 +1,32 @@ +// Copyright 2014 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "net/url" + "testing" +) + +var hostPortNoPortTests = []struct { + u *url.URL + hostPort, hostNoPort string +}{ + {&url.URL{Scheme: "ws", Host: "example.com"}, "example.com:80", "example.com"}, + {&url.URL{Scheme: "wss", Host: "example.com"}, "example.com:443", "example.com"}, + {&url.URL{Scheme: "ws", Host: "example.com:7777"}, "example.com:7777", "example.com"}, + {&url.URL{Scheme: "wss", Host: "example.com:7777"}, "example.com:7777", "example.com"}, +} + +func TestHostPortNoPort(t *testing.T) { + for _, tt := range hostPortNoPortTests { + hostPort, hostNoPort := hostPortNoPort(tt.u) + if hostPort != tt.hostPort { + t.Errorf("hostPortNoPort(%v) returned hostPort %q, want %q", tt.u, hostPort, tt.hostPort) + } + if hostNoPort != tt.hostNoPort { + t.Errorf("hostPortNoPort(%v) returned hostNoPort %q, want %q", tt.u, hostNoPort, tt.hostNoPort) + } + } +} diff --git a/vendor/github.com/gorilla/websocket/compression.go b/vendor/github.com/gorilla/websocket/compression.go new file mode 100644 index 000000000..813ffb1e8 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/compression.go @@ -0,0 +1,148 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "compress/flate" + "errors" + "io" + "strings" + "sync" +) + +const ( + minCompressionLevel = -2 // flate.HuffmanOnly not defined in Go < 1.6 + maxCompressionLevel = flate.BestCompression + defaultCompressionLevel = 1 +) + +var ( + flateWriterPools [maxCompressionLevel - minCompressionLevel + 1]sync.Pool + flateReaderPool = sync.Pool{New: func() interface{} { + return flate.NewReader(nil) + }} +) + +func decompressNoContextTakeover(r io.Reader) io.ReadCloser { + const tail = + // Add four bytes as specified in RFC + "\x00\x00\xff\xff" + + // Add final block to squelch unexpected EOF error from flate reader. + "\x01\x00\x00\xff\xff" + + fr, _ := flateReaderPool.Get().(io.ReadCloser) + fr.(flate.Resetter).Reset(io.MultiReader(r, strings.NewReader(tail)), nil) + return &flateReadWrapper{fr} +} + +func isValidCompressionLevel(level int) bool { + return minCompressionLevel <= level && level <= maxCompressionLevel +} + +func compressNoContextTakeover(w io.WriteCloser, level int) io.WriteCloser { + p := &flateWriterPools[level-minCompressionLevel] + tw := &truncWriter{w: w} + fw, _ := p.Get().(*flate.Writer) + if fw == nil { + fw, _ = flate.NewWriter(tw, level) + } else { + fw.Reset(tw) + } + return &flateWriteWrapper{fw: fw, tw: tw, p: p} +} + +// truncWriter is an io.Writer that writes all but the last four bytes of the +// stream to another io.Writer. +type truncWriter struct { + w io.WriteCloser + n int + p [4]byte +} + +func (w *truncWriter) Write(p []byte) (int, error) { + n := 0 + + // fill buffer first for simplicity. + if w.n < len(w.p) { + n = copy(w.p[w.n:], p) + p = p[n:] + w.n += n + if len(p) == 0 { + return n, nil + } + } + + m := len(p) + if m > len(w.p) { + m = len(w.p) + } + + if nn, err := w.w.Write(w.p[:m]); err != nil { + return n + nn, err + } + + copy(w.p[:], w.p[m:]) + copy(w.p[len(w.p)-m:], p[len(p)-m:]) + nn, err := w.w.Write(p[:len(p)-m]) + return n + nn, err +} + +type flateWriteWrapper struct { + fw *flate.Writer + tw *truncWriter + p *sync.Pool +} + +func (w *flateWriteWrapper) Write(p []byte) (int, error) { + if w.fw == nil { + return 0, errWriteClosed + } + return w.fw.Write(p) +} + +func (w *flateWriteWrapper) Close() error { + if w.fw == nil { + return errWriteClosed + } + err1 := w.fw.Flush() + w.p.Put(w.fw) + w.fw = nil + if w.tw.p != [4]byte{0, 0, 0xff, 0xff} { + return errors.New("websocket: internal error, unexpected bytes at end of flate stream") + } + err2 := w.tw.w.Close() + if err1 != nil { + return err1 + } + return err2 +} + +type flateReadWrapper struct { + fr io.ReadCloser +} + +func (r *flateReadWrapper) Read(p []byte) (int, error) { + if r.fr == nil { + return 0, io.ErrClosedPipe + } + n, err := r.fr.Read(p) + if err == io.EOF { + // Preemptively place the reader back in the pool. This helps with + // scenarios where the application does not call NextReader() soon after + // this final read. + r.Close() + } + return n, err +} + +func (r *flateReadWrapper) Close() error { + if r.fr == nil { + return io.ErrClosedPipe + } + err := r.fr.Close() + flateReaderPool.Put(r.fr) + r.fr = nil + return err +} diff --git a/vendor/github.com/gorilla/websocket/compression_test.go b/vendor/github.com/gorilla/websocket/compression_test.go new file mode 100644 index 000000000..8a26b30f1 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/compression_test.go @@ -0,0 +1,80 @@ +package websocket + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "testing" +) + +type nopCloser struct{ io.Writer } + +func (nopCloser) Close() error { return nil } + +func TestTruncWriter(t *testing.T) { + const data = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijlkmnopqrstuvwxyz987654321" + for n := 1; n <= 10; n++ { + var b bytes.Buffer + w := &truncWriter{w: nopCloser{&b}} + p := []byte(data) + for len(p) > 0 { + m := len(p) + if m > n { + m = n + } + w.Write(p[:m]) + p = p[m:] + } + if b.String() != data[:len(data)-len(w.p)] { + t.Errorf("%d: %q", n, b.String()) + } + } +} + +func textMessages(num int) [][]byte { + messages := make([][]byte, num) + for i := 0; i < num; i++ { + msg := fmt.Sprintf("planet: %d, country: %d, city: %d, street: %d", i, i, i, i) + messages[i] = []byte(msg) + } + return messages +} + +func BenchmarkWriteNoCompression(b *testing.B) { + w := ioutil.Discard + c := newTestConn(nil, w, false) + messages := textMessages(100) + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.WriteMessage(TextMessage, messages[i%len(messages)]) + } + b.ReportAllocs() +} + +func BenchmarkWriteWithCompression(b *testing.B) { + w := ioutil.Discard + c := newTestConn(nil, w, false) + messages := textMessages(100) + c.enableWriteCompression = true + c.newCompressionWriter = compressNoContextTakeover + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.WriteMessage(TextMessage, messages[i%len(messages)]) + } + b.ReportAllocs() +} + +func TestValidCompressionLevel(t *testing.T) { + c := newTestConn(nil, nil, false) + for _, level := range []int{minCompressionLevel - 1, maxCompressionLevel + 1} { + if err := c.SetCompressionLevel(level); err == nil { + t.Errorf("no error for level %d", level) + } + } + for _, level := range []int{minCompressionLevel, maxCompressionLevel} { + if err := c.SetCompressionLevel(level); err != nil { + t.Errorf("error for level %d", level) + } + } +} diff --git a/vendor/github.com/gorilla/websocket/conn.go b/vendor/github.com/gorilla/websocket/conn.go new file mode 100644 index 000000000..3848ab4a9 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn.go @@ -0,0 +1,1166 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "encoding/binary" + "errors" + "io" + "io/ioutil" + "math/rand" + "net" + "strconv" + "sync" + "time" + "unicode/utf8" +) + +const ( + // Frame header byte 0 bits from Section 5.2 of RFC 6455 + finalBit = 1 << 7 + rsv1Bit = 1 << 6 + rsv2Bit = 1 << 5 + rsv3Bit = 1 << 4 + + // Frame header byte 1 bits from Section 5.2 of RFC 6455 + maskBit = 1 << 7 + + maxFrameHeaderSize = 2 + 8 + 4 // Fixed header + length + mask + maxControlFramePayloadSize = 125 + + writeWait = time.Second + + defaultReadBufferSize = 4096 + defaultWriteBufferSize = 4096 + + continuationFrame = 0 + noFrame = -1 +) + +// Close codes defined in RFC 6455, section 11.7. +const ( + CloseNormalClosure = 1000 + CloseGoingAway = 1001 + CloseProtocolError = 1002 + CloseUnsupportedData = 1003 + CloseNoStatusReceived = 1005 + CloseAbnormalClosure = 1006 + CloseInvalidFramePayloadData = 1007 + ClosePolicyViolation = 1008 + CloseMessageTooBig = 1009 + CloseMandatoryExtension = 1010 + CloseInternalServerErr = 1011 + CloseServiceRestart = 1012 + CloseTryAgainLater = 1013 + CloseTLSHandshake = 1015 +) + +// The message types are defined in RFC 6455, section 11.8. +const ( + // TextMessage denotes a text data message. The text message payload is + // interpreted as UTF-8 encoded text data. + TextMessage = 1 + + // BinaryMessage denotes a binary data message. + BinaryMessage = 2 + + // CloseMessage denotes a close control message. The optional message + // payload contains a numeric code and text. Use the FormatCloseMessage + // function to format a close message payload. + CloseMessage = 8 + + // PingMessage denotes a ping control message. The optional message payload + // is UTF-8 encoded text. + PingMessage = 9 + + // PongMessage denotes a pong control message. The optional message payload + // is UTF-8 encoded text. + PongMessage = 10 +) + +// ErrCloseSent is returned when the application writes a message to the +// connection after sending a close message. +var ErrCloseSent = errors.New("websocket: close sent") + +// ErrReadLimit is returned when reading a message that is larger than the +// read limit set for the connection. +var ErrReadLimit = errors.New("websocket: read limit exceeded") + +// netError satisfies the net Error interface. +type netError struct { + msg string + temporary bool + timeout bool +} + +func (e *netError) Error() string { return e.msg } +func (e *netError) Temporary() bool { return e.temporary } +func (e *netError) Timeout() bool { return e.timeout } + +// CloseError represents a close message. +type CloseError struct { + // Code is defined in RFC 6455, section 11.7. + Code int + + // Text is the optional text payload. + Text string +} + +func (e *CloseError) Error() string { + s := []byte("websocket: close ") + s = strconv.AppendInt(s, int64(e.Code), 10) + switch e.Code { + case CloseNormalClosure: + s = append(s, " (normal)"...) + case CloseGoingAway: + s = append(s, " (going away)"...) + case CloseProtocolError: + s = append(s, " (protocol error)"...) + case CloseUnsupportedData: + s = append(s, " (unsupported data)"...) + case CloseNoStatusReceived: + s = append(s, " (no status)"...) + case CloseAbnormalClosure: + s = append(s, " (abnormal closure)"...) + case CloseInvalidFramePayloadData: + s = append(s, " (invalid payload data)"...) + case ClosePolicyViolation: + s = append(s, " (policy violation)"...) + case CloseMessageTooBig: + s = append(s, " (message too big)"...) + case CloseMandatoryExtension: + s = append(s, " (mandatory extension missing)"...) + case CloseInternalServerErr: + s = append(s, " (internal server error)"...) + case CloseTLSHandshake: + s = append(s, " (TLS handshake error)"...) + } + if e.Text != "" { + s = append(s, ": "...) + s = append(s, e.Text...) + } + return string(s) +} + +// IsCloseError returns boolean indicating whether the error is a *CloseError +// with one of the specified codes. +func IsCloseError(err error, codes ...int) bool { + if e, ok := err.(*CloseError); ok { + for _, code := range codes { + if e.Code == code { + return true + } + } + } + return false +} + +// IsUnexpectedCloseError returns boolean indicating whether the error is a +// *CloseError with a code not in the list of expected codes. +func IsUnexpectedCloseError(err error, expectedCodes ...int) bool { + if e, ok := err.(*CloseError); ok { + for _, code := range expectedCodes { + if e.Code == code { + return false + } + } + return true + } + return false +} + +var ( + errWriteTimeout = &netError{msg: "websocket: write timeout", timeout: true, temporary: true} + errUnexpectedEOF = &CloseError{Code: CloseAbnormalClosure, Text: io.ErrUnexpectedEOF.Error()} + errBadWriteOpCode = errors.New("websocket: bad write message type") + errWriteClosed = errors.New("websocket: write closed") + errInvalidControlFrame = errors.New("websocket: invalid control frame") +) + +func newMaskKey() [4]byte { + n := rand.Uint32() + return [4]byte{byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 24)} +} + +func hideTempErr(err error) error { + if e, ok := err.(net.Error); ok && e.Temporary() { + err = &netError{msg: e.Error(), timeout: e.Timeout()} + } + return err +} + +func isControl(frameType int) bool { + return frameType == CloseMessage || frameType == PingMessage || frameType == PongMessage +} + +func isData(frameType int) bool { + return frameType == TextMessage || frameType == BinaryMessage +} + +var validReceivedCloseCodes = map[int]bool{ + // see http://www.iana.org/assignments/websocket/websocket.xhtml#close-code-number + + CloseNormalClosure: true, + CloseGoingAway: true, + CloseProtocolError: true, + CloseUnsupportedData: true, + CloseNoStatusReceived: false, + CloseAbnormalClosure: false, + CloseInvalidFramePayloadData: true, + ClosePolicyViolation: true, + CloseMessageTooBig: true, + CloseMandatoryExtension: true, + CloseInternalServerErr: true, + CloseServiceRestart: true, + CloseTryAgainLater: true, + CloseTLSHandshake: false, +} + +func isValidReceivedCloseCode(code int) bool { + return validReceivedCloseCodes[code] || (code >= 3000 && code <= 4999) +} + +// BufferPool represents a pool of buffers. The *sync.Pool type satisfies this +// interface. The type of the value stored in a pool is not specified. +type BufferPool interface { + // Get gets a value from the pool or returns nil if the pool is empty. + Get() interface{} + // Put adds a value to the pool. + Put(interface{}) +} + +// writePoolData is the type added to the write buffer pool. This wrapper is +// used to prevent applications from peeking at and depending on the values +// added to the pool. +type writePoolData struct{ buf []byte } + +// The Conn type represents a WebSocket connection. +type Conn struct { + conn net.Conn + isServer bool + subprotocol string + + // Write fields + mu chan bool // used as mutex to protect write to conn + writeBuf []byte // frame is constructed in this buffer. + writePool BufferPool + writeBufSize int + writeDeadline time.Time + writer io.WriteCloser // the current writer returned to the application + isWriting bool // for best-effort concurrent write detection + + writeErrMu sync.Mutex + writeErr error + + enableWriteCompression bool + compressionLevel int + newCompressionWriter func(io.WriteCloser, int) io.WriteCloser + + // Read fields + reader io.ReadCloser // the current reader returned to the application + readErr error + br *bufio.Reader + readRemaining int64 // bytes remaining in current frame. + readFinal bool // true the current message has more frames. + readLength int64 // Message size. + readLimit int64 // Maximum message size. + readMaskPos int + readMaskKey [4]byte + handlePong func(string) error + handlePing func(string) error + handleClose func(int, string) error + readErrCount int + messageReader *messageReader // the current low-level reader + + readDecompress bool // whether last read frame had RSV1 set + newDecompressionReader func(io.Reader) io.ReadCloser +} + +func newConn(conn net.Conn, isServer bool, readBufferSize, writeBufferSize int, writeBufferPool BufferPool, br *bufio.Reader, writeBuf []byte) *Conn { + + if br == nil { + if readBufferSize == 0 { + readBufferSize = defaultReadBufferSize + } else if readBufferSize < maxControlFramePayloadSize { + // must be large enough for control frame + readBufferSize = maxControlFramePayloadSize + } + br = bufio.NewReaderSize(conn, readBufferSize) + } + + if writeBufferSize <= 0 { + writeBufferSize = defaultWriteBufferSize + } + writeBufferSize += maxFrameHeaderSize + + if writeBuf == nil && writeBufferPool == nil { + writeBuf = make([]byte, writeBufferSize) + } + + mu := make(chan bool, 1) + mu <- true + c := &Conn{ + isServer: isServer, + br: br, + conn: conn, + mu: mu, + readFinal: true, + writeBuf: writeBuf, + writePool: writeBufferPool, + writeBufSize: writeBufferSize, + enableWriteCompression: true, + compressionLevel: defaultCompressionLevel, + } + c.SetCloseHandler(nil) + c.SetPingHandler(nil) + c.SetPongHandler(nil) + return c +} + +// Subprotocol returns the negotiated protocol for the connection. +func (c *Conn) Subprotocol() string { + return c.subprotocol +} + +// Close closes the underlying network connection without sending or waiting +// for a close message. +func (c *Conn) Close() error { + return c.conn.Close() +} + +// LocalAddr returns the local network address. +func (c *Conn) LocalAddr() net.Addr { + return c.conn.LocalAddr() +} + +// RemoteAddr returns the remote network address. +func (c *Conn) RemoteAddr() net.Addr { + return c.conn.RemoteAddr() +} + +// Write methods + +func (c *Conn) writeFatal(err error) error { + err = hideTempErr(err) + c.writeErrMu.Lock() + if c.writeErr == nil { + c.writeErr = err + } + c.writeErrMu.Unlock() + return err +} + +func (c *Conn) read(n int) ([]byte, error) { + p, err := c.br.Peek(n) + if err == io.EOF { + err = errUnexpectedEOF + } + c.br.Discard(len(p)) + return p, err +} + +func (c *Conn) write(frameType int, deadline time.Time, buf0, buf1 []byte) error { + <-c.mu + defer func() { c.mu <- true }() + + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err + } + + c.conn.SetWriteDeadline(deadline) + if len(buf1) == 0 { + _, err = c.conn.Write(buf0) + } else { + err = c.writeBufs(buf0, buf1) + } + if err != nil { + return c.writeFatal(err) + } + if frameType == CloseMessage { + c.writeFatal(ErrCloseSent) + } + return nil +} + +// WriteControl writes a control message with the given deadline. The allowed +// message types are CloseMessage, PingMessage and PongMessage. +func (c *Conn) WriteControl(messageType int, data []byte, deadline time.Time) error { + if !isControl(messageType) { + return errBadWriteOpCode + } + if len(data) > maxControlFramePayloadSize { + return errInvalidControlFrame + } + + b0 := byte(messageType) | finalBit + b1 := byte(len(data)) + if !c.isServer { + b1 |= maskBit + } + + buf := make([]byte, 0, maxFrameHeaderSize+maxControlFramePayloadSize) + buf = append(buf, b0, b1) + + if c.isServer { + buf = append(buf, data...) + } else { + key := newMaskKey() + buf = append(buf, key[:]...) + buf = append(buf, data...) + maskBytes(key, 0, buf[6:]) + } + + d := time.Hour * 1000 + if !deadline.IsZero() { + d = deadline.Sub(time.Now()) + if d < 0 { + return errWriteTimeout + } + } + + timer := time.NewTimer(d) + select { + case <-c.mu: + timer.Stop() + case <-timer.C: + return errWriteTimeout + } + defer func() { c.mu <- true }() + + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err + } + + c.conn.SetWriteDeadline(deadline) + _, err = c.conn.Write(buf) + if err != nil { + return c.writeFatal(err) + } + if messageType == CloseMessage { + c.writeFatal(ErrCloseSent) + } + return err +} + +// beginMessage prepares a connection and message writer for a new message. +func (c *Conn) beginMessage(mw *messageWriter, messageType int) error { + // Close previous writer if not already closed by the application. It's + // probably better to return an error in this situation, but we cannot + // change this without breaking existing applications. + if c.writer != nil { + c.writer.Close() + c.writer = nil + } + + if !isControl(messageType) && !isData(messageType) { + return errBadWriteOpCode + } + + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err + } + + mw.c = c + mw.frameType = messageType + mw.pos = maxFrameHeaderSize + + if c.writeBuf == nil { + wpd, ok := c.writePool.Get().(writePoolData) + if ok { + c.writeBuf = wpd.buf + } else { + c.writeBuf = make([]byte, c.writeBufSize) + } + } + return nil +} + +// NextWriter returns a writer for the next message to send. The writer's Close +// method flushes the complete message to the network. +// +// There can be at most one open writer on a connection. NextWriter closes the +// previous writer if the application has not already done so. +// +// All message types (TextMessage, BinaryMessage, CloseMessage, PingMessage and +// PongMessage) are supported. +func (c *Conn) NextWriter(messageType int) (io.WriteCloser, error) { + var mw messageWriter + if err := c.beginMessage(&mw, messageType); err != nil { + return nil, err + } + c.writer = &mw + if c.newCompressionWriter != nil && c.enableWriteCompression && isData(messageType) { + w := c.newCompressionWriter(c.writer, c.compressionLevel) + mw.compress = true + c.writer = w + } + return c.writer, nil +} + +type messageWriter struct { + c *Conn + compress bool // whether next call to flushFrame should set RSV1 + pos int // end of data in writeBuf. + frameType int // type of the current frame. + err error +} + +func (w *messageWriter) endMessage(err error) error { + if w.err != nil { + return err + } + c := w.c + w.err = err + c.writer = nil + if c.writePool != nil { + c.writePool.Put(writePoolData{buf: c.writeBuf}) + c.writeBuf = nil + } + return err +} + +// flushFrame writes buffered data and extra as a frame to the network. The +// final argument indicates that this is the last frame in the message. +func (w *messageWriter) flushFrame(final bool, extra []byte) error { + c := w.c + length := w.pos - maxFrameHeaderSize + len(extra) + + // Check for invalid control frames. + if isControl(w.frameType) && + (!final || length > maxControlFramePayloadSize) { + return w.endMessage(errInvalidControlFrame) + } + + b0 := byte(w.frameType) + if final { + b0 |= finalBit + } + if w.compress { + b0 |= rsv1Bit + } + w.compress = false + + b1 := byte(0) + if !c.isServer { + b1 |= maskBit + } + + // Assume that the frame starts at beginning of c.writeBuf. + framePos := 0 + if c.isServer { + // Adjust up if mask not included in the header. + framePos = 4 + } + + switch { + case length >= 65536: + c.writeBuf[framePos] = b0 + c.writeBuf[framePos+1] = b1 | 127 + binary.BigEndian.PutUint64(c.writeBuf[framePos+2:], uint64(length)) + case length > 125: + framePos += 6 + c.writeBuf[framePos] = b0 + c.writeBuf[framePos+1] = b1 | 126 + binary.BigEndian.PutUint16(c.writeBuf[framePos+2:], uint16(length)) + default: + framePos += 8 + c.writeBuf[framePos] = b0 + c.writeBuf[framePos+1] = b1 | byte(length) + } + + if !c.isServer { + key := newMaskKey() + copy(c.writeBuf[maxFrameHeaderSize-4:], key[:]) + maskBytes(key, 0, c.writeBuf[maxFrameHeaderSize:w.pos]) + if len(extra) > 0 { + return w.endMessage(c.writeFatal(errors.New("websocket: internal error, extra used in client mode"))) + } + } + + // Write the buffers to the connection with best-effort detection of + // concurrent writes. See the concurrency section in the package + // documentation for more info. + + if c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = true + + err := c.write(w.frameType, c.writeDeadline, c.writeBuf[framePos:w.pos], extra) + + if !c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = false + + if err != nil { + return w.endMessage(err) + } + + if final { + w.endMessage(errWriteClosed) + return nil + } + + // Setup for next frame. + w.pos = maxFrameHeaderSize + w.frameType = continuationFrame + return nil +} + +func (w *messageWriter) ncopy(max int) (int, error) { + n := len(w.c.writeBuf) - w.pos + if n <= 0 { + if err := w.flushFrame(false, nil); err != nil { + return 0, err + } + n = len(w.c.writeBuf) - w.pos + } + if n > max { + n = max + } + return n, nil +} + +func (w *messageWriter) Write(p []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + + if len(p) > 2*len(w.c.writeBuf) && w.c.isServer { + // Don't buffer large messages. + err := w.flushFrame(false, p) + if err != nil { + return 0, err + } + return len(p), nil + } + + nn := len(p) + for len(p) > 0 { + n, err := w.ncopy(len(p)) + if err != nil { + return 0, err + } + copy(w.c.writeBuf[w.pos:], p[:n]) + w.pos += n + p = p[n:] + } + return nn, nil +} + +func (w *messageWriter) WriteString(p string) (int, error) { + if w.err != nil { + return 0, w.err + } + + nn := len(p) + for len(p) > 0 { + n, err := w.ncopy(len(p)) + if err != nil { + return 0, err + } + copy(w.c.writeBuf[w.pos:], p[:n]) + w.pos += n + p = p[n:] + } + return nn, nil +} + +func (w *messageWriter) ReadFrom(r io.Reader) (nn int64, err error) { + if w.err != nil { + return 0, w.err + } + for { + if w.pos == len(w.c.writeBuf) { + err = w.flushFrame(false, nil) + if err != nil { + break + } + } + var n int + n, err = r.Read(w.c.writeBuf[w.pos:]) + w.pos += n + nn += int64(n) + if err != nil { + if err == io.EOF { + err = nil + } + break + } + } + return nn, err +} + +func (w *messageWriter) Close() error { + if w.err != nil { + return w.err + } + if err := w.flushFrame(true, nil); err != nil { + return err + } + return nil +} + +// WritePreparedMessage writes prepared message into connection. +func (c *Conn) WritePreparedMessage(pm *PreparedMessage) error { + frameType, frameData, err := pm.frame(prepareKey{ + isServer: c.isServer, + compress: c.newCompressionWriter != nil && c.enableWriteCompression && isData(pm.messageType), + compressionLevel: c.compressionLevel, + }) + if err != nil { + return err + } + if c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = true + err = c.write(frameType, c.writeDeadline, frameData, nil) + if !c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = false + return err +} + +// WriteMessage is a helper method for getting a writer using NextWriter, +// writing the message and closing the writer. +func (c *Conn) WriteMessage(messageType int, data []byte) error { + + if c.isServer && (c.newCompressionWriter == nil || !c.enableWriteCompression) { + // Fast path with no allocations and single frame. + + var mw messageWriter + if err := c.beginMessage(&mw, messageType); err != nil { + return err + } + n := copy(c.writeBuf[mw.pos:], data) + mw.pos += n + data = data[n:] + return mw.flushFrame(true, data) + } + + w, err := c.NextWriter(messageType) + if err != nil { + return err + } + if _, err = w.Write(data); err != nil { + return err + } + return w.Close() +} + +// SetWriteDeadline sets the write deadline on the underlying network +// connection. After a write has timed out, the websocket state is corrupt and +// all future writes will return an error. A zero value for t means writes will +// not time out. +func (c *Conn) SetWriteDeadline(t time.Time) error { + c.writeDeadline = t + return nil +} + +// Read methods + +func (c *Conn) advanceFrame() (int, error) { + // 1. Skip remainder of previous frame. + + if c.readRemaining > 0 { + if _, err := io.CopyN(ioutil.Discard, c.br, c.readRemaining); err != nil { + return noFrame, err + } + } + + // 2. Read and parse first two bytes of frame header. + + p, err := c.read(2) + if err != nil { + return noFrame, err + } + + final := p[0]&finalBit != 0 + frameType := int(p[0] & 0xf) + mask := p[1]&maskBit != 0 + c.readRemaining = int64(p[1] & 0x7f) + + c.readDecompress = false + if c.newDecompressionReader != nil && (p[0]&rsv1Bit) != 0 { + c.readDecompress = true + p[0] &^= rsv1Bit + } + + if rsv := p[0] & (rsv1Bit | rsv2Bit | rsv3Bit); rsv != 0 { + return noFrame, c.handleProtocolError("unexpected reserved bits 0x" + strconv.FormatInt(int64(rsv), 16)) + } + + switch frameType { + case CloseMessage, PingMessage, PongMessage: + if c.readRemaining > maxControlFramePayloadSize { + return noFrame, c.handleProtocolError("control frame length > 125") + } + if !final { + return noFrame, c.handleProtocolError("control frame not final") + } + case TextMessage, BinaryMessage: + if !c.readFinal { + return noFrame, c.handleProtocolError("message start before final message frame") + } + c.readFinal = final + case continuationFrame: + if c.readFinal { + return noFrame, c.handleProtocolError("continuation after final message frame") + } + c.readFinal = final + default: + return noFrame, c.handleProtocolError("unknown opcode " + strconv.Itoa(frameType)) + } + + // 3. Read and parse frame length. + + switch c.readRemaining { + case 126: + p, err := c.read(2) + if err != nil { + return noFrame, err + } + c.readRemaining = int64(binary.BigEndian.Uint16(p)) + case 127: + p, err := c.read(8) + if err != nil { + return noFrame, err + } + c.readRemaining = int64(binary.BigEndian.Uint64(p)) + } + + // 4. Handle frame masking. + + if mask != c.isServer { + return noFrame, c.handleProtocolError("incorrect mask flag") + } + + if mask { + c.readMaskPos = 0 + p, err := c.read(len(c.readMaskKey)) + if err != nil { + return noFrame, err + } + copy(c.readMaskKey[:], p) + } + + // 5. For text and binary messages, enforce read limit and return. + + if frameType == continuationFrame || frameType == TextMessage || frameType == BinaryMessage { + + c.readLength += c.readRemaining + if c.readLimit > 0 && c.readLength > c.readLimit { + c.WriteControl(CloseMessage, FormatCloseMessage(CloseMessageTooBig, ""), time.Now().Add(writeWait)) + return noFrame, ErrReadLimit + } + + return frameType, nil + } + + // 6. Read control frame payload. + + var payload []byte + if c.readRemaining > 0 { + payload, err = c.read(int(c.readRemaining)) + c.readRemaining = 0 + if err != nil { + return noFrame, err + } + if c.isServer { + maskBytes(c.readMaskKey, 0, payload) + } + } + + // 7. Process control frame payload. + + switch frameType { + case PongMessage: + if err := c.handlePong(string(payload)); err != nil { + return noFrame, err + } + case PingMessage: + if err := c.handlePing(string(payload)); err != nil { + return noFrame, err + } + case CloseMessage: + closeCode := CloseNoStatusReceived + closeText := "" + if len(payload) >= 2 { + closeCode = int(binary.BigEndian.Uint16(payload)) + if !isValidReceivedCloseCode(closeCode) { + return noFrame, c.handleProtocolError("invalid close code") + } + closeText = string(payload[2:]) + if !utf8.ValidString(closeText) { + return noFrame, c.handleProtocolError("invalid utf8 payload in close frame") + } + } + if err := c.handleClose(closeCode, closeText); err != nil { + return noFrame, err + } + return noFrame, &CloseError{Code: closeCode, Text: closeText} + } + + return frameType, nil +} + +func (c *Conn) handleProtocolError(message string) error { + c.WriteControl(CloseMessage, FormatCloseMessage(CloseProtocolError, message), time.Now().Add(writeWait)) + return errors.New("websocket: " + message) +} + +// NextReader returns the next data message received from the peer. The +// returned messageType is either TextMessage or BinaryMessage. +// +// There can be at most one open reader on a connection. NextReader discards +// the previous message if the application has not already consumed it. +// +// Applications must break out of the application's read loop when this method +// returns a non-nil error value. Errors returned from this method are +// permanent. Once this method returns a non-nil error, all subsequent calls to +// this method return the same error. +func (c *Conn) NextReader() (messageType int, r io.Reader, err error) { + // Close previous reader, only relevant for decompression. + if c.reader != nil { + c.reader.Close() + c.reader = nil + } + + c.messageReader = nil + c.readLength = 0 + + for c.readErr == nil { + frameType, err := c.advanceFrame() + if err != nil { + c.readErr = hideTempErr(err) + break + } + if frameType == TextMessage || frameType == BinaryMessage { + c.messageReader = &messageReader{c} + c.reader = c.messageReader + if c.readDecompress { + c.reader = c.newDecompressionReader(c.reader) + } + return frameType, c.reader, nil + } + } + + // Applications that do handle the error returned from this method spin in + // tight loop on connection failure. To help application developers detect + // this error, panic on repeated reads to the failed connection. + c.readErrCount++ + if c.readErrCount >= 1000 { + panic("repeated read on failed websocket connection") + } + + return noFrame, nil, c.readErr +} + +type messageReader struct{ c *Conn } + +func (r *messageReader) Read(b []byte) (int, error) { + c := r.c + if c.messageReader != r { + return 0, io.EOF + } + + for c.readErr == nil { + + if c.readRemaining > 0 { + if int64(len(b)) > c.readRemaining { + b = b[:c.readRemaining] + } + n, err := c.br.Read(b) + c.readErr = hideTempErr(err) + if c.isServer { + c.readMaskPos = maskBytes(c.readMaskKey, c.readMaskPos, b[:n]) + } + c.readRemaining -= int64(n) + if c.readRemaining > 0 && c.readErr == io.EOF { + c.readErr = errUnexpectedEOF + } + return n, c.readErr + } + + if c.readFinal { + c.messageReader = nil + return 0, io.EOF + } + + frameType, err := c.advanceFrame() + switch { + case err != nil: + c.readErr = hideTempErr(err) + case frameType == TextMessage || frameType == BinaryMessage: + c.readErr = errors.New("websocket: internal error, unexpected text or binary in Reader") + } + } + + err := c.readErr + if err == io.EOF && c.messageReader == r { + err = errUnexpectedEOF + } + return 0, err +} + +func (r *messageReader) Close() error { + return nil +} + +// ReadMessage is a helper method for getting a reader using NextReader and +// reading from that reader to a buffer. +func (c *Conn) ReadMessage() (messageType int, p []byte, err error) { + var r io.Reader + messageType, r, err = c.NextReader() + if err != nil { + return messageType, nil, err + } + p, err = ioutil.ReadAll(r) + return messageType, p, err +} + +// SetReadDeadline sets the read deadline on the underlying network connection. +// After a read has timed out, the websocket connection state is corrupt and +// all future reads will return an error. A zero value for t means reads will +// not time out. +func (c *Conn) SetReadDeadline(t time.Time) error { + return c.conn.SetReadDeadline(t) +} + +// SetReadLimit sets the maximum size in bytes for a message read from the peer. If a +// message exceeds the limit, the connection sends a close message to the peer +// and returns ErrReadLimit to the application. +func (c *Conn) SetReadLimit(limit int64) { + c.readLimit = limit +} + +// CloseHandler returns the current close handler +func (c *Conn) CloseHandler() func(code int, text string) error { + return c.handleClose +} + +// SetCloseHandler sets the handler for close messages received from the peer. +// The code argument to h is the received close code or CloseNoStatusReceived +// if the close message is empty. The default close handler sends a close +// message back to the peer. +// +// The handler function is called from the NextReader, ReadMessage and message +// reader Read methods. The application must read the connection to process +// close messages as described in the section on Control Messages above. +// +// The connection read methods return a CloseError when a close message is +// received. Most applications should handle close messages as part of their +// normal error handling. Applications should only set a close handler when the +// application must perform some action before sending a close message back to +// the peer. +func (c *Conn) SetCloseHandler(h func(code int, text string) error) { + if h == nil { + h = func(code int, text string) error { + message := FormatCloseMessage(code, "") + c.WriteControl(CloseMessage, message, time.Now().Add(writeWait)) + return nil + } + } + c.handleClose = h +} + +// PingHandler returns the current ping handler +func (c *Conn) PingHandler() func(appData string) error { + return c.handlePing +} + +// SetPingHandler sets the handler for ping messages received from the peer. +// The appData argument to h is the PING message application data. The default +// ping handler sends a pong to the peer. +// +// The handler function is called from the NextReader, ReadMessage and message +// reader Read methods. The application must read the connection to process +// ping messages as described in the section on Control Messages above. +func (c *Conn) SetPingHandler(h func(appData string) error) { + if h == nil { + h = func(message string) error { + err := c.WriteControl(PongMessage, []byte(message), time.Now().Add(writeWait)) + if err == ErrCloseSent { + return nil + } else if e, ok := err.(net.Error); ok && e.Temporary() { + return nil + } + return err + } + } + c.handlePing = h +} + +// PongHandler returns the current pong handler +func (c *Conn) PongHandler() func(appData string) error { + return c.handlePong +} + +// SetPongHandler sets the handler for pong messages received from the peer. +// The appData argument to h is the PONG message application data. The default +// pong handler does nothing. +// +// The handler function is called from the NextReader, ReadMessage and message +// reader Read methods. The application must read the connection to process +// pong messages as described in the section on Control Messages above. +func (c *Conn) SetPongHandler(h func(appData string) error) { + if h == nil { + h = func(string) error { return nil } + } + c.handlePong = h +} + +// UnderlyingConn returns the internal net.Conn. This can be used to further +// modifications to connection specific flags. +func (c *Conn) UnderlyingConn() net.Conn { + return c.conn +} + +// EnableWriteCompression enables and disables write compression of +// subsequent text and binary messages. This function is a noop if +// compression was not negotiated with the peer. +func (c *Conn) EnableWriteCompression(enable bool) { + c.enableWriteCompression = enable +} + +// SetCompressionLevel sets the flate compression level for subsequent text and +// binary messages. This function is a noop if compression was not negotiated +// with the peer. See the compress/flate package for a description of +// compression levels. +func (c *Conn) SetCompressionLevel(level int) error { + if !isValidCompressionLevel(level) { + return errors.New("websocket: invalid compression level") + } + c.compressionLevel = level + return nil +} + +// FormatCloseMessage formats closeCode and text as a WebSocket close message. +// An empty message is returned for code CloseNoStatusReceived. +func FormatCloseMessage(closeCode int, text string) []byte { + if closeCode == CloseNoStatusReceived { + // Return empty message because it's illegal to send + // CloseNoStatusReceived. Return non-nil value in case application + // checks for nil. + return []byte{} + } + buf := make([]byte, 2+len(text)) + binary.BigEndian.PutUint16(buf, uint16(closeCode)) + copy(buf[2:], text) + return buf +} diff --git a/vendor/github.com/gorilla/websocket/conn_broadcast_test.go b/vendor/github.com/gorilla/websocket/conn_broadcast_test.go new file mode 100644 index 000000000..cb88cbb0b --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_broadcast_test.go @@ -0,0 +1,132 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "io" + "io/ioutil" + "sync/atomic" + "testing" +) + +// broadcastBench allows to run broadcast benchmarks. +// In every broadcast benchmark we create many connections, then send the same +// message into every connection and wait for all writes complete. This emulates +// an application where many connections listen to the same data - i.e. PUB/SUB +// scenarios with many subscribers in one channel. +type broadcastBench struct { + w io.Writer + message *broadcastMessage + closeCh chan struct{} + doneCh chan struct{} + count int32 + conns []*broadcastConn + compression bool + usePrepared bool +} + +type broadcastMessage struct { + payload []byte + prepared *PreparedMessage +} + +type broadcastConn struct { + conn *Conn + msgCh chan *broadcastMessage +} + +func newBroadcastConn(c *Conn) *broadcastConn { + return &broadcastConn{ + conn: c, + msgCh: make(chan *broadcastMessage, 1), + } +} + +func newBroadcastBench(usePrepared, compression bool) *broadcastBench { + bench := &broadcastBench{ + w: ioutil.Discard, + doneCh: make(chan struct{}), + closeCh: make(chan struct{}), + usePrepared: usePrepared, + compression: compression, + } + msg := &broadcastMessage{ + payload: textMessages(1)[0], + } + if usePrepared { + pm, _ := NewPreparedMessage(TextMessage, msg.payload) + msg.prepared = pm + } + bench.message = msg + bench.makeConns(10000) + return bench +} + +func (b *broadcastBench) makeConns(numConns int) { + conns := make([]*broadcastConn, numConns) + + for i := 0; i < numConns; i++ { + c := newTestConn(nil, b.w, true) + if b.compression { + c.enableWriteCompression = true + c.newCompressionWriter = compressNoContextTakeover + } + conns[i] = newBroadcastConn(c) + go func(c *broadcastConn) { + for { + select { + case msg := <-c.msgCh: + if b.usePrepared { + c.conn.WritePreparedMessage(msg.prepared) + } else { + c.conn.WriteMessage(TextMessage, msg.payload) + } + val := atomic.AddInt32(&b.count, 1) + if val%int32(numConns) == 0 { + b.doneCh <- struct{}{} + } + case <-b.closeCh: + return + } + } + }(conns[i]) + } + b.conns = conns +} + +func (b *broadcastBench) close() { + close(b.closeCh) +} + +func (b *broadcastBench) runOnce() { + for _, c := range b.conns { + c.msgCh <- b.message + } + <-b.doneCh +} + +func BenchmarkBroadcast(b *testing.B) { + benchmarks := []struct { + name string + usePrepared bool + compression bool + }{ + {"NoCompression", false, false}, + {"WithCompression", false, true}, + {"NoCompressionPrepared", true, false}, + {"WithCompressionPrepared", true, true}, + } + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + bench := newBroadcastBench(bm.usePrepared, bm.compression) + defer bench.close() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bench.runOnce() + } + b.ReportAllocs() + }) + } +} diff --git a/vendor/github.com/gorilla/websocket/conn_test.go b/vendor/github.com/gorilla/websocket/conn_test.go new file mode 100644 index 000000000..ad906b14c --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_test.go @@ -0,0 +1,635 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "reflect" + "sync" + "testing" + "testing/iotest" + "time" +) + +var _ net.Error = errWriteTimeout + +type fakeNetConn struct { + io.Reader + io.Writer +} + +func (c fakeNetConn) Close() error { return nil } +func (c fakeNetConn) LocalAddr() net.Addr { return localAddr } +func (c fakeNetConn) RemoteAddr() net.Addr { return remoteAddr } +func (c fakeNetConn) SetDeadline(t time.Time) error { return nil } +func (c fakeNetConn) SetReadDeadline(t time.Time) error { return nil } +func (c fakeNetConn) SetWriteDeadline(t time.Time) error { return nil } + +type fakeAddr int + +var ( + localAddr = fakeAddr(1) + remoteAddr = fakeAddr(2) +) + +func (a fakeAddr) Network() string { + return "net" +} + +func (a fakeAddr) String() string { + return "str" +} + +// newTestConn creates a connnection backed by a fake network connection using +// default values for buffering. +func newTestConn(r io.Reader, w io.Writer, isServer bool) *Conn { + return newConn(fakeNetConn{Reader: r, Writer: w}, isServer, 1024, 1024, nil, nil, nil) +} + +func TestFraming(t *testing.T) { + frameSizes := []int{0, 1, 2, 124, 125, 126, 127, 128, 129, 65534, 65535, 65536, 65537} + var readChunkers = []struct { + name string + f func(io.Reader) io.Reader + }{ + {"half", iotest.HalfReader}, + {"one", iotest.OneByteReader}, + {"asis", func(r io.Reader) io.Reader { return r }}, + } + writeBuf := make([]byte, 65537) + for i := range writeBuf { + writeBuf[i] = byte(i) + } + var writers = []struct { + name string + f func(w io.Writer, n int) (int, error) + }{ + {"iocopy", func(w io.Writer, n int) (int, error) { + nn, err := io.Copy(w, bytes.NewReader(writeBuf[:n])) + return int(nn), err + }}, + {"write", func(w io.Writer, n int) (int, error) { + return w.Write(writeBuf[:n]) + }}, + {"string", func(w io.Writer, n int) (int, error) { + return io.WriteString(w, string(writeBuf[:n])) + }}, + } + + for _, compress := range []bool{false, true} { + for _, isServer := range []bool{true, false} { + for _, chunker := range readChunkers { + + var connBuf bytes.Buffer + wc := newTestConn(nil, &connBuf, isServer) + rc := newTestConn(chunker.f(&connBuf), nil, !isServer) + if compress { + wc.newCompressionWriter = compressNoContextTakeover + rc.newDecompressionReader = decompressNoContextTakeover + } + for _, n := range frameSizes { + for _, writer := range writers { + name := fmt.Sprintf("z:%v, s:%v, r:%s, n:%d w:%s", compress, isServer, chunker.name, n, writer.name) + + w, err := wc.NextWriter(TextMessage) + if err != nil { + t.Errorf("%s: wc.NextWriter() returned %v", name, err) + continue + } + nn, err := writer.f(w, n) + if err != nil || nn != n { + t.Errorf("%s: w.Write(writeBuf[:n]) returned %d, %v", name, nn, err) + continue + } + err = w.Close() + if err != nil { + t.Errorf("%s: w.Close() returned %v", name, err) + continue + } + + opCode, r, err := rc.NextReader() + if err != nil || opCode != TextMessage { + t.Errorf("%s: NextReader() returned %d, r, %v", name, opCode, err) + continue + } + rbuf, err := ioutil.ReadAll(r) + if err != nil { + t.Errorf("%s: ReadFull() returned rbuf, %v", name, err) + continue + } + + if len(rbuf) != n { + t.Errorf("%s: len(rbuf) is %d, want %d", name, len(rbuf), n) + continue + } + + for i, b := range rbuf { + if byte(i) != b { + t.Errorf("%s: bad byte at offset %d", name, i) + break + } + } + } + } + } + } + } +} + +func TestControl(t *testing.T) { + const message = "this is a ping/pong messsage" + for _, isServer := range []bool{true, false} { + for _, isWriteControl := range []bool{true, false} { + name := fmt.Sprintf("s:%v, wc:%v", isServer, isWriteControl) + var connBuf bytes.Buffer + wc := newTestConn(nil, &connBuf, isServer) + rc := newTestConn(&connBuf, nil, !isServer) + if isWriteControl { + wc.WriteControl(PongMessage, []byte(message), time.Now().Add(time.Second)) + } else { + w, err := wc.NextWriter(PongMessage) + if err != nil { + t.Errorf("%s: wc.NextWriter() returned %v", name, err) + continue + } + if _, err := w.Write([]byte(message)); err != nil { + t.Errorf("%s: w.Write() returned %v", name, err) + continue + } + if err := w.Close(); err != nil { + t.Errorf("%s: w.Close() returned %v", name, err) + continue + } + var actualMessage string + rc.SetPongHandler(func(s string) error { actualMessage = s; return nil }) + rc.NextReader() + if actualMessage != message { + t.Errorf("%s: pong=%q, want %q", name, actualMessage, message) + continue + } + } + } + } +} + +// simpleBufferPool is an implementation of BufferPool for TestWriteBufferPool. +type simpleBufferPool struct { + v interface{} +} + +func (p *simpleBufferPool) Get() interface{} { + v := p.v + p.v = nil + return v +} + +func (p *simpleBufferPool) Put(v interface{}) { + p.v = v +} + +func TestWriteBufferPool(t *testing.T) { + const message = "Now is the time for all good people to come to the aid of the party." + + var buf bytes.Buffer + var pool simpleBufferPool + rc := newTestConn(&buf, nil, false) + + // Specify writeBufferSize smaller than message size to ensure that pooling + // works with fragmented messages. + wc := newConn(fakeNetConn{Writer: &buf}, true, 1024, len(message)-1, &pool, nil, nil) + + if wc.writeBuf != nil { + t.Fatal("writeBuf not nil after create") + } + + // Part 1: test NextWriter/Write/Close + + w, err := wc.NextWriter(TextMessage) + if err != nil { + t.Fatalf("wc.NextWriter() returned %v", err) + } + + if wc.writeBuf == nil { + t.Fatal("writeBuf is nil after NextWriter") + } + + writeBufAddr := &wc.writeBuf[0] + + if _, err := io.WriteString(w, message); err != nil { + t.Fatalf("io.WriteString(w, message) returned %v", err) + } + + if err := w.Close(); err != nil { + t.Fatalf("w.Close() returned %v", err) + } + + if wc.writeBuf != nil { + t.Fatal("writeBuf not nil after w.Close()") + } + + if wpd, ok := pool.v.(writePoolData); !ok || len(wpd.buf) == 0 || &wpd.buf[0] != writeBufAddr { + t.Fatal("writeBuf not returned to pool") + } + + opCode, p, err := rc.ReadMessage() + if opCode != TextMessage || err != nil { + t.Fatalf("ReadMessage() returned %d, p, %v", opCode, err) + } + + if s := string(p); s != message { + t.Fatalf("message is %s, want %s", s, message) + } + + // Part 2: Test WriteMessage. + + if err := wc.WriteMessage(TextMessage, []byte(message)); err != nil { + t.Fatalf("wc.WriteMessage() returned %v", err) + } + + if wc.writeBuf != nil { + t.Fatal("writeBuf not nil after wc.WriteMessage()") + } + + if wpd, ok := pool.v.(writePoolData); !ok || len(wpd.buf) == 0 || &wpd.buf[0] != writeBufAddr { + t.Fatal("writeBuf not returned to pool after WriteMessage") + } + + opCode, p, err = rc.ReadMessage() + if opCode != TextMessage || err != nil { + t.Fatalf("ReadMessage() returned %d, p, %v", opCode, err) + } + + if s := string(p); s != message { + t.Fatalf("message is %s, want %s", s, message) + } +} + +// TestWriteBufferPoolSync ensures that *sync.Pool works as a buffer pool. +func TestWriteBufferPoolSync(t *testing.T) { + var buf bytes.Buffer + var pool sync.Pool + wc := newConn(fakeNetConn{Writer: &buf}, true, 1024, 1024, &pool, nil, nil) + rc := newTestConn(&buf, nil, false) + + const message = "Hello World!" + for i := 0; i < 3; i++ { + if err := wc.WriteMessage(TextMessage, []byte(message)); err != nil { + t.Fatalf("wc.WriteMessage() returned %v", err) + } + opCode, p, err := rc.ReadMessage() + if opCode != TextMessage || err != nil { + t.Fatalf("ReadMessage() returned %d, p, %v", opCode, err) + } + if s := string(p); s != message { + t.Fatalf("message is %s, want %s", s, message) + } + } +} + +// errorWriter is an io.Writer than returns an error on all writes. +type errorWriter struct{} + +func (ew errorWriter) Write(p []byte) (int, error) { return 0, errors.New("Error!") } + +// TestWriteBufferPoolError ensures that buffer is returned to pool after error +// on write. +func TestWriteBufferPoolError(t *testing.T) { + + // Part 1: Test NextWriter/Write/Close + + var pool simpleBufferPool + wc := newConn(fakeNetConn{Writer: errorWriter{}}, true, 1024, 1024, &pool, nil, nil) + + w, err := wc.NextWriter(TextMessage) + if err != nil { + t.Fatalf("wc.NextWriter() returned %v", err) + } + + if wc.writeBuf == nil { + t.Fatal("writeBuf is nil after NextWriter") + } + + writeBufAddr := &wc.writeBuf[0] + + if _, err := io.WriteString(w, "Hello"); err != nil { + t.Fatalf("io.WriteString(w, message) returned %v", err) + } + + if err := w.Close(); err == nil { + t.Fatalf("w.Close() did not return error") + } + + if wpd, ok := pool.v.(writePoolData); !ok || len(wpd.buf) == 0 || &wpd.buf[0] != writeBufAddr { + t.Fatal("writeBuf not returned to pool") + } + + // Part 2: Test WriteMessage + + wc = newConn(fakeNetConn{Writer: errorWriter{}}, true, 1024, 1024, &pool, nil, nil) + + if err := wc.WriteMessage(TextMessage, []byte("Hello")); err == nil { + t.Fatalf("wc.WriteMessage did not return error") + } + + if wpd, ok := pool.v.(writePoolData); !ok || len(wpd.buf) == 0 || &wpd.buf[0] != writeBufAddr { + t.Fatal("writeBuf not returned to pool") + } +} + +func TestCloseFrameBeforeFinalMessageFrame(t *testing.T) { + const bufSize = 512 + + expectedErr := &CloseError{Code: CloseNormalClosure, Text: "hello"} + + var b1, b2 bytes.Buffer + wc := newConn(&fakeNetConn{Reader: nil, Writer: &b1}, false, 1024, bufSize, nil, nil, nil) + rc := newTestConn(&b1, &b2, true) + + w, _ := wc.NextWriter(BinaryMessage) + w.Write(make([]byte, bufSize+bufSize/2)) + wc.WriteControl(CloseMessage, FormatCloseMessage(expectedErr.Code, expectedErr.Text), time.Now().Add(10*time.Second)) + w.Close() + + op, r, err := rc.NextReader() + if op != BinaryMessage || err != nil { + t.Fatalf("NextReader() returned %d, %v", op, err) + } + _, err = io.Copy(ioutil.Discard, r) + if !reflect.DeepEqual(err, expectedErr) { + t.Fatalf("io.Copy() returned %v, want %v", err, expectedErr) + } + _, _, err = rc.NextReader() + if !reflect.DeepEqual(err, expectedErr) { + t.Fatalf("NextReader() returned %v, want %v", err, expectedErr) + } +} + +func TestEOFWithinFrame(t *testing.T) { + const bufSize = 64 + + for n := 0; ; n++ { + var b bytes.Buffer + wc := newTestConn(nil, &b, false) + rc := newTestConn(&b, nil, true) + + w, _ := wc.NextWriter(BinaryMessage) + w.Write(make([]byte, bufSize)) + w.Close() + + if n >= b.Len() { + break + } + b.Truncate(n) + + op, r, err := rc.NextReader() + if err == errUnexpectedEOF { + continue + } + if op != BinaryMessage || err != nil { + t.Fatalf("%d: NextReader() returned %d, %v", n, op, err) + } + _, err = io.Copy(ioutil.Discard, r) + if err != errUnexpectedEOF { + t.Fatalf("%d: io.Copy() returned %v, want %v", n, err, errUnexpectedEOF) + } + _, _, err = rc.NextReader() + if err != errUnexpectedEOF { + t.Fatalf("%d: NextReader() returned %v, want %v", n, err, errUnexpectedEOF) + } + } +} + +func TestEOFBeforeFinalFrame(t *testing.T) { + const bufSize = 512 + + var b1, b2 bytes.Buffer + wc := newConn(&fakeNetConn{Writer: &b1}, false, 1024, bufSize, nil, nil, nil) + rc := newTestConn(&b1, &b2, true) + + w, _ := wc.NextWriter(BinaryMessage) + w.Write(make([]byte, bufSize+bufSize/2)) + + op, r, err := rc.NextReader() + if op != BinaryMessage || err != nil { + t.Fatalf("NextReader() returned %d, %v", op, err) + } + _, err = io.Copy(ioutil.Discard, r) + if err != errUnexpectedEOF { + t.Fatalf("io.Copy() returned %v, want %v", err, errUnexpectedEOF) + } + _, _, err = rc.NextReader() + if err != errUnexpectedEOF { + t.Fatalf("NextReader() returned %v, want %v", err, errUnexpectedEOF) + } +} + +func TestWriteAfterMessageWriterClose(t *testing.T) { + wc := newTestConn(nil, &bytes.Buffer{}, false) + w, _ := wc.NextWriter(BinaryMessage) + io.WriteString(w, "hello") + if err := w.Close(); err != nil { + t.Fatalf("unxpected error closing message writer, %v", err) + } + + if _, err := io.WriteString(w, "world"); err == nil { + t.Fatalf("no error writing after close") + } + + w, _ = wc.NextWriter(BinaryMessage) + io.WriteString(w, "hello") + + // close w by getting next writer + _, err := wc.NextWriter(BinaryMessage) + if err != nil { + t.Fatalf("unexpected error getting next writer, %v", err) + } + + if _, err := io.WriteString(w, "world"); err == nil { + t.Fatalf("no error writing after close") + } +} + +func TestReadLimit(t *testing.T) { + + const readLimit = 512 + message := make([]byte, readLimit+1) + + var b1, b2 bytes.Buffer + wc := newConn(&fakeNetConn{Writer: &b1}, false, 1024, readLimit-2, nil, nil, nil) + rc := newTestConn(&b1, &b2, true) + rc.SetReadLimit(readLimit) + + // Send message at the limit with interleaved pong. + w, _ := wc.NextWriter(BinaryMessage) + w.Write(message[:readLimit-1]) + wc.WriteControl(PongMessage, []byte("this is a pong"), time.Now().Add(10*time.Second)) + w.Write(message[:1]) + w.Close() + + // Send message larger than the limit. + wc.WriteMessage(BinaryMessage, message[:readLimit+1]) + + op, _, err := rc.NextReader() + if op != BinaryMessage || err != nil { + t.Fatalf("1: NextReader() returned %d, %v", op, err) + } + op, r, err := rc.NextReader() + if op != BinaryMessage || err != nil { + t.Fatalf("2: NextReader() returned %d, %v", op, err) + } + _, err = io.Copy(ioutil.Discard, r) + if err != ErrReadLimit { + t.Fatalf("io.Copy() returned %v", err) + } +} + +func TestAddrs(t *testing.T) { + c := newTestConn(nil, nil, true) + if c.LocalAddr() != localAddr { + t.Errorf("LocalAddr = %v, want %v", c.LocalAddr(), localAddr) + } + if c.RemoteAddr() != remoteAddr { + t.Errorf("RemoteAddr = %v, want %v", c.RemoteAddr(), remoteAddr) + } +} + +func TestUnderlyingConn(t *testing.T) { + var b1, b2 bytes.Buffer + fc := fakeNetConn{Reader: &b1, Writer: &b2} + c := newConn(fc, true, 1024, 1024, nil, nil, nil) + ul := c.UnderlyingConn() + if ul != fc { + t.Fatalf("Underlying conn is not what it should be.") + } +} + +func TestBufioReadBytes(t *testing.T) { + // Test calling bufio.ReadBytes for value longer than read buffer size. + + m := make([]byte, 512) + m[len(m)-1] = '\n' + + var b1, b2 bytes.Buffer + wc := newConn(fakeNetConn{Writer: &b1}, false, len(m)+64, len(m)+64, nil, nil, nil) + rc := newConn(fakeNetConn{Reader: &b1, Writer: &b2}, true, len(m)-64, len(m)-64, nil, nil, nil) + + w, _ := wc.NextWriter(BinaryMessage) + w.Write(m) + w.Close() + + op, r, err := rc.NextReader() + if op != BinaryMessage || err != nil { + t.Fatalf("NextReader() returned %d, %v", op, err) + } + + br := bufio.NewReader(r) + p, err := br.ReadBytes('\n') + if err != nil { + t.Fatalf("ReadBytes() returned %v", err) + } + if len(p) != len(m) { + t.Fatalf("read returned %d bytes, want %d bytes", len(p), len(m)) + } +} + +var closeErrorTests = []struct { + err error + codes []int + ok bool +}{ + {&CloseError{Code: CloseNormalClosure}, []int{CloseNormalClosure}, true}, + {&CloseError{Code: CloseNormalClosure}, []int{CloseNoStatusReceived}, false}, + {&CloseError{Code: CloseNormalClosure}, []int{CloseNoStatusReceived, CloseNormalClosure}, true}, + {errors.New("hello"), []int{CloseNormalClosure}, false}, +} + +func TestCloseError(t *testing.T) { + for _, tt := range closeErrorTests { + ok := IsCloseError(tt.err, tt.codes...) + if ok != tt.ok { + t.Errorf("IsCloseError(%#v, %#v) returned %v, want %v", tt.err, tt.codes, ok, tt.ok) + } + } +} + +var unexpectedCloseErrorTests = []struct { + err error + codes []int + ok bool +}{ + {&CloseError{Code: CloseNormalClosure}, []int{CloseNormalClosure}, false}, + {&CloseError{Code: CloseNormalClosure}, []int{CloseNoStatusReceived}, true}, + {&CloseError{Code: CloseNormalClosure}, []int{CloseNoStatusReceived, CloseNormalClosure}, false}, + {errors.New("hello"), []int{CloseNormalClosure}, false}, +} + +func TestUnexpectedCloseErrors(t *testing.T) { + for _, tt := range unexpectedCloseErrorTests { + ok := IsUnexpectedCloseError(tt.err, tt.codes...) + if ok != tt.ok { + t.Errorf("IsUnexpectedCloseError(%#v, %#v) returned %v, want %v", tt.err, tt.codes, ok, tt.ok) + } + } +} + +type blockingWriter struct { + c1, c2 chan struct{} +} + +func (w blockingWriter) Write(p []byte) (int, error) { + // Allow main to continue + close(w.c1) + // Wait for panic in main + <-w.c2 + return len(p), nil +} + +func TestConcurrentWritePanic(t *testing.T) { + w := blockingWriter{make(chan struct{}), make(chan struct{})} + c := newTestConn(nil, w, false) + go func() { + c.WriteMessage(TextMessage, []byte{}) + }() + + // wait for goroutine to block in write. + <-w.c1 + + defer func() { + close(w.c2) + if v := recover(); v != nil { + return + } + }() + + c.WriteMessage(TextMessage, []byte{}) + t.Fatal("should not get here") +} + +type failingReader struct{} + +func (r failingReader) Read(p []byte) (int, error) { + return 0, io.EOF +} + +func TestFailedConnectionReadPanic(t *testing.T) { + c := newTestConn(failingReader{}, nil, false) + + defer func() { + if v := recover(); v != nil { + return + } + }() + + for i := 0; i < 20000; i++ { + c.ReadMessage() + } + t.Fatal("should not get here") +} diff --git a/vendor/github.com/gorilla/websocket/conn_write.go b/vendor/github.com/gorilla/websocket/conn_write.go new file mode 100644 index 000000000..a509a21f8 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_write.go @@ -0,0 +1,15 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.8 + +package websocket + +import "net" + +func (c *Conn) writeBufs(bufs ...[]byte) error { + b := net.Buffers(bufs) + _, err := b.WriteTo(c.conn) + return err +} diff --git a/vendor/github.com/gorilla/websocket/conn_write_legacy.go b/vendor/github.com/gorilla/websocket/conn_write_legacy.go new file mode 100644 index 000000000..37edaff5a --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_write_legacy.go @@ -0,0 +1,18 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.8 + +package websocket + +func (c *Conn) writeBufs(bufs ...[]byte) error { + for _, buf := range bufs { + if len(buf) > 0 { + if _, err := c.conn.Write(buf); err != nil { + return err + } + } + } + return nil +} diff --git a/vendor/github.com/gorilla/websocket/doc.go b/vendor/github.com/gorilla/websocket/doc.go new file mode 100644 index 000000000..c6f4df896 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/doc.go @@ -0,0 +1,227 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package websocket implements the WebSocket protocol defined in RFC 6455. +// +// Overview +// +// The Conn type represents a WebSocket connection. A server application calls +// the Upgrader.Upgrade method from an HTTP request handler to get a *Conn: +// +// var upgrader = websocket.Upgrader{ +// ReadBufferSize: 1024, +// WriteBufferSize: 1024, +// } +// +// func handler(w http.ResponseWriter, r *http.Request) { +// conn, err := upgrader.Upgrade(w, r, nil) +// if err != nil { +// log.Println(err) +// return +// } +// ... Use conn to send and receive messages. +// } +// +// Call the connection's WriteMessage and ReadMessage methods to send and +// receive messages as a slice of bytes. This snippet of code shows how to echo +// messages using these methods: +// +// for { +// messageType, p, err := conn.ReadMessage() +// if err != nil { +// log.Println(err) +// return +// } +// if err := conn.WriteMessage(messageType, p); err != nil { +// log.Println(err) +// return +// } +// } +// +// In above snippet of code, p is a []byte and messageType is an int with value +// websocket.BinaryMessage or websocket.TextMessage. +// +// An application can also send and receive messages using the io.WriteCloser +// and io.Reader interfaces. To send a message, call the connection NextWriter +// method to get an io.WriteCloser, write the message to the writer and close +// the writer when done. To receive a message, call the connection NextReader +// method to get an io.Reader and read until io.EOF is returned. This snippet +// shows how to echo messages using the NextWriter and NextReader methods: +// +// for { +// messageType, r, err := conn.NextReader() +// if err != nil { +// return +// } +// w, err := conn.NextWriter(messageType) +// if err != nil { +// return err +// } +// if _, err := io.Copy(w, r); err != nil { +// return err +// } +// if err := w.Close(); err != nil { +// return err +// } +// } +// +// Data Messages +// +// The WebSocket protocol distinguishes between text and binary data messages. +// Text messages are interpreted as UTF-8 encoded text. The interpretation of +// binary messages is left to the application. +// +// This package uses the TextMessage and BinaryMessage integer constants to +// identify the two data message types. The ReadMessage and NextReader methods +// return the type of the received message. The messageType argument to the +// WriteMessage and NextWriter methods specifies the type of a sent message. +// +// It is the application's responsibility to ensure that text messages are +// valid UTF-8 encoded text. +// +// Control Messages +// +// The WebSocket protocol defines three types of control messages: close, ping +// and pong. Call the connection WriteControl, WriteMessage or NextWriter +// methods to send a control message to the peer. +// +// Connections handle received close messages by calling the handler function +// set with the SetCloseHandler method and by returning a *CloseError from the +// NextReader, ReadMessage or the message Read method. The default close +// handler sends a close message to the peer. +// +// Connections handle received ping messages by calling the handler function +// set with the SetPingHandler method. The default ping handler sends a pong +// message to the peer. +// +// Connections handle received pong messages by calling the handler function +// set with the SetPongHandler method. The default pong handler does nothing. +// If an application sends ping messages, then the application should set a +// pong handler to receive the corresponding pong. +// +// The control message handler functions are called from the NextReader, +// ReadMessage and message reader Read methods. The default close and ping +// handlers can block these methods for a short time when the handler writes to +// the connection. +// +// The application must read the connection to process close, ping and pong +// messages sent from the peer. If the application is not otherwise interested +// in messages from the peer, then the application should start a goroutine to +// read and discard messages from the peer. A simple example is: +// +// func readLoop(c *websocket.Conn) { +// for { +// if _, _, err := c.NextReader(); err != nil { +// c.Close() +// break +// } +// } +// } +// +// Concurrency +// +// Connections support one concurrent reader and one concurrent writer. +// +// Applications are responsible for ensuring that no more than one goroutine +// calls the write methods (NextWriter, SetWriteDeadline, WriteMessage, +// WriteJSON, EnableWriteCompression, SetCompressionLevel) concurrently and +// that no more than one goroutine calls the read methods (NextReader, +// SetReadDeadline, ReadMessage, ReadJSON, SetPongHandler, SetPingHandler) +// concurrently. +// +// The Close and WriteControl methods can be called concurrently with all other +// methods. +// +// Origin Considerations +// +// Web browsers allow Javascript applications to open a WebSocket connection to +// any host. It's up to the server to enforce an origin policy using the Origin +// request header sent by the browser. +// +// The Upgrader calls the function specified in the CheckOrigin field to check +// the origin. If the CheckOrigin function returns false, then the Upgrade +// method fails the WebSocket handshake with HTTP status 403. +// +// If the CheckOrigin field is nil, then the Upgrader uses a safe default: fail +// the handshake if the Origin request header is present and the Origin host is +// not equal to the Host request header. +// +// The deprecated package-level Upgrade function does not perform origin +// checking. The application is responsible for checking the Origin header +// before calling the Upgrade function. +// +// Buffers +// +// Connections buffer network input and output to reduce the number +// of system calls when reading or writing messages. +// +// Write buffers are also used for constructing WebSocket frames. See RFC 6455, +// Section 5 for a discussion of message framing. A WebSocket frame header is +// written to the network each time a write buffer is flushed to the network. +// Decreasing the size of the write buffer can increase the amount of framing +// overhead on the connection. +// +// The buffer sizes in bytes are specified by the ReadBufferSize and +// WriteBufferSize fields in the Dialer and Upgrader. The Dialer uses a default +// size of 4096 when a buffer size field is set to zero. The Upgrader reuses +// buffers created by the HTTP server when a buffer size field is set to zero. +// The HTTP server buffers have a size of 4096 at the time of this writing. +// +// The buffer sizes do not limit the size of a message that can be read or +// written by a connection. +// +// Buffers are held for the lifetime of the connection by default. If the +// Dialer or Upgrader WriteBufferPool field is set, then a connection holds the +// write buffer only when writing a message. +// +// Applications should tune the buffer sizes to balance memory use and +// performance. Increasing the buffer size uses more memory, but can reduce the +// number of system calls to read or write the network. In the case of writing, +// increasing the buffer size can reduce the number of frame headers written to +// the network. +// +// Some guidelines for setting buffer parameters are: +// +// Limit the buffer sizes to the maximum expected message size. Buffers larger +// than the largest message do not provide any benefit. +// +// Depending on the distribution of message sizes, setting the buffer size to +// to a value less than the maximum expected message size can greatly reduce +// memory use with a small impact on performance. Here's an example: If 99% of +// the messages are smaller than 256 bytes and the maximum message size is 512 +// bytes, then a buffer size of 256 bytes will result in 1.01 more system calls +// than a buffer size of 512 bytes. The memory savings is 50%. +// +// A write buffer pool is useful when the application has a modest number +// writes over a large number of connections. when buffers are pooled, a larger +// buffer size has a reduced impact on total memory use and has the benefit of +// reducing system calls and frame overhead. +// +// Compression EXPERIMENTAL +// +// Per message compression extensions (RFC 7692) are experimentally supported +// by this package in a limited capacity. Setting the EnableCompression option +// to true in Dialer or Upgrader will attempt to negotiate per message deflate +// support. +// +// var upgrader = websocket.Upgrader{ +// EnableCompression: true, +// } +// +// If compression was successfully negotiated with the connection's peer, any +// message received in compressed form will be automatically decompressed. +// All Read methods will return uncompressed bytes. +// +// Per message compression of messages written to a connection can be enabled +// or disabled by calling the corresponding Conn method: +// +// conn.EnableWriteCompression(false) +// +// Currently this package does not support compression with "context takeover". +// This means that messages must be compressed and decompressed in isolation, +// without retaining sliding window or dictionary state across messages. For +// more details refer to RFC 7692. +// +// Use of compression is experimental and may result in decreased performance. +package websocket diff --git a/vendor/github.com/gorilla/websocket/example_test.go b/vendor/github.com/gorilla/websocket/example_test.go new file mode 100644 index 000000000..96449eac7 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/example_test.go @@ -0,0 +1,46 @@ +// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket_test + +import ( + "log" + "net/http" + "testing" + + "github.com/gorilla/websocket" +) + +var ( + c *websocket.Conn + req *http.Request +) + +// The websocket.IsUnexpectedCloseError function is useful for identifying +// application and protocol errors. +// +// This server application works with a client application running in the +// browser. The client application does not explicitly close the websocket. The +// only expected close message from the client has the code +// websocket.CloseGoingAway. All other other close messages are likely the +// result of an application or protocol error and are logged to aid debugging. +func ExampleIsUnexpectedCloseError() { + + for { + messageType, p, err := c.ReadMessage() + if err != nil { + if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) { + log.Printf("error: %v, user-agent: %v", err, req.Header.Get("User-Agent")) + } + return + } + processMesage(messageType, p) + } +} + +func processMesage(mt int, p []byte) {} + +// TestX prevents godoc from showing this entire file in the example. Remove +// this function when a second example is added. +func TestX(t *testing.T) {} diff --git a/vendor/github.com/gorilla/websocket/examples/autobahn/README.md b/vendor/github.com/gorilla/websocket/examples/autobahn/README.md new file mode 100644 index 000000000..dde8525a0 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/autobahn/README.md @@ -0,0 +1,13 @@ +# Test Server + +This package contains a server for the [Autobahn WebSockets Test Suite](https://github.com/crossbario/autobahn-testsuite). + +To test the server, run + + go run server.go + +and start the client test driver + + wstest -m fuzzingclient -s fuzzingclient.json + +When the client completes, it writes a report to reports/clients/index.html. diff --git a/vendor/github.com/gorilla/websocket/examples/autobahn/fuzzingclient.json b/vendor/github.com/gorilla/websocket/examples/autobahn/fuzzingclient.json new file mode 100644 index 000000000..aa3a0bc0a --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/autobahn/fuzzingclient.json @@ -0,0 +1,15 @@ + +{ + "options": {"failByDrop": false}, + "outdir": "./reports/clients", + "servers": [ + {"agent": "ReadAllWriteMessage", "url": "ws://localhost:9000/m", "options": {"version": 18}}, + {"agent": "ReadAllWritePreparedMessage", "url": "ws://localhost:9000/p", "options": {"version": 18}}, + {"agent": "ReadAllWrite", "url": "ws://localhost:9000/r", "options": {"version": 18}}, + {"agent": "CopyFull", "url": "ws://localhost:9000/f", "options": {"version": 18}}, + {"agent": "CopyWriterOnly", "url": "ws://localhost:9000/c", "options": {"version": 18}} + ], + "cases": ["*"], + "exclude-cases": [], + "exclude-agent-cases": {} +} diff --git a/vendor/github.com/gorilla/websocket/examples/autobahn/server.go b/vendor/github.com/gorilla/websocket/examples/autobahn/server.go new file mode 100644 index 000000000..c2d6ee504 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/autobahn/server.go @@ -0,0 +1,265 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Command server is a test server for the Autobahn WebSockets Test Suite. +package main + +import ( + "errors" + "flag" + "io" + "log" + "net/http" + "time" + "unicode/utf8" + + "github.com/gorilla/websocket" +) + +var upgrader = websocket.Upgrader{ + ReadBufferSize: 4096, + WriteBufferSize: 4096, + EnableCompression: true, + CheckOrigin: func(r *http.Request) bool { + return true + }, +} + +// echoCopy echoes messages from the client using io.Copy. +func echoCopy(w http.ResponseWriter, r *http.Request, writerOnly bool) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Println("Upgrade:", err) + return + } + defer conn.Close() + for { + mt, r, err := conn.NextReader() + if err != nil { + if err != io.EOF { + log.Println("NextReader:", err) + } + return + } + if mt == websocket.TextMessage { + r = &validator{r: r} + } + w, err := conn.NextWriter(mt) + if err != nil { + log.Println("NextWriter:", err) + return + } + if mt == websocket.TextMessage { + r = &validator{r: r} + } + if writerOnly { + _, err = io.Copy(struct{ io.Writer }{w}, r) + } else { + _, err = io.Copy(w, r) + } + if err != nil { + if err == errInvalidUTF8 { + conn.WriteControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseInvalidFramePayloadData, ""), + time.Time{}) + } + log.Println("Copy:", err) + return + } + err = w.Close() + if err != nil { + log.Println("Close:", err) + return + } + } +} + +func echoCopyWriterOnly(w http.ResponseWriter, r *http.Request) { + echoCopy(w, r, true) +} + +func echoCopyFull(w http.ResponseWriter, r *http.Request) { + echoCopy(w, r, false) +} + +// echoReadAll echoes messages from the client by reading the entire message +// with ioutil.ReadAll. +func echoReadAll(w http.ResponseWriter, r *http.Request, writeMessage, writePrepared bool) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Println("Upgrade:", err) + return + } + defer conn.Close() + for { + mt, b, err := conn.ReadMessage() + if err != nil { + if err != io.EOF { + log.Println("NextReader:", err) + } + return + } + if mt == websocket.TextMessage { + if !utf8.Valid(b) { + conn.WriteControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseInvalidFramePayloadData, ""), + time.Time{}) + log.Println("ReadAll: invalid utf8") + } + } + if writeMessage { + if !writePrepared { + err = conn.WriteMessage(mt, b) + if err != nil { + log.Println("WriteMessage:", err) + } + } else { + pm, err := websocket.NewPreparedMessage(mt, b) + if err != nil { + log.Println("NewPreparedMessage:", err) + return + } + err = conn.WritePreparedMessage(pm) + if err != nil { + log.Println("WritePreparedMessage:", err) + } + } + } else { + w, err := conn.NextWriter(mt) + if err != nil { + log.Println("NextWriter:", err) + return + } + if _, err := w.Write(b); err != nil { + log.Println("Writer:", err) + return + } + if err := w.Close(); err != nil { + log.Println("Close:", err) + return + } + } + } +} + +func echoReadAllWriter(w http.ResponseWriter, r *http.Request) { + echoReadAll(w, r, false, false) +} + +func echoReadAllWriteMessage(w http.ResponseWriter, r *http.Request) { + echoReadAll(w, r, true, false) +} + +func echoReadAllWritePreparedMessage(w http.ResponseWriter, r *http.Request) { + echoReadAll(w, r, true, true) +} + +func serveHome(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.Error(w, "Not found.", http.StatusNotFound) + return + } + if r.Method != "GET" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + io.WriteString(w, "Echo Server") +} + +var addr = flag.String("addr", ":9000", "http service address") + +func main() { + flag.Parse() + http.HandleFunc("/", serveHome) + http.HandleFunc("/c", echoCopyWriterOnly) + http.HandleFunc("/f", echoCopyFull) + http.HandleFunc("/r", echoReadAllWriter) + http.HandleFunc("/m", echoReadAllWriteMessage) + http.HandleFunc("/p", echoReadAllWritePreparedMessage) + err := http.ListenAndServe(*addr, nil) + if err != nil { + log.Fatal("ListenAndServe: ", err) + } +} + +type validator struct { + state int + x rune + r io.Reader +} + +var errInvalidUTF8 = errors.New("invalid utf8") + +func (r *validator) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + state := r.state + x := r.x + for _, b := range p[:n] { + state, x = decode(state, x, b) + if state == utf8Reject { + break + } + } + r.state = state + r.x = x + if state == utf8Reject || (err == io.EOF && state != utf8Accept) { + return n, errInvalidUTF8 + } + return n, err +} + +// UTF-8 decoder from http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +// +// Copyright (c) 2008-2009 Bjoern Hoehrmann +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +var utf8d = [...]byte{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7f + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9f + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // a0..bf + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // c0..df + 0xa, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // e0..ef + 0xb, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // f0..ff + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // s7..s8 +} + +const ( + utf8Accept = 0 + utf8Reject = 1 +) + +func decode(state int, x rune, b byte) (int, rune) { + t := utf8d[b] + if state != utf8Accept { + x = rune(b&0x3f) | (x << 6) + } else { + x = rune((0xff >> t) & b) + } + state = int(utf8d[256+state*16+int(t)]) + return state, x +} diff --git a/vendor/github.com/gorilla/websocket/examples/chat/README.md b/vendor/github.com/gorilla/websocket/examples/chat/README.md new file mode 100644 index 000000000..7baf3e328 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/chat/README.md @@ -0,0 +1,102 @@ +# Chat Example + +This application shows how to use the +[websocket](https://github.com/gorilla/websocket) package to implement a simple +web chat application. + +## Running the example + +The example requires a working Go development environment. The [Getting +Started](http://golang.org/doc/install) page describes how to install the +development environment. + +Once you have Go up and running, you can download, build and run the example +using the following commands. + + $ go get github.com/gorilla/websocket + $ cd `go list -f '{{.Dir}}' github.com/gorilla/websocket/examples/chat` + $ go run *.go + +To use the chat example, open http://localhost:8080/ in your browser. + +## Server + +The server application defines two types, `Client` and `Hub`. The server +creates an instance of the `Client` type for each websocket connection. A +`Client` acts as an intermediary between the websocket connection and a single +instance of the `Hub` type. The `Hub` maintains a set of registered clients and +broadcasts messages to the clients. + +The application runs one goroutine for the `Hub` and two goroutines for each +`Client`. The goroutines communicate with each other using channels. The `Hub` +has channels for registering clients, unregistering clients and broadcasting +messages. A `Client` has a buffered channel of outbound messages. One of the +client's goroutines reads messages from this channel and writes the messages to +the websocket. The other client goroutine reads messages from the websocket and +sends them to the hub. + +### Hub + +The code for the `Hub` type is in +[hub.go](https://github.com/gorilla/websocket/blob/master/examples/chat/hub.go). +The application's `main` function starts the hub's `run` method as a goroutine. +Clients send requests to the hub using the `register`, `unregister` and +`broadcast` channels. + +The hub registers clients by adding the client pointer as a key in the +`clients` map. The map value is always true. + +The unregister code is a little more complicated. In addition to deleting the +client pointer from the `clients` map, the hub closes the clients's `send` +channel to signal the client that no more messages will be sent to the client. + +The hub handles messages by looping over the registered clients and sending the +message to the client's `send` channel. If the client's `send` buffer is full, +then the hub assumes that the client is dead or stuck. In this case, the hub +unregisters the client and closes the websocket. + +### Client + +The code for the `Client` type is in [client.go](https://github.com/gorilla/websocket/blob/master/examples/chat/client.go). + +The `serveWs` function is registered by the application's `main` function as +an HTTP handler. The handler upgrades the HTTP connection to the WebSocket +protocol, creates a client, registers the client with the hub and schedules the +client to be unregistered using a defer statement. + +Next, the HTTP handler starts the client's `writePump` method as a goroutine. +This method transfers messages from the client's send channel to the websocket +connection. The writer method exits when the channel is closed by the hub or +there's an error writing to the websocket connection. + +Finally, the HTTP handler calls the client's `readPump` method. This method +transfers inbound messages from the websocket to the hub. + +WebSocket connections [support one concurrent reader and one concurrent +writer](https://godoc.org/github.com/gorilla/websocket#hdr-Concurrency). The +application ensures that these concurrency requirements are met by executing +all reads from the `readPump` goroutine and all writes from the `writePump` +goroutine. + +To improve efficiency under high load, the `writePump` function coalesces +pending chat messages in the `send` channel to a single WebSocket message. This +reduces the number of system calls and the amount of data sent over the +network. + +## Frontend + +The frontend code is in [home.html](https://github.com/gorilla/websocket/blob/master/examples/chat/home.html). + +On document load, the script checks for websocket functionality in the browser. +If websocket functionality is available, then the script opens a connection to +the server and registers a callback to handle messages from the server. The +callback appends the message to the chat log using the appendLog function. + +To allow the user to manually scroll through the chat log without interruption +from new messages, the `appendLog` function checks the scroll position before +adding new content. If the chat log is scrolled to the bottom, then the +function scrolls new content into view after adding the content. Otherwise, the +scroll position is not changed. + +The form handler writes the user input to the websocket and clears the input +field. diff --git a/vendor/github.com/gorilla/websocket/examples/chat/client.go b/vendor/github.com/gorilla/websocket/examples/chat/client.go new file mode 100644 index 000000000..9461c1ea0 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/chat/client.go @@ -0,0 +1,137 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "log" + "net/http" + "time" + + "github.com/gorilla/websocket" +) + +const ( + // Time allowed to write a message to the peer. + writeWait = 10 * time.Second + + // Time allowed to read the next pong message from the peer. + pongWait = 60 * time.Second + + // Send pings to peer with this period. Must be less than pongWait. + pingPeriod = (pongWait * 9) / 10 + + // Maximum message size allowed from peer. + maxMessageSize = 512 +) + +var ( + newline = []byte{'\n'} + space = []byte{' '} +) + +var upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, +} + +// Client is a middleman between the websocket connection and the hub. +type Client struct { + hub *Hub + + // The websocket connection. + conn *websocket.Conn + + // Buffered channel of outbound messages. + send chan []byte +} + +// readPump pumps messages from the websocket connection to the hub. +// +// The application runs readPump in a per-connection goroutine. The application +// ensures that there is at most one reader on a connection by executing all +// reads from this goroutine. +func (c *Client) readPump() { + defer func() { + c.hub.unregister <- c + c.conn.Close() + }() + c.conn.SetReadLimit(maxMessageSize) + c.conn.SetReadDeadline(time.Now().Add(pongWait)) + c.conn.SetPongHandler(func(string) error { c.conn.SetReadDeadline(time.Now().Add(pongWait)); return nil }) + for { + _, message, err := c.conn.ReadMessage() + if err != nil { + if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) { + log.Printf("error: %v", err) + } + break + } + message = bytes.TrimSpace(bytes.Replace(message, newline, space, -1)) + c.hub.broadcast <- message + } +} + +// writePump pumps messages from the hub to the websocket connection. +// +// A goroutine running writePump is started for each connection. The +// application ensures that there is at most one writer to a connection by +// executing all writes from this goroutine. +func (c *Client) writePump() { + ticker := time.NewTicker(pingPeriod) + defer func() { + ticker.Stop() + c.conn.Close() + }() + for { + select { + case message, ok := <-c.send: + c.conn.SetWriteDeadline(time.Now().Add(writeWait)) + if !ok { + // The hub closed the channel. + c.conn.WriteMessage(websocket.CloseMessage, []byte{}) + return + } + + w, err := c.conn.NextWriter(websocket.TextMessage) + if err != nil { + return + } + w.Write(message) + + // Add queued chat messages to the current websocket message. + n := len(c.send) + for i := 0; i < n; i++ { + w.Write(newline) + w.Write(<-c.send) + } + + if err := w.Close(); err != nil { + return + } + case <-ticker.C: + c.conn.SetWriteDeadline(time.Now().Add(writeWait)) + if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { + return + } + } + } +} + +// serveWs handles websocket requests from the peer. +func serveWs(hub *Hub, w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Println(err) + return + } + client := &Client{hub: hub, conn: conn, send: make(chan []byte, 256)} + client.hub.register <- client + + // Allow collection of memory referenced by the caller by doing all work in + // new goroutines. + go client.writePump() + go client.readPump() +} diff --git a/vendor/github.com/gorilla/websocket/examples/chat/home.html b/vendor/github.com/gorilla/websocket/examples/chat/home.html new file mode 100644 index 000000000..a39a0c276 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/chat/home.html @@ -0,0 +1,98 @@ + + + +Chat Example + + + + +
+
+ + +
+ + diff --git a/vendor/github.com/gorilla/websocket/examples/chat/hub.go b/vendor/github.com/gorilla/websocket/examples/chat/hub.go new file mode 100644 index 000000000..bb5c0e3bd --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/chat/hub.go @@ -0,0 +1,53 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// Hub maintains the set of active clients and broadcasts messages to the +// clients. +type Hub struct { + // Registered clients. + clients map[*Client]bool + + // Inbound messages from the clients. + broadcast chan []byte + + // Register requests from the clients. + register chan *Client + + // Unregister requests from clients. + unregister chan *Client +} + +func newHub() *Hub { + return &Hub{ + broadcast: make(chan []byte), + register: make(chan *Client), + unregister: make(chan *Client), + clients: make(map[*Client]bool), + } +} + +func (h *Hub) run() { + for { + select { + case client := <-h.register: + h.clients[client] = true + case client := <-h.unregister: + if _, ok := h.clients[client]; ok { + delete(h.clients, client) + close(client.send) + } + case message := <-h.broadcast: + for client := range h.clients { + select { + case client.send <- message: + default: + close(client.send) + delete(h.clients, client) + } + } + } + } +} diff --git a/vendor/github.com/gorilla/websocket/examples/chat/main.go b/vendor/github.com/gorilla/websocket/examples/chat/main.go new file mode 100644 index 000000000..9d4737a6f --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/chat/main.go @@ -0,0 +1,40 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "log" + "net/http" +) + +var addr = flag.String("addr", ":8080", "http service address") + +func serveHome(w http.ResponseWriter, r *http.Request) { + log.Println(r.URL) + if r.URL.Path != "/" { + http.Error(w, "Not found", http.StatusNotFound) + return + } + if r.Method != "GET" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + http.ServeFile(w, r, "home.html") +} + +func main() { + flag.Parse() + hub := newHub() + go hub.run() + http.HandleFunc("/", serveHome) + http.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) { + serveWs(hub, w, r) + }) + err := http.ListenAndServe(*addr, nil) + if err != nil { + log.Fatal("ListenAndServe: ", err) + } +} diff --git a/vendor/github.com/gorilla/websocket/examples/command/README.md b/vendor/github.com/gorilla/websocket/examples/command/README.md new file mode 100644 index 000000000..ed6f78684 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/command/README.md @@ -0,0 +1,19 @@ +# Command example + +This example connects a websocket connection to stdin and stdout of a command. +Received messages are written to stdin followed by a `\n`. Each line read from +standard out is sent as a message to the client. + + $ go get github.com/gorilla/websocket + $ cd `go list -f '{{.Dir}}' github.com/gorilla/websocket/examples/command` + $ go run main.go + # Open http://localhost:8080/ . + +Try the following commands. + + # Echo sent messages to the output area. + $ go run main.go cat + + # Run a shell.Try sending "ls" and "cat main.go". + $ go run main.go sh + diff --git a/vendor/github.com/gorilla/websocket/examples/command/home.html b/vendor/github.com/gorilla/websocket/examples/command/home.html new file mode 100644 index 000000000..19c46128a --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/command/home.html @@ -0,0 +1,102 @@ + + + +Command Example + + + + +
+
+ + +
+ + diff --git a/vendor/github.com/gorilla/websocket/examples/command/main.go b/vendor/github.com/gorilla/websocket/examples/command/main.go new file mode 100644 index 000000000..304f1a528 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/command/main.go @@ -0,0 +1,193 @@ +// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "flag" + "io" + "log" + "net/http" + "os" + "os/exec" + "time" + + "github.com/gorilla/websocket" +) + +var ( + addr = flag.String("addr", "127.0.0.1:8080", "http service address") + cmdPath string +) + +const ( + // Time allowed to write a message to the peer. + writeWait = 10 * time.Second + + // Maximum message size allowed from peer. + maxMessageSize = 8192 + + // Time allowed to read the next pong message from the peer. + pongWait = 60 * time.Second + + // Send pings to peer with this period. Must be less than pongWait. + pingPeriod = (pongWait * 9) / 10 + + // Time to wait before force close on connection. + closeGracePeriod = 10 * time.Second +) + +func pumpStdin(ws *websocket.Conn, w io.Writer) { + defer ws.Close() + ws.SetReadLimit(maxMessageSize) + ws.SetReadDeadline(time.Now().Add(pongWait)) + ws.SetPongHandler(func(string) error { ws.SetReadDeadline(time.Now().Add(pongWait)); return nil }) + for { + _, message, err := ws.ReadMessage() + if err != nil { + break + } + message = append(message, '\n') + if _, err := w.Write(message); err != nil { + break + } + } +} + +func pumpStdout(ws *websocket.Conn, r io.Reader, done chan struct{}) { + defer func() { + }() + s := bufio.NewScanner(r) + for s.Scan() { + ws.SetWriteDeadline(time.Now().Add(writeWait)) + if err := ws.WriteMessage(websocket.TextMessage, s.Bytes()); err != nil { + ws.Close() + break + } + } + if s.Err() != nil { + log.Println("scan:", s.Err()) + } + close(done) + + ws.SetWriteDeadline(time.Now().Add(writeWait)) + ws.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + time.Sleep(closeGracePeriod) + ws.Close() +} + +func ping(ws *websocket.Conn, done chan struct{}) { + ticker := time.NewTicker(pingPeriod) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait)); err != nil { + log.Println("ping:", err) + } + case <-done: + return + } + } +} + +func internalError(ws *websocket.Conn, msg string, err error) { + log.Println(msg, err) + ws.WriteMessage(websocket.TextMessage, []byte("Internal server error.")) +} + +var upgrader = websocket.Upgrader{} + +func serveWs(w http.ResponseWriter, r *http.Request) { + ws, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Println("upgrade:", err) + return + } + + defer ws.Close() + + outr, outw, err := os.Pipe() + if err != nil { + internalError(ws, "stdout:", err) + return + } + defer outr.Close() + defer outw.Close() + + inr, inw, err := os.Pipe() + if err != nil { + internalError(ws, "stdin:", err) + return + } + defer inr.Close() + defer inw.Close() + + proc, err := os.StartProcess(cmdPath, flag.Args(), &os.ProcAttr{ + Files: []*os.File{inr, outw, outw}, + }) + if err != nil { + internalError(ws, "start:", err) + return + } + + inr.Close() + outw.Close() + + stdoutDone := make(chan struct{}) + go pumpStdout(ws, outr, stdoutDone) + go ping(ws, stdoutDone) + + pumpStdin(ws, inw) + + // Some commands will exit when stdin is closed. + inw.Close() + + // Other commands need a bonk on the head. + if err := proc.Signal(os.Interrupt); err != nil { + log.Println("inter:", err) + } + + select { + case <-stdoutDone: + case <-time.After(time.Second): + // A bigger bonk on the head. + if err := proc.Signal(os.Kill); err != nil { + log.Println("term:", err) + } + <-stdoutDone + } + + if _, err := proc.Wait(); err != nil { + log.Println("wait:", err) + } +} + +func serveHome(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.Error(w, "Not found", http.StatusNotFound) + return + } + if r.Method != "GET" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + http.ServeFile(w, r, "home.html") +} + +func main() { + flag.Parse() + if len(flag.Args()) < 1 { + log.Fatal("must specify at least one argument") + } + var err error + cmdPath, err = exec.LookPath(flag.Args()[0]) + if err != nil { + log.Fatal(err) + } + http.HandleFunc("/", serveHome) + http.HandleFunc("/ws", serveWs) + log.Fatal(http.ListenAndServe(*addr, nil)) +} diff --git a/vendor/github.com/gorilla/websocket/examples/echo/README.md b/vendor/github.com/gorilla/websocket/examples/echo/README.md new file mode 100644 index 000000000..6ad79ed76 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/echo/README.md @@ -0,0 +1,17 @@ +# Client and server example + +This example shows a simple client and server. + +The server echoes messages sent to it. The client sends a message every second +and prints all messages received. + +To run the example, start the server: + + $ go run server.go + +Next, start the client: + + $ go run client.go + +The server includes a simple web client. To use the client, open +http://127.0.0.1:8080 in the browser and follow the instructions on the page. diff --git a/vendor/github.com/gorilla/websocket/examples/echo/client.go b/vendor/github.com/gorilla/websocket/examples/echo/client.go new file mode 100644 index 000000000..bf0e65733 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/echo/client.go @@ -0,0 +1,82 @@ +// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "flag" + "log" + "net/url" + "os" + "os/signal" + "time" + + "github.com/gorilla/websocket" +) + +var addr = flag.String("addr", "localhost:8080", "http service address") + +func main() { + flag.Parse() + log.SetFlags(0) + + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + + u := url.URL{Scheme: "ws", Host: *addr, Path: "/echo"} + log.Printf("connecting to %s", u.String()) + + c, _, err := websocket.DefaultDialer.Dial(u.String(), nil) + if err != nil { + log.Fatal("dial:", err) + } + defer c.Close() + + done := make(chan struct{}) + + go func() { + defer close(done) + for { + _, message, err := c.ReadMessage() + if err != nil { + log.Println("read:", err) + return + } + log.Printf("recv: %s", message) + } + }() + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for { + select { + case <-done: + return + case t := <-ticker.C: + err := c.WriteMessage(websocket.TextMessage, []byte(t.String())) + if err != nil { + log.Println("write:", err) + return + } + case <-interrupt: + log.Println("interrupt") + + // Cleanly close the connection by sending a close message and then + // waiting (with timeout) for the server to close the connection. + err := c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + if err != nil { + log.Println("write close:", err) + return + } + select { + case <-done: + case <-time.After(time.Second): + } + return + } + } +} diff --git a/vendor/github.com/gorilla/websocket/examples/echo/server.go b/vendor/github.com/gorilla/websocket/examples/echo/server.go new file mode 100644 index 000000000..ecc680c8b --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/echo/server.go @@ -0,0 +1,133 @@ +// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "flag" + "html/template" + "log" + "net/http" + + "github.com/gorilla/websocket" +) + +var addr = flag.String("addr", "localhost:8080", "http service address") + +var upgrader = websocket.Upgrader{} // use default options + +func echo(w http.ResponseWriter, r *http.Request) { + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Print("upgrade:", err) + return + } + defer c.Close() + for { + mt, message, err := c.ReadMessage() + if err != nil { + log.Println("read:", err) + break + } + log.Printf("recv: %s", message) + err = c.WriteMessage(mt, message) + if err != nil { + log.Println("write:", err) + break + } + } +} + +func home(w http.ResponseWriter, r *http.Request) { + homeTemplate.Execute(w, "ws://"+r.Host+"/echo") +} + +func main() { + flag.Parse() + log.SetFlags(0) + http.HandleFunc("/echo", echo) + http.HandleFunc("/", home) + log.Fatal(http.ListenAndServe(*addr, nil)) +} + +var homeTemplate = template.Must(template.New("").Parse(` + + + + + + + + +
+

Click "Open" to create a connection to the server, +"Send" to send a message to the server and "Close" to close the connection. +You can change the message and send multiple times. +

+

+ + +

+ +

+		 +
+
+ + +`)) diff --git a/vendor/github.com/gorilla/websocket/examples/filewatch/README.md b/vendor/github.com/gorilla/websocket/examples/filewatch/README.md new file mode 100644 index 000000000..ca4931f3b --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/filewatch/README.md @@ -0,0 +1,9 @@ +# File Watch example. + +This example sends a file to the browser client for display whenever the file is modified. + + $ go get github.com/gorilla/websocket + $ cd `go list -f '{{.Dir}}' github.com/gorilla/websocket/examples/filewatch` + $ go run main.go + # Open http://localhost:8080/ . + # Modify the file to see it update in the browser. diff --git a/vendor/github.com/gorilla/websocket/examples/filewatch/main.go b/vendor/github.com/gorilla/websocket/examples/filewatch/main.go new file mode 100644 index 000000000..b834ed397 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/examples/filewatch/main.go @@ -0,0 +1,193 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "html/template" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" + "time" + + "github.com/gorilla/websocket" +) + +const ( + // Time allowed to write the file to the client. + writeWait = 10 * time.Second + + // Time allowed to read the next pong message from the client. + pongWait = 60 * time.Second + + // Send pings to client with this period. Must be less than pongWait. + pingPeriod = (pongWait * 9) / 10 + + // Poll file for changes with this period. + filePeriod = 10 * time.Second +) + +var ( + addr = flag.String("addr", ":8080", "http service address") + homeTempl = template.Must(template.New("").Parse(homeHTML)) + filename string + upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + } +) + +func readFileIfModified(lastMod time.Time) ([]byte, time.Time, error) { + fi, err := os.Stat(filename) + if err != nil { + return nil, lastMod, err + } + if !fi.ModTime().After(lastMod) { + return nil, lastMod, nil + } + p, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fi.ModTime(), err + } + return p, fi.ModTime(), nil +} + +func reader(ws *websocket.Conn) { + defer ws.Close() + ws.SetReadLimit(512) + ws.SetReadDeadline(time.Now().Add(pongWait)) + ws.SetPongHandler(func(string) error { ws.SetReadDeadline(time.Now().Add(pongWait)); return nil }) + for { + _, _, err := ws.ReadMessage() + if err != nil { + break + } + } +} + +func writer(ws *websocket.Conn, lastMod time.Time) { + lastError := "" + pingTicker := time.NewTicker(pingPeriod) + fileTicker := time.NewTicker(filePeriod) + defer func() { + pingTicker.Stop() + fileTicker.Stop() + ws.Close() + }() + for { + select { + case <-fileTicker.C: + var p []byte + var err error + + p, lastMod, err = readFileIfModified(lastMod) + + if err != nil { + if s := err.Error(); s != lastError { + lastError = s + p = []byte(lastError) + } + } else { + lastError = "" + } + + if p != nil { + ws.SetWriteDeadline(time.Now().Add(writeWait)) + if err := ws.WriteMessage(websocket.TextMessage, p); err != nil { + return + } + } + case <-pingTicker.C: + ws.SetWriteDeadline(time.Now().Add(writeWait)) + if err := ws.WriteMessage(websocket.PingMessage, []byte{}); err != nil { + return + } + } + } +} + +func serveWs(w http.ResponseWriter, r *http.Request) { + ws, err := upgrader.Upgrade(w, r, nil) + if err != nil { + if _, ok := err.(websocket.HandshakeError); !ok { + log.Println(err) + } + return + } + + var lastMod time.Time + if n, err := strconv.ParseInt(r.FormValue("lastMod"), 16, 64); err == nil { + lastMod = time.Unix(0, n) + } + + go writer(ws, lastMod) + reader(ws) +} + +func serveHome(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.Error(w, "Not found", http.StatusNotFound) + return + } + if r.Method != "GET" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + p, lastMod, err := readFileIfModified(time.Time{}) + if err != nil { + p = []byte(err.Error()) + lastMod = time.Unix(0, 0) + } + var v = struct { + Host string + Data string + LastMod string + }{ + r.Host, + string(p), + strconv.FormatInt(lastMod.UnixNano(), 16), + } + homeTempl.Execute(w, &v) +} + +func main() { + flag.Parse() + if flag.NArg() != 1 { + log.Fatal("filename not specified") + } + filename = flag.Args()[0] + http.HandleFunc("/", serveHome) + http.HandleFunc("/ws", serveWs) + if err := http.ListenAndServe(*addr, nil); err != nil { + log.Fatal(err) + } +} + +const homeHTML = ` + + + WebSocket Example + + + 
{{.Data}}
+ + + +` diff --git a/vendor/github.com/gorilla/websocket/go.mod b/vendor/github.com/gorilla/websocket/go.mod new file mode 100644 index 000000000..93a9e924a --- /dev/null +++ b/vendor/github.com/gorilla/websocket/go.mod @@ -0,0 +1 @@ +module github.com/gorilla/websocket diff --git a/vendor/github.com/gorilla/websocket/go.sum b/vendor/github.com/gorilla/websocket/go.sum new file mode 100644 index 000000000..cf4fbbaa0 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= diff --git a/vendor/github.com/gorilla/websocket/join.go b/vendor/github.com/gorilla/websocket/join.go new file mode 100644 index 000000000..c64f8c829 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/join.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "io" + "strings" +) + +// JoinMessages concatenates received messages to create a single io.Reader. +// The string term is appended to each message. The returned reader does not +// support concurrent calls to the Read method. +func JoinMessages(c *Conn, term string) io.Reader { + return &joinReader{c: c, term: term} +} + +type joinReader struct { + c *Conn + term string + r io.Reader +} + +func (r *joinReader) Read(p []byte) (int, error) { + if r.r == nil { + var err error + _, r.r, err = r.c.NextReader() + if err != nil { + return 0, err + } + if r.term != "" { + r.r = io.MultiReader(r.r, strings.NewReader(r.term)) + } + } + n, err := r.r.Read(p) + if err == io.EOF { + err = nil + r.r = nil + } + return n, err +} diff --git a/vendor/github.com/gorilla/websocket/join_test.go b/vendor/github.com/gorilla/websocket/join_test.go new file mode 100644 index 000000000..961ac0457 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/join_test.go @@ -0,0 +1,36 @@ +// Copyright 2019 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bytes" + "io" + "strings" + "testing" +) + +func TestJoinMessages(t *testing.T) { + messages := []string{"a", "bc", "def", "ghij", "klmno", "0", "12", "345", "6789"} + for _, readChunk := range []int{1, 2, 3, 4, 5, 6, 7} { + for _, term := range []string{"", ","} { + var connBuf bytes.Buffer + wc := newTestConn(nil, &connBuf, true) + rc := newTestConn(&connBuf, nil, false) + for _, m := range messages { + wc.WriteMessage(BinaryMessage, []byte(m)) + } + + var result bytes.Buffer + _, err := io.CopyBuffer(&result, JoinMessages(rc, term), make([]byte, readChunk)) + if IsUnexpectedCloseError(err, CloseAbnormalClosure) { + t.Errorf("readChunk=%d, term=%q: unexpected error %v", readChunk, term, err) + } + want := strings.Join(messages, term) + term + if result.String() != want { + t.Errorf("readChunk=%d, term=%q, got %q, want %q", readChunk, term, result.String(), want) + } + } + } +} diff --git a/vendor/github.com/gorilla/websocket/json.go b/vendor/github.com/gorilla/websocket/json.go new file mode 100644 index 000000000..dc2c1f641 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/json.go @@ -0,0 +1,60 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "encoding/json" + "io" +) + +// WriteJSON writes the JSON encoding of v as a message. +// +// Deprecated: Use c.WriteJSON instead. +func WriteJSON(c *Conn, v interface{}) error { + return c.WriteJSON(v) +} + +// WriteJSON writes the JSON encoding of v as a message. +// +// See the documentation for encoding/json Marshal for details about the +// conversion of Go values to JSON. +func (c *Conn) WriteJSON(v interface{}) error { + w, err := c.NextWriter(TextMessage) + if err != nil { + return err + } + err1 := json.NewEncoder(w).Encode(v) + err2 := w.Close() + if err1 != nil { + return err1 + } + return err2 +} + +// ReadJSON reads the next JSON-encoded message from the connection and stores +// it in the value pointed to by v. +// +// Deprecated: Use c.ReadJSON instead. +func ReadJSON(c *Conn, v interface{}) error { + return c.ReadJSON(v) +} + +// ReadJSON reads the next JSON-encoded message from the connection and stores +// it in the value pointed to by v. +// +// See the documentation for the encoding/json Unmarshal function for details +// about the conversion of JSON to a Go value. +func (c *Conn) ReadJSON(v interface{}) error { + _, r, err := c.NextReader() + if err != nil { + return err + } + err = json.NewDecoder(r).Decode(v) + if err == io.EOF { + // One value is expected in the message. + err = io.ErrUnexpectedEOF + } + return err +} diff --git a/vendor/github.com/gorilla/websocket/json_test.go b/vendor/github.com/gorilla/websocket/json_test.go new file mode 100644 index 000000000..e4c4bdfe5 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/json_test.go @@ -0,0 +1,116 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bytes" + "encoding/json" + "io" + "reflect" + "testing" +) + +func TestJSON(t *testing.T) { + var buf bytes.Buffer + wc := newTestConn(nil, &buf, true) + rc := newTestConn(&buf, nil, false) + + var actual, expect struct { + A int + B string + } + expect.A = 1 + expect.B = "hello" + + if err := wc.WriteJSON(&expect); err != nil { + t.Fatal("write", err) + } + + if err := rc.ReadJSON(&actual); err != nil { + t.Fatal("read", err) + } + + if !reflect.DeepEqual(&actual, &expect) { + t.Fatal("equal", actual, expect) + } +} + +func TestPartialJSONRead(t *testing.T) { + var buf0, buf1 bytes.Buffer + wc := newTestConn(nil, &buf0, true) + rc := newTestConn(&buf0, &buf1, false) + + var v struct { + A int + B string + } + v.A = 1 + v.B = "hello" + + messageCount := 0 + + // Partial JSON values. + + data, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + for i := len(data) - 1; i >= 0; i-- { + if err := wc.WriteMessage(TextMessage, data[:i]); err != nil { + t.Fatal(err) + } + messageCount++ + } + + // Whitespace. + + if err := wc.WriteMessage(TextMessage, []byte(" ")); err != nil { + t.Fatal(err) + } + messageCount++ + + // Close. + + if err := wc.WriteMessage(CloseMessage, FormatCloseMessage(CloseNormalClosure, "")); err != nil { + t.Fatal(err) + } + + for i := 0; i < messageCount; i++ { + err := rc.ReadJSON(&v) + if err != io.ErrUnexpectedEOF { + t.Error("read", i, err) + } + } + + err = rc.ReadJSON(&v) + if _, ok := err.(*CloseError); !ok { + t.Error("final", err) + } +} + +func TestDeprecatedJSON(t *testing.T) { + var buf bytes.Buffer + wc := newTestConn(nil, &buf, true) + rc := newTestConn(&buf, nil, false) + + var actual, expect struct { + A int + B string + } + expect.A = 1 + expect.B = "hello" + + if err := WriteJSON(wc, &expect); err != nil { + t.Fatal("write", err) + } + + if err := ReadJSON(rc, &actual); err != nil { + t.Fatal("read", err) + } + + if !reflect.DeepEqual(&actual, &expect) { + t.Fatal("equal", actual, expect) + } +} diff --git a/vendor/github.com/gorilla/websocket/mask.go b/vendor/github.com/gorilla/websocket/mask.go new file mode 100644 index 000000000..577fce9ef --- /dev/null +++ b/vendor/github.com/gorilla/websocket/mask.go @@ -0,0 +1,54 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of +// this source code is governed by a BSD-style license that can be found in the +// LICENSE file. + +// +build !appengine + +package websocket + +import "unsafe" + +const wordSize = int(unsafe.Sizeof(uintptr(0))) + +func maskBytes(key [4]byte, pos int, b []byte) int { + // Mask one byte at a time for small buffers. + if len(b) < 2*wordSize { + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + return pos & 3 + } + + // Mask one byte at a time to word boundary. + if n := int(uintptr(unsafe.Pointer(&b[0]))) % wordSize; n != 0 { + n = wordSize - n + for i := range b[:n] { + b[i] ^= key[pos&3] + pos++ + } + b = b[n:] + } + + // Create aligned word size key. + var k [wordSize]byte + for i := range k { + k[i] = key[(pos+i)&3] + } + kw := *(*uintptr)(unsafe.Pointer(&k)) + + // Mask one word at a time. + n := (len(b) / wordSize) * wordSize + for i := 0; i < n; i += wordSize { + *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&b[0])) + uintptr(i))) ^= kw + } + + // Mask one byte at a time for remaining bytes. + b = b[n:] + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + + return pos & 3 +} diff --git a/vendor/github.com/gorilla/websocket/mask_safe.go b/vendor/github.com/gorilla/websocket/mask_safe.go new file mode 100644 index 000000000..2aac060e5 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/mask_safe.go @@ -0,0 +1,15 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of +// this source code is governed by a BSD-style license that can be found in the +// LICENSE file. + +// +build appengine + +package websocket + +func maskBytes(key [4]byte, pos int, b []byte) int { + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + return pos & 3 +} diff --git a/vendor/github.com/gorilla/websocket/mask_test.go b/vendor/github.com/gorilla/websocket/mask_test.go new file mode 100644 index 000000000..6389f436b --- /dev/null +++ b/vendor/github.com/gorilla/websocket/mask_test.go @@ -0,0 +1,72 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of +// this source code is governed by a BSD-style license that can be found in the +// LICENSE file. + +// !appengine + +package websocket + +import ( + "fmt" + "testing" +) + +func maskBytesByByte(key [4]byte, pos int, b []byte) int { + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + return pos & 3 +} + +func notzero(b []byte) int { + for i := range b { + if b[i] != 0 { + return i + } + } + return -1 +} + +func TestMaskBytes(t *testing.T) { + key := [4]byte{1, 2, 3, 4} + for size := 1; size <= 1024; size++ { + for align := 0; align < wordSize; align++ { + for pos := 0; pos < 4; pos++ { + b := make([]byte, size+align)[align:] + maskBytes(key, pos, b) + maskBytesByByte(key, pos, b) + if i := notzero(b); i >= 0 { + t.Errorf("size:%d, align:%d, pos:%d, offset:%d", size, align, pos, i) + } + } + } + } +} + +func BenchmarkMaskBytes(b *testing.B) { + for _, size := range []int{2, 4, 8, 16, 32, 512, 1024} { + b.Run(fmt.Sprintf("size-%d", size), func(b *testing.B) { + for _, align := range []int{wordSize / 2} { + b.Run(fmt.Sprintf("align-%d", align), func(b *testing.B) { + for _, fn := range []struct { + name string + fn func(key [4]byte, pos int, b []byte) int + }{ + {"byte", maskBytesByByte}, + {"word", maskBytes}, + } { + b.Run(fn.name, func(b *testing.B) { + key := newMaskKey() + data := make([]byte, size+align)[align:] + for i := 0; i < b.N; i++ { + fn.fn(key, 0, data) + } + b.SetBytes(int64(len(data))) + }) + } + }) + } + }) + } +} diff --git a/vendor/github.com/gorilla/websocket/prepared.go b/vendor/github.com/gorilla/websocket/prepared.go new file mode 100644 index 000000000..74ec565d2 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/prepared.go @@ -0,0 +1,102 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bytes" + "net" + "sync" + "time" +) + +// PreparedMessage caches on the wire representations of a message payload. +// Use PreparedMessage to efficiently send a message payload to multiple +// connections. PreparedMessage is especially useful when compression is used +// because the CPU and memory expensive compression operation can be executed +// once for a given set of compression options. +type PreparedMessage struct { + messageType int + data []byte + mu sync.Mutex + frames map[prepareKey]*preparedFrame +} + +// prepareKey defines a unique set of options to cache prepared frames in PreparedMessage. +type prepareKey struct { + isServer bool + compress bool + compressionLevel int +} + +// preparedFrame contains data in wire representation. +type preparedFrame struct { + once sync.Once + data []byte +} + +// NewPreparedMessage returns an initialized PreparedMessage. You can then send +// it to connection using WritePreparedMessage method. Valid wire +// representation will be calculated lazily only once for a set of current +// connection options. +func NewPreparedMessage(messageType int, data []byte) (*PreparedMessage, error) { + pm := &PreparedMessage{ + messageType: messageType, + frames: make(map[prepareKey]*preparedFrame), + data: data, + } + + // Prepare a plain server frame. + _, frameData, err := pm.frame(prepareKey{isServer: true, compress: false}) + if err != nil { + return nil, err + } + + // To protect against caller modifying the data argument, remember the data + // copied to the plain server frame. + pm.data = frameData[len(frameData)-len(data):] + return pm, nil +} + +func (pm *PreparedMessage) frame(key prepareKey) (int, []byte, error) { + pm.mu.Lock() + frame, ok := pm.frames[key] + if !ok { + frame = &preparedFrame{} + pm.frames[key] = frame + } + pm.mu.Unlock() + + var err error + frame.once.Do(func() { + // Prepare a frame using a 'fake' connection. + // TODO: Refactor code in conn.go to allow more direct construction of + // the frame. + mu := make(chan bool, 1) + mu <- true + var nc prepareConn + c := &Conn{ + conn: &nc, + mu: mu, + isServer: key.isServer, + compressionLevel: key.compressionLevel, + enableWriteCompression: true, + writeBuf: make([]byte, defaultWriteBufferSize+maxFrameHeaderSize), + } + if key.compress { + c.newCompressionWriter = compressNoContextTakeover + } + err = c.WriteMessage(pm.messageType, pm.data) + frame.data = nc.buf.Bytes() + }) + return pm.messageType, frame.data, err +} + +type prepareConn struct { + buf bytes.Buffer + net.Conn +} + +func (pc *prepareConn) Write(p []byte) (int, error) { return pc.buf.Write(p) } +func (pc *prepareConn) SetWriteDeadline(t time.Time) error { return nil } diff --git a/vendor/github.com/gorilla/websocket/prepared_test.go b/vendor/github.com/gorilla/websocket/prepared_test.go new file mode 100644 index 000000000..229780210 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/prepared_test.go @@ -0,0 +1,74 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bytes" + "compress/flate" + "math/rand" + "testing" +) + +var preparedMessageTests = []struct { + messageType int + isServer bool + enableWriteCompression bool + compressionLevel int +}{ + // Server + {TextMessage, true, false, flate.BestSpeed}, + {TextMessage, true, true, flate.BestSpeed}, + {TextMessage, true, true, flate.BestCompression}, + {PingMessage, true, false, flate.BestSpeed}, + {PingMessage, true, true, flate.BestSpeed}, + + // Client + {TextMessage, false, false, flate.BestSpeed}, + {TextMessage, false, true, flate.BestSpeed}, + {TextMessage, false, true, flate.BestCompression}, + {PingMessage, false, false, flate.BestSpeed}, + {PingMessage, false, true, flate.BestSpeed}, +} + +func TestPreparedMessage(t *testing.T) { + for _, tt := range preparedMessageTests { + var data = []byte("this is a test") + var buf bytes.Buffer + c := newTestConn(nil, &buf, tt.isServer) + if tt.enableWriteCompression { + c.newCompressionWriter = compressNoContextTakeover + } + c.SetCompressionLevel(tt.compressionLevel) + + // Seed random number generator for consistent frame mask. + rand.Seed(1234) + + if err := c.WriteMessage(tt.messageType, data); err != nil { + t.Fatal(err) + } + want := buf.String() + + pm, err := NewPreparedMessage(tt.messageType, data) + if err != nil { + t.Fatal(err) + } + + // Scribble on data to ensure that NewPreparedMessage takes a snapshot. + copy(data, "hello world") + + // Seed random number generator for consistent frame mask. + rand.Seed(1234) + + buf.Reset() + if err := c.WritePreparedMessage(pm); err != nil { + t.Fatal(err) + } + got := buf.String() + + if got != want { + t.Errorf("write message != prepared message for %+v", tt) + } + } +} diff --git a/vendor/github.com/gorilla/websocket/proxy.go b/vendor/github.com/gorilla/websocket/proxy.go new file mode 100644 index 000000000..e87a8c9f0 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/proxy.go @@ -0,0 +1,77 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "encoding/base64" + "errors" + "net" + "net/http" + "net/url" + "strings" +) + +type netDialerFunc func(network, addr string) (net.Conn, error) + +func (fn netDialerFunc) Dial(network, addr string) (net.Conn, error) { + return fn(network, addr) +} + +func init() { + proxy_RegisterDialerType("http", func(proxyURL *url.URL, forwardDialer proxy_Dialer) (proxy_Dialer, error) { + return &httpProxyDialer{proxyURL: proxyURL, forwardDial: forwardDialer.Dial}, nil + }) +} + +type httpProxyDialer struct { + proxyURL *url.URL + forwardDial func(network, addr string) (net.Conn, error) +} + +func (hpd *httpProxyDialer) Dial(network string, addr string) (net.Conn, error) { + hostPort, _ := hostPortNoPort(hpd.proxyURL) + conn, err := hpd.forwardDial(network, hostPort) + if err != nil { + return nil, err + } + + connectHeader := make(http.Header) + if user := hpd.proxyURL.User; user != nil { + proxyUser := user.Username() + if proxyPassword, passwordSet := user.Password(); passwordSet { + credential := base64.StdEncoding.EncodeToString([]byte(proxyUser + ":" + proxyPassword)) + connectHeader.Set("Proxy-Authorization", "Basic "+credential) + } + } + + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: addr}, + Host: addr, + Header: connectHeader, + } + + if err := connectReq.Write(conn); err != nil { + conn.Close() + return nil, err + } + + // Read response. It's OK to use and discard buffered reader here becaue + // the remote server does not speak until spoken to. + br := bufio.NewReader(conn) + resp, err := http.ReadResponse(br, connectReq) + if err != nil { + conn.Close() + return nil, err + } + + if resp.StatusCode != 200 { + conn.Close() + f := strings.SplitN(resp.Status, " ", 2) + return nil, errors.New(f[1]) + } + return conn, nil +} diff --git a/vendor/github.com/gorilla/websocket/server.go b/vendor/github.com/gorilla/websocket/server.go new file mode 100644 index 000000000..3d4480a47 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/server.go @@ -0,0 +1,363 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "errors" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// HandshakeError describes an error with the handshake from the peer. +type HandshakeError struct { + message string +} + +func (e HandshakeError) Error() string { return e.message } + +// Upgrader specifies parameters for upgrading an HTTP connection to a +// WebSocket connection. +type Upgrader struct { + // HandshakeTimeout specifies the duration for the handshake to complete. + HandshakeTimeout time.Duration + + // ReadBufferSize and WriteBufferSize specify I/O buffer sizes in bytes. If a buffer + // size is zero, then buffers allocated by the HTTP server are used. The + // I/O buffer sizes do not limit the size of the messages that can be sent + // or received. + ReadBufferSize, WriteBufferSize int + + // WriteBufferPool is a pool of buffers for write operations. If the value + // is not set, then write buffers are allocated to the connection for the + // lifetime of the connection. + // + // A pool is most useful when the application has a modest volume of writes + // across a large number of connections. + // + // Applications should use a single pool for each unique value of + // WriteBufferSize. + WriteBufferPool BufferPool + + // Subprotocols specifies the server's supported protocols in order of + // preference. If this field is not nil, then the Upgrade method negotiates a + // subprotocol by selecting the first match in this list with a protocol + // requested by the client. If there's no match, then no protocol is + // negotiated (the Sec-Websocket-Protocol header is not included in the + // handshake response). + Subprotocols []string + + // Error specifies the function for generating HTTP error responses. If Error + // is nil, then http.Error is used to generate the HTTP response. + Error func(w http.ResponseWriter, r *http.Request, status int, reason error) + + // CheckOrigin returns true if the request Origin header is acceptable. If + // CheckOrigin is nil, then a safe default is used: return false if the + // Origin request header is present and the origin host is not equal to + // request Host header. + // + // A CheckOrigin function should carefully validate the request origin to + // prevent cross-site request forgery. + CheckOrigin func(r *http.Request) bool + + // EnableCompression specify if the server should attempt to negotiate per + // message compression (RFC 7692). Setting this value to true does not + // guarantee that compression will be supported. Currently only "no context + // takeover" modes are supported. + EnableCompression bool +} + +func (u *Upgrader) returnError(w http.ResponseWriter, r *http.Request, status int, reason string) (*Conn, error) { + err := HandshakeError{reason} + if u.Error != nil { + u.Error(w, r, status, err) + } else { + w.Header().Set("Sec-Websocket-Version", "13") + http.Error(w, http.StatusText(status), status) + } + return nil, err +} + +// checkSameOrigin returns true if the origin is not set or is equal to the request host. +func checkSameOrigin(r *http.Request) bool { + origin := r.Header["Origin"] + if len(origin) == 0 { + return true + } + u, err := url.Parse(origin[0]) + if err != nil { + return false + } + return equalASCIIFold(u.Host, r.Host) +} + +func (u *Upgrader) selectSubprotocol(r *http.Request, responseHeader http.Header) string { + if u.Subprotocols != nil { + clientProtocols := Subprotocols(r) + for _, serverProtocol := range u.Subprotocols { + for _, clientProtocol := range clientProtocols { + if clientProtocol == serverProtocol { + return clientProtocol + } + } + } + } else if responseHeader != nil { + return responseHeader.Get("Sec-Websocket-Protocol") + } + return "" +} + +// Upgrade upgrades the HTTP server connection to the WebSocket protocol. +// +// The responseHeader is included in the response to the client's upgrade +// request. Use the responseHeader to specify cookies (Set-Cookie) and the +// application negotiated subprotocol (Sec-WebSocket-Protocol). +// +// If the upgrade fails, then Upgrade replies to the client with an HTTP error +// response. +func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header) (*Conn, error) { + const badHandshake = "websocket: the client is not using the websocket protocol: " + + if !tokenListContainsValue(r.Header, "Connection", "upgrade") { + return u.returnError(w, r, http.StatusBadRequest, badHandshake+"'upgrade' token not found in 'Connection' header") + } + + if !tokenListContainsValue(r.Header, "Upgrade", "websocket") { + return u.returnError(w, r, http.StatusBadRequest, badHandshake+"'websocket' token not found in 'Upgrade' header") + } + + if r.Method != "GET" { + return u.returnError(w, r, http.StatusMethodNotAllowed, badHandshake+"request method is not GET") + } + + if !tokenListContainsValue(r.Header, "Sec-Websocket-Version", "13") { + return u.returnError(w, r, http.StatusBadRequest, "websocket: unsupported version: 13 not found in 'Sec-Websocket-Version' header") + } + + if _, ok := responseHeader["Sec-Websocket-Extensions"]; ok { + return u.returnError(w, r, http.StatusInternalServerError, "websocket: application specific 'Sec-WebSocket-Extensions' headers are unsupported") + } + + checkOrigin := u.CheckOrigin + if checkOrigin == nil { + checkOrigin = checkSameOrigin + } + if !checkOrigin(r) { + return u.returnError(w, r, http.StatusForbidden, "websocket: request origin not allowed by Upgrader.CheckOrigin") + } + + challengeKey := r.Header.Get("Sec-Websocket-Key") + if challengeKey == "" { + return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: `Sec-WebSocket-Key' header is missing or blank") + } + + subprotocol := u.selectSubprotocol(r, responseHeader) + + // Negotiate PMCE + var compress bool + if u.EnableCompression { + for _, ext := range parseExtensions(r.Header) { + if ext[""] != "permessage-deflate" { + continue + } + compress = true + break + } + } + + h, ok := w.(http.Hijacker) + if !ok { + return u.returnError(w, r, http.StatusInternalServerError, "websocket: response does not implement http.Hijacker") + } + var brw *bufio.ReadWriter + netConn, brw, err := h.Hijack() + if err != nil { + return u.returnError(w, r, http.StatusInternalServerError, err.Error()) + } + + if brw.Reader.Buffered() > 0 { + netConn.Close() + return nil, errors.New("websocket: client sent data before handshake is complete") + } + + var br *bufio.Reader + if u.ReadBufferSize == 0 && bufioReaderSize(netConn, brw.Reader) > 256 { + // Reuse hijacked buffered reader as connection reader. + br = brw.Reader + } + + buf := bufioWriterBuffer(netConn, brw.Writer) + + var writeBuf []byte + if u.WriteBufferPool == nil && u.WriteBufferSize == 0 && len(buf) >= maxFrameHeaderSize+256 { + // Reuse hijacked write buffer as connection buffer. + writeBuf = buf + } + + c := newConn(netConn, true, u.ReadBufferSize, u.WriteBufferSize, u.WriteBufferPool, br, writeBuf) + c.subprotocol = subprotocol + + if compress { + c.newCompressionWriter = compressNoContextTakeover + c.newDecompressionReader = decompressNoContextTakeover + } + + // Use larger of hijacked buffer and connection write buffer for header. + p := buf + if len(c.writeBuf) > len(p) { + p = c.writeBuf + } + p = p[:0] + + p = append(p, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: "...) + p = append(p, computeAcceptKey(challengeKey)...) + p = append(p, "\r\n"...) + if c.subprotocol != "" { + p = append(p, "Sec-WebSocket-Protocol: "...) + p = append(p, c.subprotocol...) + p = append(p, "\r\n"...) + } + if compress { + p = append(p, "Sec-WebSocket-Extensions: permessage-deflate; server_no_context_takeover; client_no_context_takeover\r\n"...) + } + for k, vs := range responseHeader { + if k == "Sec-Websocket-Protocol" { + continue + } + for _, v := range vs { + p = append(p, k...) + p = append(p, ": "...) + for i := 0; i < len(v); i++ { + b := v[i] + if b <= 31 { + // prevent response splitting. + b = ' ' + } + p = append(p, b) + } + p = append(p, "\r\n"...) + } + } + p = append(p, "\r\n"...) + + // Clear deadlines set by HTTP server. + netConn.SetDeadline(time.Time{}) + + if u.HandshakeTimeout > 0 { + netConn.SetWriteDeadline(time.Now().Add(u.HandshakeTimeout)) + } + if _, err = netConn.Write(p); err != nil { + netConn.Close() + return nil, err + } + if u.HandshakeTimeout > 0 { + netConn.SetWriteDeadline(time.Time{}) + } + + return c, nil +} + +// Upgrade upgrades the HTTP server connection to the WebSocket protocol. +// +// Deprecated: Use websocket.Upgrader instead. +// +// Upgrade does not perform origin checking. The application is responsible for +// checking the Origin header before calling Upgrade. An example implementation +// of the same origin policy check is: +// +// if req.Header.Get("Origin") != "http://"+req.Host { +// http.Error(w, "Origin not allowed", http.StatusForbidden) +// return +// } +// +// If the endpoint supports subprotocols, then the application is responsible +// for negotiating the protocol used on the connection. Use the Subprotocols() +// function to get the subprotocols requested by the client. Use the +// Sec-Websocket-Protocol response header to specify the subprotocol selected +// by the application. +// +// The responseHeader is included in the response to the client's upgrade +// request. Use the responseHeader to specify cookies (Set-Cookie) and the +// negotiated subprotocol (Sec-Websocket-Protocol). +// +// The connection buffers IO to the underlying network connection. The +// readBufSize and writeBufSize parameters specify the size of the buffers to +// use. Messages can be larger than the buffers. +// +// If the request is not a valid WebSocket handshake, then Upgrade returns an +// error of type HandshakeError. Applications should handle this error by +// replying to the client with an HTTP error response. +func Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header, readBufSize, writeBufSize int) (*Conn, error) { + u := Upgrader{ReadBufferSize: readBufSize, WriteBufferSize: writeBufSize} + u.Error = func(w http.ResponseWriter, r *http.Request, status int, reason error) { + // don't return errors to maintain backwards compatibility + } + u.CheckOrigin = func(r *http.Request) bool { + // allow all connections by default + return true + } + return u.Upgrade(w, r, responseHeader) +} + +// Subprotocols returns the subprotocols requested by the client in the +// Sec-Websocket-Protocol header. +func Subprotocols(r *http.Request) []string { + h := strings.TrimSpace(r.Header.Get("Sec-Websocket-Protocol")) + if h == "" { + return nil + } + protocols := strings.Split(h, ",") + for i := range protocols { + protocols[i] = strings.TrimSpace(protocols[i]) + } + return protocols +} + +// IsWebSocketUpgrade returns true if the client requested upgrade to the +// WebSocket protocol. +func IsWebSocketUpgrade(r *http.Request) bool { + return tokenListContainsValue(r.Header, "Connection", "upgrade") && + tokenListContainsValue(r.Header, "Upgrade", "websocket") +} + +// bufioReaderSize size returns the size of a bufio.Reader. +func bufioReaderSize(originalReader io.Reader, br *bufio.Reader) int { + // This code assumes that peek on a reset reader returns + // bufio.Reader.buf[:0]. + // TODO: Use bufio.Reader.Size() after Go 1.10 + br.Reset(originalReader) + if p, err := br.Peek(0); err == nil { + return cap(p) + } + return 0 +} + +// writeHook is an io.Writer that records the last slice passed to it vio +// io.Writer.Write. +type writeHook struct { + p []byte +} + +func (wh *writeHook) Write(p []byte) (int, error) { + wh.p = p + return len(p), nil +} + +// bufioWriterBuffer grabs the buffer from a bufio.Writer. +func bufioWriterBuffer(originalWriter io.Writer, bw *bufio.Writer) []byte { + // This code assumes that bufio.Writer.buf[:1] is passed to the + // bufio.Writer's underlying writer. + var wh writeHook + bw.Reset(&wh) + bw.WriteByte(0) + bw.Flush() + + bw.Reset(originalWriter) + + return wh.p[:cap(wh.p)] +} diff --git a/vendor/github.com/gorilla/websocket/server_test.go b/vendor/github.com/gorilla/websocket/server_test.go new file mode 100644 index 000000000..456c1db5e --- /dev/null +++ b/vendor/github.com/gorilla/websocket/server_test.go @@ -0,0 +1,119 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "bytes" + "net" + "net/http" + "reflect" + "strings" + "testing" +) + +var subprotocolTests = []struct { + h string + protocols []string +}{ + {"", nil}, + {"foo", []string{"foo"}}, + {"foo,bar", []string{"foo", "bar"}}, + {"foo, bar", []string{"foo", "bar"}}, + {" foo, bar", []string{"foo", "bar"}}, + {" foo, bar ", []string{"foo", "bar"}}, +} + +func TestSubprotocols(t *testing.T) { + for _, st := range subprotocolTests { + r := http.Request{Header: http.Header{"Sec-Websocket-Protocol": {st.h}}} + protocols := Subprotocols(&r) + if !reflect.DeepEqual(st.protocols, protocols) { + t.Errorf("SubProtocols(%q) returned %#v, want %#v", st.h, protocols, st.protocols) + } + } +} + +var isWebSocketUpgradeTests = []struct { + ok bool + h http.Header +}{ + {false, http.Header{"Upgrade": {"websocket"}}}, + {false, http.Header{"Connection": {"upgrade"}}}, + {true, http.Header{"Connection": {"upgRade"}, "Upgrade": {"WebSocket"}}}, +} + +func TestIsWebSocketUpgrade(t *testing.T) { + for _, tt := range isWebSocketUpgradeTests { + ok := IsWebSocketUpgrade(&http.Request{Header: tt.h}) + if tt.ok != ok { + t.Errorf("IsWebSocketUpgrade(%v) returned %v, want %v", tt.h, ok, tt.ok) + } + } +} + +var checkSameOriginTests = []struct { + ok bool + r *http.Request +}{ + {false, &http.Request{Host: "example.org", Header: map[string][]string{"Origin": {"https://other.org"}}}}, + {true, &http.Request{Host: "example.org", Header: map[string][]string{"Origin": {"https://example.org"}}}}, + {true, &http.Request{Host: "Example.org", Header: map[string][]string{"Origin": {"https://example.org"}}}}, +} + +func TestCheckSameOrigin(t *testing.T) { + for _, tt := range checkSameOriginTests { + ok := checkSameOrigin(tt.r) + if tt.ok != ok { + t.Errorf("checkSameOrigin(%+v) returned %v, want %v", tt.r, ok, tt.ok) + } + } +} + +type reuseTestResponseWriter struct { + brw *bufio.ReadWriter + http.ResponseWriter +} + +func (resp *reuseTestResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + return fakeNetConn{strings.NewReader(""), &bytes.Buffer{}}, resp.brw, nil +} + +var bufioReuseTests = []struct { + n int + reuse bool +}{ + {4096, true}, + {128, false}, +} + +func TestBufioReuse(t *testing.T) { + for i, tt := range bufioReuseTests { + br := bufio.NewReaderSize(strings.NewReader(""), tt.n) + bw := bufio.NewWriterSize(&bytes.Buffer{}, tt.n) + resp := &reuseTestResponseWriter{ + brw: bufio.NewReadWriter(br, bw), + } + upgrader := Upgrader{} + c, err := upgrader.Upgrade(resp, &http.Request{ + Method: "GET", + Header: http.Header{ + "Upgrade": []string{"websocket"}, + "Connection": []string{"upgrade"}, + "Sec-Websocket-Key": []string{"dGhlIHNhbXBsZSBub25jZQ=="}, + "Sec-Websocket-Version": []string{"13"}, + }}, nil) + if err != nil { + t.Fatal(err) + } + if reuse := c.br == br; reuse != tt.reuse { + t.Errorf("%d: buffered reader reuse=%v, want %v", i, reuse, tt.reuse) + } + writeBuf := bufioWriterBuffer(c.UnderlyingConn(), bw) + if reuse := &c.writeBuf[0] == &writeBuf[0]; reuse != tt.reuse { + t.Errorf("%d: write buffer reuse=%v, want %v", i, reuse, tt.reuse) + } + } +} diff --git a/vendor/github.com/gorilla/websocket/trace.go b/vendor/github.com/gorilla/websocket/trace.go new file mode 100644 index 000000000..834f122a0 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/trace.go @@ -0,0 +1,19 @@ +// +build go1.8 + +package websocket + +import ( + "crypto/tls" + "net/http/httptrace" +) + +func doHandshakeWithTrace(trace *httptrace.ClientTrace, tlsConn *tls.Conn, cfg *tls.Config) error { + if trace.TLSHandshakeStart != nil { + trace.TLSHandshakeStart() + } + err := doHandshake(tlsConn, cfg) + if trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(tlsConn.ConnectionState(), err) + } + return err +} diff --git a/vendor/github.com/gorilla/websocket/trace_17.go b/vendor/github.com/gorilla/websocket/trace_17.go new file mode 100644 index 000000000..77d05a0b5 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/trace_17.go @@ -0,0 +1,12 @@ +// +build !go1.8 + +package websocket + +import ( + "crypto/tls" + "net/http/httptrace" +) + +func doHandshakeWithTrace(trace *httptrace.ClientTrace, tlsConn *tls.Conn, cfg *tls.Config) error { + return doHandshake(tlsConn, cfg) +} diff --git a/vendor/github.com/gorilla/websocket/util.go b/vendor/github.com/gorilla/websocket/util.go new file mode 100644 index 000000000..7bf2f66c6 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/util.go @@ -0,0 +1,283 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "crypto/rand" + "crypto/sha1" + "encoding/base64" + "io" + "net/http" + "strings" + "unicode/utf8" +) + +var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") + +func computeAcceptKey(challengeKey string) string { + h := sha1.New() + h.Write([]byte(challengeKey)) + h.Write(keyGUID) + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} + +func generateChallengeKey() (string, error) { + p := make([]byte, 16) + if _, err := io.ReadFull(rand.Reader, p); err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(p), nil +} + +// Token octets per RFC 2616. +var isTokenOctet = [256]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, +} + +// skipSpace returns a slice of the string s with all leading RFC 2616 linear +// whitespace removed. +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if b := s[i]; b != ' ' && b != '\t' { + break + } + } + return s[i:] +} + +// nextToken returns the leading RFC 2616 token of s and the string following +// the token. +func nextToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if !isTokenOctet[s[i]] { + break + } + } + return s[:i], s[i:] +} + +// nextTokenOrQuoted returns the leading token or quoted string per RFC 2616 +// and the string following the token or quoted string. +func nextTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return nextToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + 1; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} + +// equalASCIIFold returns true if s is equal to t with ASCII case folding as +// defined in RFC 4790. +func equalASCIIFold(s, t string) bool { + for s != "" && t != "" { + sr, size := utf8.DecodeRuneInString(s) + s = s[size:] + tr, size := utf8.DecodeRuneInString(t) + t = t[size:] + if sr == tr { + continue + } + if 'A' <= sr && sr <= 'Z' { + sr = sr + 'a' - 'A' + } + if 'A' <= tr && tr <= 'Z' { + tr = tr + 'a' - 'A' + } + if sr != tr { + return false + } + } + return s == t +} + +// tokenListContainsValue returns true if the 1#token header with the given +// name contains a token equal to value with ASCII case folding. +func tokenListContainsValue(header http.Header, name string, value string) bool { +headers: + for _, s := range header[name] { + for { + var t string + t, s = nextToken(skipSpace(s)) + if t == "" { + continue headers + } + s = skipSpace(s) + if s != "" && s[0] != ',' { + continue headers + } + if equalASCIIFold(t, value) { + return true + } + if s == "" { + continue headers + } + s = s[1:] + } + } + return false +} + +// parseExtensions parses WebSocket extensions from a header. +func parseExtensions(header http.Header) []map[string]string { + // From RFC 6455: + // + // Sec-WebSocket-Extensions = extension-list + // extension-list = 1#extension + // extension = extension-token *( ";" extension-param ) + // extension-token = registered-token + // registered-token = token + // extension-param = token [ "=" (token | quoted-string) ] + // ;When using the quoted-string syntax variant, the value + // ;after quoted-string unescaping MUST conform to the + // ;'token' ABNF. + + var result []map[string]string +headers: + for _, s := range header["Sec-Websocket-Extensions"] { + for { + var t string + t, s = nextToken(skipSpace(s)) + if t == "" { + continue headers + } + ext := map[string]string{"": t} + for { + s = skipSpace(s) + if !strings.HasPrefix(s, ";") { + break + } + var k string + k, s = nextToken(skipSpace(s[1:])) + if k == "" { + continue headers + } + s = skipSpace(s) + var v string + if strings.HasPrefix(s, "=") { + v, s = nextTokenOrQuoted(skipSpace(s[1:])) + s = skipSpace(s) + } + if s != "" && s[0] != ',' && s[0] != ';' { + continue headers + } + ext[k] = v + } + if s != "" && s[0] != ',' { + continue headers + } + result = append(result, ext) + if s == "" { + continue headers + } + s = s[1:] + } + } + return result +} diff --git a/vendor/github.com/gorilla/websocket/util_test.go b/vendor/github.com/gorilla/websocket/util_test.go new file mode 100644 index 000000000..af710ba76 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/util_test.go @@ -0,0 +1,96 @@ +// Copyright 2014 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "net/http" + "reflect" + "testing" +) + +var equalASCIIFoldTests = []struct { + t, s string + eq bool +}{ + {"WebSocket", "websocket", true}, + {"websocket", "WebSocket", true}, + {"Öyster", "öyster", false}, + {"WebSocket", "WetSocket", false}, +} + +func TestEqualASCIIFold(t *testing.T) { + for _, tt := range equalASCIIFoldTests { + eq := equalASCIIFold(tt.s, tt.t) + if eq != tt.eq { + t.Errorf("equalASCIIFold(%q, %q) = %v, want %v", tt.s, tt.t, eq, tt.eq) + } + } +} + +var tokenListContainsValueTests = []struct { + value string + ok bool +}{ + {"WebSocket", true}, + {"WEBSOCKET", true}, + {"websocket", true}, + {"websockets", false}, + {"x websocket", false}, + {"websocket x", false}, + {"other,websocket,more", true}, + {"other, websocket, more", true}, +} + +func TestTokenListContainsValue(t *testing.T) { + for _, tt := range tokenListContainsValueTests { + h := http.Header{"Upgrade": {tt.value}} + ok := tokenListContainsValue(h, "Upgrade", "websocket") + if ok != tt.ok { + t.Errorf("tokenListContainsValue(h, n, %q) = %v, want %v", tt.value, ok, tt.ok) + } + } +} + +var parseExtensionTests = []struct { + value string + extensions []map[string]string +}{ + {`foo`, []map[string]string{{"": "foo"}}}, + {`foo, bar; baz=2`, []map[string]string{ + {"": "foo"}, + {"": "bar", "baz": "2"}}}, + {`foo; bar="b,a;z"`, []map[string]string{ + {"": "foo", "bar": "b,a;z"}}}, + {`foo , bar; baz = 2`, []map[string]string{ + {"": "foo"}, + {"": "bar", "baz": "2"}}}, + {`foo, bar; baz=2 junk`, []map[string]string{ + {"": "foo"}}}, + {`foo junk, bar; baz=2 junk`, nil}, + {`mux; max-channels=4; flow-control, deflate-stream`, []map[string]string{ + {"": "mux", "max-channels": "4", "flow-control": ""}, + {"": "deflate-stream"}}}, + {`permessage-foo; x="10"`, []map[string]string{ + {"": "permessage-foo", "x": "10"}}}, + {`permessage-foo; use_y, permessage-foo`, []map[string]string{ + {"": "permessage-foo", "use_y": ""}, + {"": "permessage-foo"}}}, + {`permessage-deflate; client_max_window_bits; server_max_window_bits=10 , permessage-deflate; client_max_window_bits`, []map[string]string{ + {"": "permessage-deflate", "client_max_window_bits": "", "server_max_window_bits": "10"}, + {"": "permessage-deflate", "client_max_window_bits": ""}}}, + {"permessage-deflate; server_no_context_takeover; client_max_window_bits=15", []map[string]string{ + {"": "permessage-deflate", "server_no_context_takeover": "", "client_max_window_bits": "15"}, + }}, +} + +func TestParseExtensions(t *testing.T) { + for _, tt := range parseExtensionTests { + h := http.Header{http.CanonicalHeaderKey("Sec-WebSocket-Extensions"): {tt.value}} + extensions := parseExtensions(h) + if !reflect.DeepEqual(extensions, tt.extensions) { + t.Errorf("parseExtensions(%q)\n = %v,\nwant %v", tt.value, extensions, tt.extensions) + } + } +} diff --git a/vendor/github.com/gorilla/websocket/x_net_proxy.go b/vendor/github.com/gorilla/websocket/x_net_proxy.go new file mode 100644 index 000000000..2e668f6b8 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/x_net_proxy.go @@ -0,0 +1,473 @@ +// Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. +//go:generate bundle -o x_net_proxy.go golang.org/x/net/proxy + +// Package proxy provides support for a variety of protocols to proxy network +// data. +// + +package websocket + +import ( + "errors" + "io" + "net" + "net/url" + "os" + "strconv" + "strings" + "sync" +) + +type proxy_direct struct{} + +// Direct is a direct proxy: one that makes network connections directly. +var proxy_Direct = proxy_direct{} + +func (proxy_direct) Dial(network, addr string) (net.Conn, error) { + return net.Dial(network, addr) +} + +// A PerHost directs connections to a default Dialer unless the host name +// requested matches one of a number of exceptions. +type proxy_PerHost struct { + def, bypass proxy_Dialer + + bypassNetworks []*net.IPNet + bypassIPs []net.IP + bypassZones []string + bypassHosts []string +} + +// NewPerHost returns a PerHost Dialer that directs connections to either +// defaultDialer or bypass, depending on whether the connection matches one of +// the configured rules. +func proxy_NewPerHost(defaultDialer, bypass proxy_Dialer) *proxy_PerHost { + return &proxy_PerHost{ + def: defaultDialer, + bypass: bypass, + } +} + +// Dial connects to the address addr on the given network through either +// defaultDialer or bypass. +func (p *proxy_PerHost) Dial(network, addr string) (c net.Conn, err error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + return p.dialerForRequest(host).Dial(network, addr) +} + +func (p *proxy_PerHost) dialerForRequest(host string) proxy_Dialer { + if ip := net.ParseIP(host); ip != nil { + for _, net := range p.bypassNetworks { + if net.Contains(ip) { + return p.bypass + } + } + for _, bypassIP := range p.bypassIPs { + if bypassIP.Equal(ip) { + return p.bypass + } + } + return p.def + } + + for _, zone := range p.bypassZones { + if strings.HasSuffix(host, zone) { + return p.bypass + } + if host == zone[1:] { + // For a zone ".example.com", we match "example.com" + // too. + return p.bypass + } + } + for _, bypassHost := range p.bypassHosts { + if bypassHost == host { + return p.bypass + } + } + return p.def +} + +// AddFromString parses a string that contains comma-separated values +// specifying hosts that should use the bypass proxy. Each value is either an +// IP address, a CIDR range, a zone (*.example.com) or a host name +// (localhost). A best effort is made to parse the string and errors are +// ignored. +func (p *proxy_PerHost) AddFromString(s string) { + hosts := strings.Split(s, ",") + for _, host := range hosts { + host = strings.TrimSpace(host) + if len(host) == 0 { + continue + } + if strings.Contains(host, "/") { + // We assume that it's a CIDR address like 127.0.0.0/8 + if _, net, err := net.ParseCIDR(host); err == nil { + p.AddNetwork(net) + } + continue + } + if ip := net.ParseIP(host); ip != nil { + p.AddIP(ip) + continue + } + if strings.HasPrefix(host, "*.") { + p.AddZone(host[1:]) + continue + } + p.AddHost(host) + } +} + +// AddIP specifies an IP address that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match an IP. +func (p *proxy_PerHost) AddIP(ip net.IP) { + p.bypassIPs = append(p.bypassIPs, ip) +} + +// AddNetwork specifies an IP range that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match. +func (p *proxy_PerHost) AddNetwork(net *net.IPNet) { + p.bypassNetworks = append(p.bypassNetworks, net) +} + +// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of +// "example.com" matches "example.com" and all of its subdomains. +func (p *proxy_PerHost) AddZone(zone string) { + if strings.HasSuffix(zone, ".") { + zone = zone[:len(zone)-1] + } + if !strings.HasPrefix(zone, ".") { + zone = "." + zone + } + p.bypassZones = append(p.bypassZones, zone) +} + +// AddHost specifies a host name that will use the bypass proxy. +func (p *proxy_PerHost) AddHost(host string) { + if strings.HasSuffix(host, ".") { + host = host[:len(host)-1] + } + p.bypassHosts = append(p.bypassHosts, host) +} + +// A Dialer is a means to establish a connection. +type proxy_Dialer interface { + // Dial connects to the given address via the proxy. + Dial(network, addr string) (c net.Conn, err error) +} + +// Auth contains authentication parameters that specific Dialers may require. +type proxy_Auth struct { + User, Password string +} + +// FromEnvironment returns the dialer specified by the proxy related variables in +// the environment. +func proxy_FromEnvironment() proxy_Dialer { + allProxy := proxy_allProxyEnv.Get() + if len(allProxy) == 0 { + return proxy_Direct + } + + proxyURL, err := url.Parse(allProxy) + if err != nil { + return proxy_Direct + } + proxy, err := proxy_FromURL(proxyURL, proxy_Direct) + if err != nil { + return proxy_Direct + } + + noProxy := proxy_noProxyEnv.Get() + if len(noProxy) == 0 { + return proxy + } + + perHost := proxy_NewPerHost(proxy, proxy_Direct) + perHost.AddFromString(noProxy) + return perHost +} + +// proxySchemes is a map from URL schemes to a function that creates a Dialer +// from a URL with such a scheme. +var proxy_proxySchemes map[string]func(*url.URL, proxy_Dialer) (proxy_Dialer, error) + +// RegisterDialerType takes a URL scheme and a function to generate Dialers from +// a URL with that scheme and a forwarding Dialer. Registered schemes are used +// by FromURL. +func proxy_RegisterDialerType(scheme string, f func(*url.URL, proxy_Dialer) (proxy_Dialer, error)) { + if proxy_proxySchemes == nil { + proxy_proxySchemes = make(map[string]func(*url.URL, proxy_Dialer) (proxy_Dialer, error)) + } + proxy_proxySchemes[scheme] = f +} + +// FromURL returns a Dialer given a URL specification and an underlying +// Dialer for it to make network requests. +func proxy_FromURL(u *url.URL, forward proxy_Dialer) (proxy_Dialer, error) { + var auth *proxy_Auth + if u.User != nil { + auth = new(proxy_Auth) + auth.User = u.User.Username() + if p, ok := u.User.Password(); ok { + auth.Password = p + } + } + + switch u.Scheme { + case "socks5": + return proxy_SOCKS5("tcp", u.Host, auth, forward) + } + + // If the scheme doesn't match any of the built-in schemes, see if it + // was registered by another package. + if proxy_proxySchemes != nil { + if f, ok := proxy_proxySchemes[u.Scheme]; ok { + return f(u, forward) + } + } + + return nil, errors.New("proxy: unknown scheme: " + u.Scheme) +} + +var ( + proxy_allProxyEnv = &proxy_envOnce{ + names: []string{"ALL_PROXY", "all_proxy"}, + } + proxy_noProxyEnv = &proxy_envOnce{ + names: []string{"NO_PROXY", "no_proxy"}, + } +) + +// envOnce looks up an environment variable (optionally by multiple +// names) once. It mitigates expensive lookups on some platforms +// (e.g. Windows). +// (Borrowed from net/http/transport.go) +type proxy_envOnce struct { + names []string + once sync.Once + val string +} + +func (e *proxy_envOnce) Get() string { + e.once.Do(e.init) + return e.val +} + +func (e *proxy_envOnce) init() { + for _, n := range e.names { + e.val = os.Getenv(n) + if e.val != "" { + return + } + } +} + +// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given address +// with an optional username and password. See RFC 1928 and RFC 1929. +func proxy_SOCKS5(network, addr string, auth *proxy_Auth, forward proxy_Dialer) (proxy_Dialer, error) { + s := &proxy_socks5{ + network: network, + addr: addr, + forward: forward, + } + if auth != nil { + s.user = auth.User + s.password = auth.Password + } + + return s, nil +} + +type proxy_socks5 struct { + user, password string + network, addr string + forward proxy_Dialer +} + +const proxy_socks5Version = 5 + +const ( + proxy_socks5AuthNone = 0 + proxy_socks5AuthPassword = 2 +) + +const proxy_socks5Connect = 1 + +const ( + proxy_socks5IP4 = 1 + proxy_socks5Domain = 3 + proxy_socks5IP6 = 4 +) + +var proxy_socks5Errors = []string{ + "", + "general failure", + "connection forbidden", + "network unreachable", + "host unreachable", + "connection refused", + "TTL expired", + "command not supported", + "address type not supported", +} + +// Dial connects to the address addr on the given network via the SOCKS5 proxy. +func (s *proxy_socks5) Dial(network, addr string) (net.Conn, error) { + switch network { + case "tcp", "tcp6", "tcp4": + default: + return nil, errors.New("proxy: no support for SOCKS5 proxy connections of type " + network) + } + + conn, err := s.forward.Dial(s.network, s.addr) + if err != nil { + return nil, err + } + if err := s.connect(conn, addr); err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + +// connect takes an existing connection to a socks5 proxy server, +// and commands the server to extend that connection to target, +// which must be a canonical address with a host and port. +func (s *proxy_socks5) connect(conn net.Conn, target string) error { + host, portStr, err := net.SplitHostPort(target) + if err != nil { + return err + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return errors.New("proxy: failed to parse port number: " + portStr) + } + if port < 1 || port > 0xffff { + return errors.New("proxy: port number out of range: " + portStr) + } + + // the size here is just an estimate + buf := make([]byte, 0, 6+len(host)) + + buf = append(buf, proxy_socks5Version) + if len(s.user) > 0 && len(s.user) < 256 && len(s.password) < 256 { + buf = append(buf, 2 /* num auth methods */, proxy_socks5AuthNone, proxy_socks5AuthPassword) + } else { + buf = append(buf, 1 /* num auth methods */, proxy_socks5AuthNone) + } + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write greeting to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read greeting from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + if buf[0] != 5 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " has unexpected version " + strconv.Itoa(int(buf[0]))) + } + if buf[1] == 0xff { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " requires authentication") + } + + // See RFC 1929 + if buf[1] == proxy_socks5AuthPassword { + buf = buf[:0] + buf = append(buf, 1 /* password protocol version */) + buf = append(buf, uint8(len(s.user))) + buf = append(buf, s.user...) + buf = append(buf, uint8(len(s.password))) + buf = append(buf, s.password...) + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write authentication request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read authentication reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if buf[1] != 0 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " rejected username/password") + } + } + + buf = buf[:0] + buf = append(buf, proxy_socks5Version, proxy_socks5Connect, 0 /* reserved */) + + if ip := net.ParseIP(host); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + buf = append(buf, proxy_socks5IP4) + ip = ip4 + } else { + buf = append(buf, proxy_socks5IP6) + } + buf = append(buf, ip...) + } else { + if len(host) > 255 { + return errors.New("proxy: destination host name too long: " + host) + } + buf = append(buf, proxy_socks5Domain) + buf = append(buf, byte(len(host))) + buf = append(buf, host...) + } + buf = append(buf, byte(port>>8), byte(port)) + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write connect request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:4]); err != nil { + return errors.New("proxy: failed to read connect reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + failure := "unknown error" + if int(buf[1]) < len(proxy_socks5Errors) { + failure = proxy_socks5Errors[buf[1]] + } + + if len(failure) > 0 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " failed to connect: " + failure) + } + + bytesToDiscard := 0 + switch buf[3] { + case proxy_socks5IP4: + bytesToDiscard = net.IPv4len + case proxy_socks5IP6: + bytesToDiscard = net.IPv6len + case proxy_socks5Domain: + _, err := io.ReadFull(conn, buf[:1]) + if err != nil { + return errors.New("proxy: failed to read domain length from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + bytesToDiscard = int(buf[0]) + default: + return errors.New("proxy: got unknown address type " + strconv.Itoa(int(buf[3])) + " from SOCKS5 proxy at " + s.addr) + } + + if cap(buf) < bytesToDiscard { + buf = make([]byte, bytesToDiscard) + } else { + buf = buf[:bytesToDiscard] + } + if _, err := io.ReadFull(conn, buf); err != nil { + return errors.New("proxy: failed to read address from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + // Also need to discard the port number + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read port from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + return nil +} diff --git a/vendor/github.com/prometheus/procfs/.golangci.yml b/vendor/github.com/prometheus/procfs/.golangci.yml new file mode 100644 index 000000000..438ca92ec --- /dev/null +++ b/vendor/github.com/prometheus/procfs/.golangci.yml @@ -0,0 +1,6 @@ +# Run only staticcheck for now. Additional linters will be enabled one-by-one. +linters: + enable: + - staticcheck + - govet + disable-all: true diff --git a/vendor/github.com/prometheus/procfs/go.sum b/vendor/github.com/prometheus/procfs/go.sum new file mode 100644 index 000000000..7827dd3d5 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/vendor/github.com/prometheus/procfs/internal/fs/fs.go b/vendor/github.com/prometheus/procfs/internal/fs/fs.go new file mode 100644 index 000000000..c66a1cf80 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/internal/fs/fs.go @@ -0,0 +1,52 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + "os" + "path/filepath" +) + +const ( + // DefaultProcMountPoint is the common mount point of the proc filesystem. + DefaultProcMountPoint = "/proc" + + // DefaultSysMountPoint is the common mount point of the sys filesystem. + DefaultSysMountPoint = "/sys" +) + +// FS represents a pseudo-filesystem, normally /proc or /sys, which provides an +// interface to kernel data structures. +type FS string + +// NewFS returns a new FS mounted under the given mountPoint. It will error +// if the mount point can't be read. +func NewFS(mountPoint string) (FS, error) { + info, err := os.Stat(mountPoint) + if err != nil { + return "", fmt.Errorf("could not read %s: %s", mountPoint, err) + } + if !info.IsDir() { + return "", fmt.Errorf("mount point %s is not a directory", mountPoint) + } + + return FS(mountPoint), nil +} + +// Path appends the given path elements to the filesystem path, adding separators +// as necessary. +func (fs FS) Path(p ...string) string { + return filepath.Join(append([]string{string(fs)}, p...)...) +} diff --git a/vendor/github.com/tiglabs/raft/.gitignore b/vendor/github.com/tiglabs/raft/.gitignore new file mode 100644 index 000000000..daf913b1b --- /dev/null +++ b/vendor/github.com/tiglabs/raft/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/vendor/modules.txt b/vendor/modules.txt new file mode 100644 index 000000000..15aee82ed --- /dev/null +++ b/vendor/modules.txt @@ -0,0 +1,56 @@ +# bazil.org/fuse v0.0.0-20180421153158-65cc252bf669 +bazil.org/fuse +bazil.org/fuse/fs +bazil.org/fuse/fuseutil +# github.com/beorn7/perks v1.0.0 +github.com/beorn7/perks/quantile +# github.com/golang/protobuf v1.3.1 +github.com/golang/protobuf/proto +# github.com/google/btree v1.0.0 +github.com/google/btree +# github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f +github.com/jacobsa/daemonize +# github.com/jacobsa/fuse v0.0.0-20180417054321-cd3959611bcb +github.com/jacobsa/fuse +github.com/jacobsa/fuse/fuseutil +github.com/jacobsa/fuse/fuseops +github.com/jacobsa/fuse/internal/buffer +github.com/jacobsa/fuse/internal/freelist +github.com/jacobsa/fuse/internal/fusekernel +# github.com/matttproud/golang_protobuf_extensions v1.0.1 +github.com/matttproud/golang_protobuf_extensions/pbutil +# github.com/moul/http2curl v1.0.0 +github.com/moul/http2curl +# github.com/parnurzeal/gorequest v0.2.15 +github.com/parnurzeal/gorequest +# github.com/pkg/errors v0.8.0 +github.com/pkg/errors +# github.com/prometheus/client_golang v1.0.0 +github.com/prometheus/client_golang/prometheus +github.com/prometheus/client_golang/prometheus/promhttp +github.com/prometheus/client_golang/prometheus/internal +# github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 +github.com/prometheus/client_model/go +# github.com/prometheus/common v0.4.1 +github.com/prometheus/common/expfmt +github.com/prometheus/common/model +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg +# github.com/prometheus/procfs v0.0.2 +github.com/prometheus/procfs +github.com/prometheus/procfs/internal/fs +# github.com/tecbot/gorocksdb v0.0.0-20190519120508-025c3cf4ffb4 +github.com/tecbot/gorocksdb +# github.com/tiglabs/raft v0.0.0-20190131082128-45667fcdb8b8 +github.com/tiglabs/raft +github.com/tiglabs/raft/proto +github.com/tiglabs/raft/logger +github.com/tiglabs/raft/storage/wal +github.com/tiglabs/raft/util/log +github.com/tiglabs/raft/storage +github.com/tiglabs/raft/util +github.com/tiglabs/raft/util/bufalloc +# golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 +golang.org/x/net/context +golang.org/x/net/publicsuffix +# golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a +golang.org/x/sys/windows diff --git a/vendor/rocksdb-5.9.2/build_tools/build_detect_platform b/vendor/rocksdb-5.9.2/build_tools/build_detect_platform index 791a7722b..c7ddb7cce 100755 --- a/vendor/rocksdb-5.9.2/build_tools/build_detect_platform +++ b/vendor/rocksdb-5.9.2/build_tools/build_detect_platform @@ -212,17 +212,15 @@ EOF # Test whether Snappy library is installed # http://code.google.com/p/snappy/ -# $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null < -# int main() {} -#EOF -# if [ "$?" = 0 ]; then + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF + if [ "$?" = 0 ]; then COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY" - PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -L$PWD/third-party/snappy-1.1.7/build -lsnappy" - PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS -I$PWD/third-party/snappy-1.1.7 -I$PWD/third-party/snappy-1.1.7/build" - PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -I$PWD/third-party/snappy-1.1.7 -I$PWD/third-party/snappy-1.1.7/build" - JAVA_LDFLAGS="$JAVA_LDFLAGS -L$PWD/third-party/snappy-1.1.7/build -lsnappy " -# fi + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy" + fi # Test whether gflags library is installed # http://gflags.github.io/gflags/ @@ -260,38 +258,38 @@ EOF fi # Test whether bzip library is installed -# $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < -# int main() {} -#EOF -# if [ "$?" = 0 ]; then -# COMMON_FLAGS="$COMMON_FLAGS -DBZIP2" -# PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2" -# JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2" -# fi + $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DBZIP2" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2" + fi # Test whether lz4 library is installed -# $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < -# #include -# int main() {} -#EOF -# if [ "$?" = 0 ]; then -# COMMON_FLAGS="$COMMON_FLAGS -DLZ4" -# PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4" -# JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4" -# fi + $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + #include + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DLZ4" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4" + JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4" + fi # Test whether zstd library is installed -# $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < -# int main() {} -#EOF -# if [ "$?" = 0 ]; then -# COMMON_FLAGS="$COMMON_FLAGS -DZSTD" -# PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd" -# JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd" -# fi + $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DZSTD" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd" + JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd" + fi # Test whether numa is available $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null <|>sUCS;hTDa$ɲiVJltZS/2Xɩpx H X0ZvIkbMMiQ XTA PU݀k+iD"Fnm\φ0E,NRU eY!+iIΗ$IMR w0oKe|VB;Ιhto'^nmTV|T J*gebP ј>+tVcLvD"Q8] ~oLʢmKM>mmm%.3cVO_Trwb O\r#/:O$d̻Y[ m% Qi1KJ;9$I^ڧ hRoɒ%YֹiVUFS^)@6e|u_Kb|M9* zqQ{iys1>u%A:MՄO# {N̐'^18\]}>w?1@@; -2:r^eY/bً,즼\_Z'n/\.J6z0Ζ]ʋ {Te/㦼eS -Ee>ituA]v4t>hP.HZbƷS^̷Ñ|bmX֒ݖNO7^(#||~PAB;*jYu.q7UvI>?t -yW:1x& ~u}n>t ʨGjc|sp>_?ݤSd~pɇo:|Y|>e}*Ha%գB>ߣ5]_1OFdY.d5~HZ°u?#ԩeYQ||ROҒYWu>ܤ:(#||~PABS,ˊD"E+TC`P>P97(ӴNs>n;ԏjE >AIU$MӠK:';U:e];̗$f+'ӎ|ʇ>~%iuV#dB|T˲4M_pXQ|'8wKX,ZiTPV@)0 \Oω]eQ hq(|HK \ No newline at end of file +5L} eT,}SYUӹpji?⢭M9* zqQ{iy +%##Z6sfiz6HWKYujJ5Wk0gBZ4wWMMNlxvumWZpM6aCn8OjGwlxTFkNV+Pi2UGHaH2IEyaVirkeCIqjN+jGiOSSEWvEXezhbNBWU4tGqLcRFvPXsIiwMegdaaOdMXmcfi5yKltHtixRtAreq0vJ4ifUqMJg0DfW2Mhfg## +startxref +%%EOF +%%EOF \ No newline at end of file diff --git a/vendor/rocksdb-5.9.2/third-party/snappy-1.1.7/testdata/plrabn12.txt b/vendor/snappy-1.1.7/testdata/plrabn12.txt similarity index 100% rename from vendor/rocksdb-5.9.2/third-party/snappy-1.1.7/testdata/plrabn12.txt rename to vendor/snappy-1.1.7/testdata/plrabn12.txt diff --git a/vendor/rocksdb-5.9.2/third-party/snappy-1.1.7/testdata/urls.10K b/vendor/snappy-1.1.7/testdata/urls.10K similarity index 100% rename from vendor/rocksdb-5.9.2/third-party/snappy-1.1.7/testdata/urls.10K rename to vendor/snappy-1.1.7/testdata/urls.10K