Compare commits
3 Commits
ec-worker-
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7f9e897e18 | ||
|
|
84e7327e7d | ||
|
|
5db3c389bc |
2
.github/pull_request_template.md
vendored
2
.github/pull_request_template.md
vendored
@ -12,7 +12,7 @@
|
||||
# Checks
|
||||
- [ ] I have added unit tests if possible.
|
||||
- [ ] I will add related wiki document changes and link to this PR after merging.
|
||||
- [ ] All AI code review comments have been addressed. No more comments to fix if reviewed again. Reviewer may request additional gemini and copilot reviews.
|
||||
- [ ] All AI code review comments have been addressed. No more comments to fix if reviewed again.
|
||||
|
||||
# Checks for AI generated PRs
|
||||
- [ ] I have reviewed every line of code.
|
||||
|
||||
9
.github/workflows/binaries_dev.yml
vendored
9
.github/workflows/binaries_dev.yml
vendored
@ -4,10 +4,6 @@ on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: binaries-dev-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@ -22,7 +18,6 @@ jobs:
|
||||
|
||||
- name: Delete old release assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
@ -43,7 +38,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
@ -92,7 +87,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
2
.github/workflows/binaries_release0.yml
vendored
2
.github/workflows/binaries_release0.yml
vendored
@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
2
.github/workflows/binaries_release1.yml
vendored
2
.github/workflows/binaries_release1.yml
vendored
@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
2
.github/workflows/binaries_release2.yml
vendored
2
.github/workflows/binaries_release2.yml
vendored
@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
2
.github/workflows/binaries_release3.yml
vendored
2
.github/workflows/binaries_release3.yml
vendored
@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
2
.github/workflows/binaries_release4.yml
vendored
2
.github/workflows/binaries_release4.yml
vendored
@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
2
.github/workflows/binaries_release5.yml
vendored
2
.github/workflows/binaries_release5.yml
vendored
@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
2
.github/workflows/codeql.yml
vendored
2
.github/workflows/codeql.yml
vendored
@ -18,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
126
.github/workflows/container_dev.yml
vendored
126
.github/workflows/container_dev.yml
vendored
@ -3,102 +3,24 @@ name: "docker: build dev containers"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-normal-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@ -109,40 +31,40 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build
|
||||
uses: docker/build-push-action@v7
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@ -126,14 +126,14 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@ -150,7 +150,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
361
.github/workflows/container_latest.yml
vendored
361
.github/workflows/container_latest.yml
vendored
@ -1,149 +1,25 @@
|
||||
name: "docker: build latest container"
|
||||
|
||||
# Manual fallback only. On tag push, container_release_unified.yml already
|
||||
# re-tags the released versioned image as `latest` / `latest_large_disk`,
|
||||
# so a full rebuild here is unnecessary. Run this manually if you need to
|
||||
# rebuild `latest` from an arbitrary ref.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_ref:
|
||||
description: 'Git ref to build (branch, tag, or commit SHA)'
|
||||
required: true
|
||||
default: 'master'
|
||||
image_tag:
|
||||
description: 'Docker tag to publish (without variant suffix)'
|
||||
required: true
|
||||
default: 'latest'
|
||||
variant:
|
||||
description: 'Variant to build manually'
|
||||
required: true
|
||||
type: choice
|
||||
default: all
|
||||
options:
|
||||
- all
|
||||
- standard
|
||||
- large_disk
|
||||
publish:
|
||||
description: 'Publish images and manifests'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
variants: ${{ steps.set-variants.outputs.variants }}
|
||||
publish: ${{ steps.set-publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Select variants for this run
|
||||
id: set-variants
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.variant }}" != "all" ]; then
|
||||
variants="[\"${{ github.event.inputs.variant }}\"]"
|
||||
else
|
||||
variants='["standard","large_disk"]'
|
||||
fi
|
||||
echo "variants=$variants" >> "$GITHUB_OUTPUT"
|
||||
- name: Select publish mode
|
||||
id: set-publish
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup, build-rust-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [amd64, arm64, arm, 386]
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
variant: [standard, large_disk]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
@ -167,47 +43,26 @@ jobs:
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
echo "build_args=" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=normal" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
tags: type=raw,value=latest,suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
- name: Set up QEMU
|
||||
if: matrix.platform != 'amd64'
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
@ -215,40 +70,40 @@ jobs:
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ needs.setup.outputs.publish == 'true' }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
# Push to GHCR only during build to avoid Docker Hub rate limits
|
||||
tags: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
|
||||
tags: ghcr.io/chrislusf/seaweedfs:latest${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.platform }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}-${{ matrix.platform }}
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
|
||||
BRANCH=${{ github.sha }}
|
||||
${{ steps.config.outputs.build_args }}
|
||||
- name: Clean up build artifacts
|
||||
if: always()
|
||||
@ -258,159 +113,16 @@ jobs:
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, build-rust-binaries]
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Checkout for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
- name: Place Rust binaries in Docker context for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
rust_variant="normal"
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
rust_variant="large-disk"
|
||||
fi
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Build local scan image tarball
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/amd64
|
||||
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
|
||||
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
|
||||
- name: Trivy report (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan amd64 only - OS packages are identical across architectures
|
||||
# since they all use the same alpine base, so a single-arch scan
|
||||
# provides sufficient coverage without multiplying CI time.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Trivy report (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
- name: Trivy gate (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Gate only on fixable high/critical vulnerabilities. Non-fixable
|
||||
# findings are still visible in the SARIF upload above.
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
- name: Trivy gate (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, trivy-scan]
|
||||
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
variant: [standard, large_disk]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Configure variant
|
||||
id: config
|
||||
@ -423,19 +135,19 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
tags: type=raw,value=latest,suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@ -450,15 +162,14 @@ jobs:
|
||||
- name: Create and push manifest
|
||||
run: |
|
||||
SUFFIX="${{ steps.config.outputs.tag_suffix }}"
|
||||
BASE_TAG="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}"
|
||||
|
||||
# Create manifest on GHCR first (no rate limits)
|
||||
echo "Creating GHCR manifest (no rate limits)..."
|
||||
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
|
||||
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-386
|
||||
|
||||
# Copy the complete multi-arch image from GHCR to Docker Hub
|
||||
# This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
|
||||
@ -494,16 +205,16 @@ jobs:
|
||||
# Use crane or skopeo to copy, fallback to docker if not available
|
||||
if command -v crane &> /dev/null; then
|
||||
echo "Using crane to copy..."
|
||||
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
|
||||
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} chrislusf/seaweedfs:latest${SUFFIX}
|
||||
elif command -v skopeo &> /dev/null; then
|
||||
echo "Using skopeo to copy..."
|
||||
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
|
||||
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} docker://chrislusf/seaweedfs:latest${SUFFIX}
|
||||
else
|
||||
echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
|
||||
# Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
|
||||
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
|
||||
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:latest${SUFFIX} \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-386
|
||||
fi
|
||||
|
||||
152
.github/workflows/container_release1.yml
vendored
Normal file
152
.github/workflows/container_release1.yml
vendored
Normal file
@ -0,0 +1,152 @@
|
||||
name: "docker: build release containers for normal volume"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [amd64, arm64, arm, 386]
|
||||
include:
|
||||
- platform: amd64
|
||||
qemu: false
|
||||
- platform: arm64
|
||||
qemu: true
|
||||
- platform: arm
|
||||
qemu: true
|
||||
- platform: 386
|
||||
qemu: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
# Remove pre-installed tools
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# Clean package managers
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
# Clean Docker aggressively
|
||||
sudo docker system prune -af --volumes
|
||||
# Clean Go cache if it exists
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag
|
||||
flavor: latest=false
|
||||
- name: Set up QEMU
|
||||
if: matrix.qemu
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Build ${{ matrix.platform }}
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
- name: Clean up build artifacts
|
||||
if: always()
|
||||
run: |
|
||||
# Clean up Docker build cache and temporary files
|
||||
sudo docker system prune -f
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag
|
||||
flavor: latest=false
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Create and push manifest
|
||||
run: |
|
||||
# Function to retry command with exponential backoff
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
# Create manifest with retry
|
||||
retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
|
||||
${{ steps.docker_meta.outputs.tags }}-amd64 \
|
||||
${{ steps.docker_meta.outputs.tags }}-arm64 \
|
||||
${{ steps.docker_meta.outputs.tags }}-arm \
|
||||
${{ steps.docker_meta.outputs.tags }}-386
|
||||
153
.github/workflows/container_release2.yml
vendored
Normal file
153
.github/workflows/container_release2.yml
vendored
Normal file
@ -0,0 +1,153 @@
|
||||
name: "docker: build release containers for large volume"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [amd64, arm64, arm, 386]
|
||||
include:
|
||||
- platform: amd64
|
||||
qemu: false
|
||||
- platform: arm64
|
||||
qemu: true
|
||||
- platform: arm
|
||||
qemu: true
|
||||
- platform: 386
|
||||
qemu: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
# Remove pre-installed tools
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# Clean package managers
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
# Clean Docker aggressively
|
||||
sudo docker system prune -af --volumes
|
||||
# Clean Go cache if it exists
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag,suffix=_large_disk
|
||||
flavor: latest=false
|
||||
- name: Set up QEMU
|
||||
if: matrix.qemu
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Build ${{ matrix.platform }}
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
build-args: |
|
||||
TAGS=5BytesOffset
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.platform }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
- name: Clean up build artifacts
|
||||
if: always()
|
||||
run: |
|
||||
# Clean up Docker build cache and temporary files
|
||||
sudo docker system prune -f
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag,suffix=_large_disk
|
||||
flavor: latest=false
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Create and push manifest
|
||||
run: |
|
||||
# Function to retry command with exponential backoff
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
# Create manifest with retry
|
||||
retry_with_backoff docker buildx imagetools create -t ${{ steps.docker_meta.outputs.tags }} \
|
||||
${{ steps.docker_meta.outputs.tags }}-amd64 \
|
||||
${{ steps.docker_meta.outputs.tags }}-arm64 \
|
||||
${{ steps.docker_meta.outputs.tags }}-arm \
|
||||
${{ steps.docker_meta.outputs.tags }}-386
|
||||
73
.github/workflows/container_release3.yml
vendored
Normal file
73
.github/workflows/container_release3.yml
vendored
Normal file
@ -0,0 +1,73 @@
|
||||
name: "docker: build release containers for rocksdb"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_rocksdb:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Free Disk Space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
|
||||
sudo docker system prune -af
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag,suffix=_large_disk_rocksdb
|
||||
flavor: latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.rocksdb_large
|
||||
build-args: |
|
||||
BRANCH=${{ github.sha }}
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
72
.github/workflows/container_release4.yml
vendored
Normal file
72
.github/workflows/container_release4.yml
vendored
Normal file
@ -0,0 +1,72 @@
|
||||
name: "docker: build release containers for all tags"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-default-release-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Free Disk Space
|
||||
run: |
|
||||
echo "Before cleanup:"
|
||||
df -h
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /usr/local/lib/android
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
||||
sudo docker system prune -af
|
||||
echo "After cleanup:"
|
||||
df -h
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_full
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
build-args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
62
.github/workflows/container_release5.yml
vendored
Normal file
62
.github/workflows/container_release5.yml
vendored
Normal file
@ -0,0 +1,62 @@
|
||||
name: "docker: build release containers for all tags and large volume"
|
||||
|
||||
# DISABLED: Merged into container_release_unified.yml
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_run:
|
||||
description: 'This workflow is disabled. Use container_release_unified.yml instead'
|
||||
required: true
|
||||
default: 'disabled'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-default-release-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Free Disk Space
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
|
||||
sudo docker system prune -af
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: chrislusf/seaweedfs
|
||||
tags: type=ref,event=tag,suffix=_large_disk_full
|
||||
flavor: latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
build-args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@ -4,19 +4,11 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag to publish (e.g. 3.93)'
|
||||
required: true
|
||||
default: ''
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_foundationdb:
|
||||
@ -29,12 +21,12 @@ jobs:
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
|
||||
type=ref,event=tag,suffix=_large_disk_foundationdb
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
@ -43,14 +35,14 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@ -65,7 +57,7 @@ jobs:
|
||||
fi
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
319
.github/workflows/container_release_unified.yml
vendored
319
.github/workflows/container_release_unified.yml
vendored
@ -4,35 +4,10 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
variant:
|
||||
description: 'Variant to build manually'
|
||||
required: true
|
||||
type: choice
|
||||
default: all
|
||||
options:
|
||||
- all
|
||||
- normal
|
||||
- large_disk
|
||||
- full
|
||||
- large_disk_full
|
||||
- rocksdb
|
||||
release_tag:
|
||||
description: 'Release tag to publish (e.g. 3.93)'
|
||||
required: true
|
||||
default: ''
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag to use when variant=rocksdb'
|
||||
required: false
|
||||
default: 'v10.10.1'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
|
||||
# Limit concurrent builds to avoid rate limits
|
||||
concurrency:
|
||||
@ -40,82 +15,7 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
# Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
|
||||
# emulated cargo build into ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
# ── Build Docker containers ─────────────────────────────────────────
|
||||
build:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# Build sequentially to avoid rate limits
|
||||
@ -128,68 +28,40 @@ jobs:
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: ""
|
||||
tag_suffix: ""
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk - multi-arch
|
||||
|
||||
# Large disk - multi-arch
|
||||
- variant: large_disk
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
rust_variant: large-disk
|
||||
|
||||
# Full tags - multi-arch
|
||||
|
||||
# Full tags - amd64 only
|
||||
- variant: full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
|
||||
# Large disk + full tags - amd64 only
|
||||
- variant: large_disk_full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
rust_variant: large-disk
|
||||
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- variant: rocksdb
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.rocksdb_large
|
||||
build_args: ""
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
rust_variant: large-disk
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
@ -203,14 +75,13 @@ jobs:
|
||||
df -h
|
||||
|
||||
- name: Docker meta
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
tags: type=ref,event=tag,suffix=${{ matrix.tag_suffix }}
|
||||
flavor: latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
@ -218,11 +89,10 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
|
||||
uses: docker/setup-qemu-action@v4
|
||||
if: contains(matrix.platforms, 'arm')
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Create BuildKit config
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
@ -230,29 +100,27 @@ jobs:
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build and push ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
@ -262,7 +130,7 @@ jobs:
|
||||
platforms: ${{ matrix.platforms }}
|
||||
# Push to GHCR to avoid Docker Hub rate limits on pulls
|
||||
tags: |
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
|
||||
@ -270,10 +138,9 @@ jobs:
|
||||
${{ matrix.build_args }}
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
|
||||
|
||||
- name: Clean up build artifacts
|
||||
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
|
||||
if: always()
|
||||
run: |
|
||||
sudo docker system prune -f
|
||||
sudo rm -rf /tmp/go-build*
|
||||
@ -299,22 +166,19 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Install crane
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
cd $(mktemp -d)
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
@ -322,7 +186,6 @@ jobs:
|
||||
crane version
|
||||
|
||||
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
# Function to retry with exponential backoff
|
||||
retry_with_backoff() {
|
||||
@ -355,136 +218,14 @@ jobs:
|
||||
# This is much more efficient than pulling/pushing individual arch images
|
||||
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
|
||||
retry_with_backoff crane copy \
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }} \
|
||||
chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
|
||||
|
||||
echo "Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
|
||||
# Security for visibility, but never blocks the release. Releases (including
|
||||
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
|
||||
# we sometimes need to publish through known findings (e.g. unfixed upstream
|
||||
# CVE, base-image lag).
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name == 'push'
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
variant: normal
|
||||
- source_suffix: _large_disk
|
||||
variant: large_disk
|
||||
steps:
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Trivy report (${{ matrix.variant }})
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan the multi-arch tag on GHCR (already pushed by the build job).
|
||||
# Trivy scans the runner's native platform; OS packages are identical
|
||||
# across architectures since they all share the same alpine base.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
|
||||
# Point `latest` (and `latest_large_disk`) at the just-released versioned
|
||||
# image. crane tag adds an extra tag to an existing manifest — no rebuild,
|
||||
# no QEMU, no separate workflow. Replaces the old container_latest.yml
|
||||
# rebuild that often failed or lagged behind the release. Independent of
|
||||
# trivy-scan: vuln findings are reported but do not block `latest`.
|
||||
tag-latest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [copy-to-dockerhub]
|
||||
if: github.event_name == 'push'
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
latest_tag: latest
|
||||
- source_suffix: _large_disk
|
||||
latest_tag: latest_large_disk
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Install crane
|
||||
run: |
|
||||
cd $(mktemp -d)
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }}
|
||||
run: |
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
|
||||
DST_TAG="${{ matrix.latest_tag }}"
|
||||
|
||||
echo "Tagging ghcr.io/chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "ghcr.io/chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [copy-to-dockerhub]
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
@ -498,3 +239,7 @@ jobs:
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
helm_version: "3.18.4"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
14
.github/workflows/container_rocksdb_version.yml
vendored
14
.github/workflows/container_rocksdb_version.yml
vendored
@ -4,9 +4,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
|
||||
description: 'RocksDB git tag or branch to build (e.g. v10.5.1)'
|
||||
required: true
|
||||
default: 'v10.10.1'
|
||||
default: 'v10.5.1'
|
||||
seaweedfs_ref:
|
||||
description: 'SeaweedFS git tag, branch, or commit to build'
|
||||
required: true
|
||||
@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Prepare Docker tag
|
||||
id: tag
|
||||
@ -82,19 +82,19 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
|
||||
2
.github/workflows/deploy_telemetry.yml
vendored
2
.github/workflows/deploy_telemetry.yml
vendored
@ -26,7 +26,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build Telemetry Server
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.deploy
|
||||
|
||||
4
.github/workflows/depsreview.yml
vendored
4
.github/workflows/depsreview.yml
vendored
@ -9,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
|
||||
uses: actions/dependency-review-action@3c4e3dcb1aa7874d2c16be7d79418e9b7efd6261
|
||||
|
||||
31
.github/workflows/e2e.yml
vendored
31
.github/workflows/e2e.yml
vendored
@ -23,16 +23,17 @@ jobs:
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
@ -57,12 +58,12 @@ jobs:
|
||||
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
|
||||
|
||||
- name: Start SeaweedFS
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
# Enable Docker buildkit for better caching
|
||||
export DOCKER_BUILDKIT=1
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
|
||||
|
||||
# Build with retry logic
|
||||
for i in {1..3}; do
|
||||
echo "Build attempt $i/3"
|
||||
@ -77,18 +78,10 @@ jobs:
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# Start services with wait
|
||||
docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Rotate buildx cache
|
||||
if: always()
|
||||
run: |
|
||||
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
|
||||
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
|
||||
rm -rf /tmp/.buildx-cache
|
||||
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
|
||||
|
||||
- name: Run FIO 4k
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
@ -142,7 +135,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: output-logs
|
||||
path: docker/output.log
|
||||
|
||||
4
.github/workflows/ec-integration-tests.yml
vendored
4
.github/workflows/ec-integration-tests.yml
vendored
@ -18,7 +18,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
@ -52,7 +52,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: ec-integration-test-logs
|
||||
path: |
|
||||
|
||||
49
.github/workflows/ec-integration.yml
vendored
Normal file
49
.github/workflows/ec-integration.yml
vendored
Normal file
@ -0,0 +1,49 @@
|
||||
name: EC Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/admin/**'
|
||||
- 'weed/worker/**'
|
||||
- 'test/erasure_coding/admin_dockertest/**'
|
||||
- '.github/workflows/ec-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/admin/**'
|
||||
- 'weed/worker/**'
|
||||
- 'test/erasure_coding/admin_dockertest/**'
|
||||
- '.github/workflows/ec-integration.yml'
|
||||
|
||||
jobs:
|
||||
ec-integration-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o ../weed_bin
|
||||
|
||||
- name: Run EC integration tests
|
||||
run: |
|
||||
cd test/erasure_coding/admin_dockertest
|
||||
go test -v -timeout 15m ec_integration_test.go
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: ec-test-logs
|
||||
path: test/erasure_coding/admin_dockertest/tmp/logs/
|
||||
retention-days: 7
|
||||
63
.github/workflows/fuse-dlm-integration.yml
vendored
63
.github/workflows/fuse-dlm-integration.yml
vendored
@ -1,63 +0,0 @@
|
||||
name: "FUSE DLM Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run DLM integration tests
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-dlm-test-logs
|
||||
path: /tmp/seaweedfs-fuse-dlm-logs/
|
||||
retention-days: 3
|
||||
215
.github/workflows/fuse-integration.yml
vendored
215
.github/workflows/fuse-integration.yml
vendored
@ -21,61 +21,214 @@ concurrency:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
fuse-integration:
|
||||
name: FUSE Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 50
|
||||
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Install FUSE and dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
# fuse3 is pre-installed on ubuntu-22.04 runners and conflicts
|
||||
# with the legacy fuse package, so only install the dev headers.
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
# Allow non-root FUSE mounts with allow_other
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
# Verify FUSE installation
|
||||
fusermount3 --version || fusermount --version || true
|
||||
ls -la /dev/fuse
|
||||
|
||||
fusermount --version || true
|
||||
ls -la /dev/fuse || true
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -o weed .
|
||||
go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
chmod +x weed
|
||||
# Verify binary
|
||||
./weed version
|
||||
# Make weed binary available in PATH for the test framework
|
||||
sudo cp weed /usr/local/bin/weed
|
||||
|
||||
- name: Install test dependencies
|
||||
|
||||
- name: Prepare FUSE Integration Tests
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
go mod download
|
||||
|
||||
# Create isolated test directory to avoid Go module conflicts
|
||||
mkdir -p /tmp/seaweedfs-fuse-tests
|
||||
|
||||
# Copy only the working test files to avoid Go module conflicts
|
||||
# These are the files we've verified work without package name issues
|
||||
cp test/fuse_integration/simple_test.go /tmp/seaweedfs-fuse-tests/ 2>/dev/null || echo "⚠️ simple_test.go not found"
|
||||
cp test/fuse_integration/working_demo_test.go /tmp/seaweedfs-fuse-tests/ 2>/dev/null || echo "⚠️ working_demo_test.go not found"
|
||||
|
||||
# Note: Other test files (framework.go, basic_operations_test.go, etc.)
|
||||
# have Go module conflicts and are skipped until resolved
|
||||
|
||||
echo "📁 Working test files copied:"
|
||||
ls -la /tmp/seaweedfs-fuse-tests/*.go 2>/dev/null || echo "ℹ️ No test files found"
|
||||
|
||||
# Initialize Go module in isolated directory
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
go mod init seaweedfs-fuse-tests
|
||||
go mod tidy
|
||||
|
||||
# Verify setup
|
||||
echo "✅ FUSE integration test environment prepared"
|
||||
ls -la /tmp/seaweedfs-fuse-tests/
|
||||
|
||||
echo ""
|
||||
echo "ℹ️ Current Status: Running working subset of FUSE tests"
|
||||
echo " • simple_test.go: Package structure verification"
|
||||
echo " • working_demo_test.go: Framework capability demonstration"
|
||||
echo " • Full framework: Available in test/fuse_integration/ (module conflicts pending resolution)"
|
||||
|
||||
- name: Run FUSE Integration Tests
|
||||
run: |
|
||||
set -o pipefail
|
||||
cd test/fuse_integration
|
||||
echo "Running full FUSE integration test suite..."
|
||||
go test -v -count=1 -timeout=45m ./... 2>&1 | tee /tmp/fuse-test-output.log
|
||||
|
||||
- name: Upload Test Logs
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
|
||||
echo "🧪 Running FUSE integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Run available working test files
|
||||
TESTS_RUN=0
|
||||
|
||||
if [ -f "simple_test.go" ]; then
|
||||
echo "📋 Running simple_test.go..."
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} simple_test.go
|
||||
TESTS_RUN=$((TESTS_RUN + 1))
|
||||
fi
|
||||
|
||||
if [ -f "working_demo_test.go" ]; then
|
||||
echo "📋 Running working_demo_test.go..."
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} working_demo_test.go
|
||||
TESTS_RUN=$((TESTS_RUN + 1))
|
||||
fi
|
||||
|
||||
# Run combined test if multiple files exist
|
||||
if [ -f "simple_test.go" ] && [ -f "working_demo_test.go" ]; then
|
||||
echo "📋 Running combined tests..."
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} simple_test.go working_demo_test.go
|
||||
fi
|
||||
|
||||
if [ $TESTS_RUN -eq 0 ]; then
|
||||
echo "⚠️ No working test files found, running module verification only"
|
||||
go version
|
||||
go mod verify
|
||||
else
|
||||
echo "✅ Successfully ran $TESTS_RUN test file(s)"
|
||||
fi
|
||||
|
||||
echo "============================================"
|
||||
echo "✅ FUSE integration tests completed"
|
||||
|
||||
- name: Run Extended Framework Validation
|
||||
run: |
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
|
||||
echo "🔍 Running extended framework validation..."
|
||||
echo "============================================"
|
||||
|
||||
# Test individual components (only run tests that exist)
|
||||
if [ -f "simple_test.go" ]; then
|
||||
echo "Testing simple verification..."
|
||||
go test -v simple_test.go
|
||||
fi
|
||||
|
||||
if [ -f "working_demo_test.go" ]; then
|
||||
echo "Testing framework demo..."
|
||||
go test -v working_demo_test.go
|
||||
fi
|
||||
|
||||
# Test combined execution if both files exist
|
||||
if [ -f "simple_test.go" ] && [ -f "working_demo_test.go" ]; then
|
||||
echo "Testing combined execution..."
|
||||
go test -v simple_test.go working_demo_test.go
|
||||
elif [ -f "simple_test.go" ] || [ -f "working_demo_test.go" ]; then
|
||||
echo "✅ Individual tests already validated above"
|
||||
else
|
||||
echo "⚠️ No working test files found for combined testing"
|
||||
fi
|
||||
|
||||
echo "============================================"
|
||||
echo "✅ Extended validation completed"
|
||||
|
||||
- name: Generate Test Coverage Report
|
||||
run: |
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
|
||||
echo "📊 Generating test coverage report..."
|
||||
go test -v -coverprofile=coverage.out .
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
|
||||
echo "Coverage report generated: coverage.html"
|
||||
|
||||
- name: Verify SeaweedFS Binary Integration
|
||||
run: |
|
||||
# Test that SeaweedFS binary is accessible from test environment
|
||||
WEED_BINARY=$(pwd)/weed/weed
|
||||
|
||||
if [ -f "$WEED_BINARY" ]; then
|
||||
echo "✅ SeaweedFS binary found at: $WEED_BINARY"
|
||||
$WEED_BINARY version
|
||||
echo "Binary is ready for full integration testing"
|
||||
else
|
||||
echo "❌ SeaweedFS binary not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload Test Artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: fuse-integration-test-results
|
||||
path: |
|
||||
/tmp/fuse-test-output.log
|
||||
/tmp/seaweedfs-fuse-logs/
|
||||
/tmp/seaweedfs-fuse-tests/coverage.out
|
||||
/tmp/seaweedfs-fuse-tests/coverage.html
|
||||
/tmp/seaweedfs-fuse-tests/*.log
|
||||
retention-days: 7
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## 🚀 FUSE Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Framework Status" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Framework Design**: Complete and validated" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Working Tests**: Core framework demonstration functional" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ⚠️ **Full Framework**: Available but requires Go module resolution" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **CI/CD Integration**: Automated testing pipeline established" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Capabilities" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 📁 **File Operations**: Create, read, write, delete, permissions" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 📂 **Directory Operations**: Create, list, delete, nested structures" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 📊 **Large Files**: Multi-megabyte file handling" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 🔄 **Concurrent Operations**: Multi-threaded stress testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ⚠️ **Error Scenarios**: Comprehensive error handling validation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Comparison with Current Tests" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Aspect | Current (FIO) | This Framework |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|--------|---------------|----------------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Scope** | Performance only | Functional + Performance |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Operations** | Read/Write only | All FUSE operations |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Concurrency** | Single-threaded | Multi-threaded stress tests |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Automation** | Manual setup | Fully automated |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Validation** | Speed metrics | Correctness + Performance |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Current Working Tests" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Framework Structure**: Package and module verification" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Configuration Management**: Test config validation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **File Operations Demo**: Basic file create/read/write simulation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Large File Handling**: 1MB+ file processing demonstration" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Concurrency Simulation**: Multi-file operation testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
|
||||
echo "1. **Module Resolution**: Fix Go package conflicts for full framework" >> $GITHUB_STEP_SUMMARY
|
||||
echo "2. **SeaweedFS Integration**: Connect with real cluster for end-to-end testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "3. **Performance Benchmarks**: Add performance regression testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "📈 **Total Framework Size**: ~1,500 lines of comprehensive testing infrastructure" >> $GITHUB_STEP_SUMMARY
|
||||
69
.github/workflows/fuse-p2p-integration.yml
vendored
69
.github/workflows/fuse-p2p-integration.yml
vendored
@ -1,69 +0,0 @@
|
||||
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/workflows/fuse-p2p-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run P2P integration tests
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-p2p-test-logs
|
||||
path: /tmp/seaweedfs-fuse-p2p-logs/
|
||||
retention-days: 3
|
||||
39
.github/workflows/go.yml
vendored
39
.github/workflows/go.yml
vendored
@ -15,19 +15,24 @@ permissions:
|
||||
|
||||
jobs:
|
||||
|
||||
vet:
|
||||
name: Go Vet
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd weed; go get -v -t -d ./...
|
||||
|
||||
- name: Go Vet (excluding protobuf lock copying)
|
||||
run: |
|
||||
cd weed
|
||||
@ -37,28 +42,8 @@ jobs:
|
||||
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
|
||||
if grep -q "vet:" vet-output.txt; then exit 1; fi
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: Build
|
||||
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: Test
|
||||
run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
|
||||
|
||||
346
.github/workflows/helm_ci.yml
vendored
346
.github/workflows/helm_ci.yml
vendored
@ -16,12 +16,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v5
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: v3.18.4
|
||||
|
||||
@ -64,144 +64,15 @@ jobs:
|
||||
echo "✓ All-in-one deployment renders correctly"
|
||||
|
||||
echo "=== Testing with security enabled ==="
|
||||
helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true > /tmp/security.yaml
|
||||
helm template test $CHART_DIR --set global.enableSecurity=true > /tmp/security.yaml
|
||||
grep -q "security-config" /tmp/security.yaml
|
||||
echo "✓ Security configuration renders correctly"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing IAM gRPC opt-in path ==="
|
||||
# Regression test: the filer registers the IAM gRPC service the
|
||||
# Admin UI Users tab calls only when jwt.filer_signing.key is in
|
||||
# security.toml. Operators must be able to enable that without
|
||||
# the cert-manager mTLS bundle.
|
||||
# Install PyYAML explicitly: this block runs before the later
|
||||
# security+S3 block that does the same install, and we don't
|
||||
# want to rely on the runner image shipping it.
|
||||
pip install pyyaml -q
|
||||
python3 - "$CHART_DIR" <<'PYEOF'
|
||||
import subprocess, sys, yaml
|
||||
chart = sys.argv[1]
|
||||
|
||||
def render(values):
|
||||
args = ["helm", "template", "test", chart]
|
||||
for k, v in values.items():
|
||||
args += ["--set", f"{k}={v}"]
|
||||
return subprocess.check_output(args, text=True)
|
||||
|
||||
def docs(manifest):
|
||||
return [d for d in yaml.safe_load_all(manifest) if d]
|
||||
|
||||
def configmap(manifest, name):
|
||||
for d in docs(manifest):
|
||||
if d.get("kind") == "ConfigMap" and d["metadata"]["name"] == name:
|
||||
return d
|
||||
return None
|
||||
|
||||
def workload_mounts(manifest, name):
|
||||
for d in docs(manifest):
|
||||
if d.get("kind") not in ("Deployment", "StatefulSet"):
|
||||
continue
|
||||
if d["metadata"]["name"] != name:
|
||||
continue
|
||||
pod = d["spec"]["template"]["spec"]
|
||||
vols = {v["name"] for v in pod.get("volumes", [])}
|
||||
mounts = set()
|
||||
for c in pod.get("containers", []):
|
||||
for vm in c.get("volumeMounts", []):
|
||||
mounts.add(vm["name"])
|
||||
return vols, mounts
|
||||
return None, None
|
||||
|
||||
failed = []
|
||||
|
||||
# Case 1: defaults. The chart historically rendered nothing
|
||||
# security-related; preserve that so this PR is non-breaking on
|
||||
# existing installs.
|
||||
out = render({})
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append("defaults: security ConfigMap should not render")
|
||||
else:
|
||||
print("✓ defaults: no security-config ConfigMap (unchanged)")
|
||||
|
||||
# Case 2: filerWrite=true alone is the documented opt-in for
|
||||
# the Admin UI Users tab. Configmap must render with
|
||||
# [jwt.filer_signing] and NO [grpc.*] sections (cert paths
|
||||
# only exist with mTLS).
|
||||
out = render({
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerWrite": "true",
|
||||
"admin.enabled": "true",
|
||||
})
|
||||
cm = configmap(out, "test-seaweedfs-security-config")
|
||||
if cm is None:
|
||||
failed.append("filerWrite=true: security ConfigMap missing")
|
||||
else:
|
||||
toml = cm["data"]["security.toml"]
|
||||
if "[jwt.filer_signing]" not in toml:
|
||||
failed.append("filerWrite=true: security.toml missing [jwt.filer_signing]")
|
||||
if "[grpc" in toml:
|
||||
failed.append("filerWrite=true: security.toml unexpectedly has [grpc.*] (would need cert mounts)")
|
||||
if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
|
||||
print("✓ filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
|
||||
|
||||
# Case 3: filer + admin pods must MOUNT the security ConfigMap
|
||||
# under filerWrite=true so the JWT key reaches both processes.
|
||||
# Cert volumes must NOT be present (no mTLS).
|
||||
for wl in ("test-seaweedfs-filer", "test-seaweedfs-admin"):
|
||||
vols, mounts = workload_mounts(out, wl)
|
||||
if vols is None:
|
||||
failed.append(f"filerWrite=true: workload {wl} not found")
|
||||
continue
|
||||
if "security-config" not in vols or "security-config" not in mounts:
|
||||
failed.append(f"filerWrite=true: {wl} does not mount security-config (IAM gRPC would still fail)")
|
||||
else:
|
||||
print(f"✓ filerWrite=true: {wl} mounts security-config")
|
||||
cert_vols = {v for v in vols if v.endswith("-cert")}
|
||||
if cert_vols:
|
||||
failed.append(f"filerWrite=true: {wl} unexpectedly has cert volumes {sorted(cert_vols)}")
|
||||
|
||||
# Case 4: enableSecurity=true must still render the full toml
|
||||
# with both [jwt.signing] and [grpc.*]. Guards against the
|
||||
# decoupling change accidentally regressing the mTLS path.
|
||||
out = render({"global.seaweedfs.enableSecurity": "true"})
|
||||
cm = configmap(out, "test-seaweedfs-security-config")
|
||||
if cm is None:
|
||||
failed.append("enableSecurity=true: security ConfigMap missing")
|
||||
else:
|
||||
toml = cm["data"]["security.toml"]
|
||||
missing = [s for s in ("[jwt.signing]", "[grpc.master]") if s not in toml]
|
||||
if missing:
|
||||
failed.append(f"enableSecurity=true: security.toml missing {missing}")
|
||||
else:
|
||||
print("✓ enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")
|
||||
|
||||
# Case 5: helper must tolerate explicit nulls (gemini-code-assist
|
||||
# PR review). securityConfig=null was the parens-pattern crash
|
||||
# the helper review caught.
|
||||
for null_path in ("global.seaweedfs.securityConfig",
|
||||
"global.seaweedfs.securityConfig.jwtSigning"):
|
||||
try:
|
||||
out = render({null_path: "null"})
|
||||
except subprocess.CalledProcessError as e:
|
||||
failed.append(f"{null_path}=null: render failed: {e.output[:200] if e.output else e}")
|
||||
continue
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append(f"{null_path}=null: should not render configmap")
|
||||
else:
|
||||
print(f"✓ {null_path}=null: render tolerates explicit null")
|
||||
|
||||
if failed:
|
||||
print("\nFAIL:", file=sys.stderr)
|
||||
for f in failed:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
echo "✓ IAM gRPC decoupling tests passed"
|
||||
|
||||
|
||||
echo "=== Testing with monitoring enabled ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.monitoring.enabled=true \
|
||||
--set global.seaweedfs.monitoring.gatewayHost=prometheus \
|
||||
--set global.seaweedfs.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml
|
||||
--set global.monitoring.enabled=true \
|
||||
--set global.monitoring.gatewayHost=prometheus \
|
||||
--set global.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml
|
||||
echo "✓ Monitoring configuration renders correctly"
|
||||
|
||||
echo "=== Testing with PVC storage ==="
|
||||
@ -245,211 +116,10 @@ jobs:
|
||||
echo "✓ COSI driver renders correctly"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing long release name: service names match DNS references ==="
|
||||
# Use a release name that, combined with chart name "seaweedfs", exceeds 63 chars.
|
||||
# fullname = "my-very-long-release-name-that-will-cause-truncation-seaweedfs" (65 chars before trunc)
|
||||
LONG_RELEASE="my-very-long-release-name-that-will-cause-truncation"
|
||||
|
||||
# --- Normal mode: master + filer-client services vs helper-produced addresses ---
|
||||
helm template "$LONG_RELEASE" $CHART_DIR \
|
||||
--set s3.enabled=true \
|
||||
--set global.seaweedfs.createBuckets[0].name=test > /tmp/longname.yaml
|
||||
|
||||
# Extract Service names from metadata
|
||||
MASTER_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \
|
||||
| grep -- '-master$')
|
||||
FILER_CLIENT_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \
|
||||
| grep -- '-filer-client$')
|
||||
|
||||
# Extract the hostname from WEED_CLUSTER_SW_MASTER in post-install-bucket-hook
|
||||
MASTER_ADDR=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname.yaml \
|
||||
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/')
|
||||
FILER_ADDR=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname.yaml \
|
||||
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/')
|
||||
|
||||
# Extract the hostname from S3 deployment -filer= argument
|
||||
S3_FILER_HOST=$(grep '\-filer=' /tmp/longname.yaml \
|
||||
| head -1 | sed 's/.*-filer=\([^:]*\).*/\1/')
|
||||
|
||||
# The address helpers produce "<svc>.<namespace>:<port>"; extract just the svc name
|
||||
MASTER_ADDR_SVC=$(echo "$MASTER_ADDR" | cut -d. -f1)
|
||||
FILER_ADDR_SVC=$(echo "$FILER_ADDR" | cut -d. -f1)
|
||||
S3_FILER_SVC=$(echo "$S3_FILER_HOST" | cut -d. -f1)
|
||||
|
||||
echo " master Service.name: $MASTER_SVC"
|
||||
echo " cluster.masterAddress svc: $MASTER_ADDR_SVC"
|
||||
echo " filer-client Service.name: $FILER_CLIENT_SVC"
|
||||
echo " cluster.filerAddress svc: $FILER_ADDR_SVC"
|
||||
echo " S3 -filer= svc: $S3_FILER_SVC"
|
||||
|
||||
[ "$MASTER_SVC" = "$MASTER_ADDR_SVC" ] || { echo "FAIL: master service name mismatch"; exit 1; }
|
||||
[ "$FILER_CLIENT_SVC" = "$FILER_ADDR_SVC" ] || { echo "FAIL: filer-client service name mismatch"; exit 1; }
|
||||
[ "$FILER_CLIENT_SVC" = "$S3_FILER_SVC" ] || { echo "FAIL: S3 -filer= does not match filer-client service"; exit 1; }
|
||||
echo "✓ Normal mode: service names match DNS references with long release name"
|
||||
|
||||
# --- All-in-one mode: all-in-one service vs both helper addresses ---
|
||||
helm template "$LONG_RELEASE" $CHART_DIR \
|
||||
--set allInOne.enabled=true \
|
||||
--set global.seaweedfs.createBuckets[0].name=test > /tmp/longname-aio.yaml
|
||||
|
||||
AIO_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname-aio.yaml \
|
||||
| grep -- '-all-in-one$')
|
||||
AIO_MASTER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname-aio.yaml \
|
||||
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1)
|
||||
AIO_FILER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname-aio.yaml \
|
||||
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1)
|
||||
|
||||
echo " all-in-one Service.name: $AIO_SVC"
|
||||
echo " cluster.masterAddress svc: $AIO_MASTER_ADDR_SVC"
|
||||
echo " cluster.filerAddress svc: $AIO_FILER_ADDR_SVC"
|
||||
|
||||
[ "$AIO_SVC" = "$AIO_MASTER_ADDR_SVC" ] || { echo "FAIL: all-in-one master address mismatch"; exit 1; }
|
||||
[ "$AIO_SVC" = "$AIO_FILER_ADDR_SVC" ] || { echo "FAIL: all-in-one filer address mismatch"; exit 1; }
|
||||
echo "✓ All-in-one mode: service names match DNS references with long release name"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing security+S3: no blank lines in shell command blocks ==="
|
||||
# Render the three manifests that include seaweedfs.s3.tlsArgs:
|
||||
# filer-statefulset, s3-deployment, all-in-one-deployment
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.enableSecurity=true \
|
||||
--set filer.s3.enabled=true \
|
||||
--set s3.enabled=true > /tmp/security-s3.yaml
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.enableSecurity=true \
|
||||
--set allInOne.enabled=true \
|
||||
--set allInOne.s3.enabled=true > /tmp/security-aio.yaml
|
||||
|
||||
pip install pyyaml -q
|
||||
python3 - /tmp/security-s3.yaml /tmp/security-aio.yaml <<'PYEOF'
|
||||
import yaml, sys
|
||||
errors = []
|
||||
for path in sys.argv[1:]:
|
||||
with open(path) as f:
|
||||
docs = list(yaml.safe_load_all(f))
|
||||
for doc in docs:
|
||||
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
|
||||
continue
|
||||
name = doc["metadata"]["name"]
|
||||
for c in doc["spec"]["template"]["spec"].get("containers", []):
|
||||
cmd = c.get("command", [])
|
||||
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
|
||||
script = cmd[2]
|
||||
for i, line in enumerate(script.splitlines(), 1):
|
||||
if line.strip() == "":
|
||||
errors.append(f"{path}: {name}/{c['name']} has blank line at script line {i}")
|
||||
if errors:
|
||||
for e in errors:
|
||||
print(f"FAIL: {e}", file=sys.stderr)
|
||||
print("Rendered with: global.seaweedfs.enableSecurity=true, filer.s3.enabled=true, s3.enabled=true, allInOne.enabled=true", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("✓ No blank lines in security+S3 command blocks")
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "=== Testing security+S3: -cert.file/-key.file gated on httpsPort (issue #9202) ==="
|
||||
# Regression test: when enableSecurity=true but *.httpsPort is 0 (the default),
|
||||
# the chart must NOT emit -cert.file / -key.file to the S3 frontend. Passing
|
||||
# them promotes weed s3's main -port to HTTPS (see weed/command/s3.go), which
|
||||
# makes the HTTP readinessProbe spam "TLS handshake error ... client sent an
|
||||
# HTTP request to an HTTPS server" into the pod log.
|
||||
#
|
||||
# When *.httpsPort > 0, both -port.https and cert/key args MUST be emitted
|
||||
# together so the opt-in HTTPS listener actually has credentials.
|
||||
python3 - "$CHART_DIR" <<'PYEOF'
|
||||
import subprocess, sys, yaml
|
||||
chart = sys.argv[1]
|
||||
|
||||
def render(values):
|
||||
args = ["helm", "template", "test", chart]
|
||||
for k, v in values.items():
|
||||
args += ["--set", f"{k}={v}"]
|
||||
return subprocess.check_output(args, text=True)
|
||||
|
||||
def script_of(manifest, kind_name):
|
||||
for doc in yaml.safe_load_all(manifest):
|
||||
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
|
||||
continue
|
||||
if doc["metadata"]["name"] != kind_name:
|
||||
continue
|
||||
for c in doc["spec"]["template"]["spec"]["containers"]:
|
||||
cmd = c.get("command", [])
|
||||
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
|
||||
return cmd[2]
|
||||
raise AssertionError(f"no container script for {kind_name}")
|
||||
|
||||
cases = [
|
||||
# (values, workload-name, httpsPort-set?, arg-prefix)
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"s3.enabled": "true"},
|
||||
"test-seaweedfs-s3", False, ""),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"s3.enabled": "true",
|
||||
"s3.httpsPort": "8443"},
|
||||
"test-seaweedfs-s3", True, ""),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"filer.s3.enabled": "true"},
|
||||
"test-seaweedfs-filer", False, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"filer.s3.enabled": "true",
|
||||
"filer.s3.httpsPort": "8444"},
|
||||
"test-seaweedfs-filer", True, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"allInOne.enabled": "true",
|
||||
"allInOne.s3.enabled": "true"},
|
||||
"test-seaweedfs-all-in-one", False, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"allInOne.enabled": "true",
|
||||
"allInOne.s3.enabled": "true",
|
||||
"allInOne.s3.httpsPort": "8445"},
|
||||
"test-seaweedfs-all-in-one", True, "s3."),
|
||||
]
|
||||
|
||||
failed = False
|
||||
for values, name, https_on, prefix in cases:
|
||||
script = script_of(render(values), name)
|
||||
cert_flag = f"-{prefix}cert.file="
|
||||
key_flag = f"-{prefix}key.file="
|
||||
https_flag = f"-{prefix}port.https="
|
||||
has_cert = cert_flag in script
|
||||
has_key = key_flag in script
|
||||
has_https = https_flag in script
|
||||
label = f"{name} (httpsPort {'set' if https_on else 'unset'})"
|
||||
if https_on:
|
||||
if not (has_cert and has_key and has_https):
|
||||
print(f"FAIL: {label}: expected {cert_flag}, {key_flag}, {https_flag} all present "
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"✓ {label}: cert/key/https args emitted together")
|
||||
else:
|
||||
if has_cert or has_key or has_https:
|
||||
print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
|
||||
f"main S3 -port would silently become HTTPS and break HTTP probes "
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"✓ {label}: no TLS args emitted, main -port stays HTTP")
|
||||
|
||||
# bash -n: pin down that the rendered script parses. Guards against
|
||||
# a future helper change that leaves a dangling `\` with nothing
|
||||
# after it (every current caller already exits cleanly because
|
||||
# bash treats trailing `\<newline><EOF>` as line-continuation to
|
||||
# an empty line — but keep the contract explicit).
|
||||
parse = subprocess.run(["bash", "-n"], input=script, text=True,
|
||||
capture_output=True)
|
||||
if parse.returncode != 0:
|
||||
print(f"FAIL: {label}: bash -n rejected rendered script: {parse.stderr.strip()}",
|
||||
file=sys.stderr)
|
||||
failed = True
|
||||
|
||||
sys.exit(1 if failed else 0)
|
||||
PYEOF
|
||||
|
||||
echo "✅ All template rendering tests passed!"
|
||||
|
||||
- name: Create kind cluster
|
||||
uses: helm/kind-action@v1.14.0
|
||||
uses: helm/kind-action@v1.13.0
|
||||
|
||||
- name: Run chart-testing (install)
|
||||
run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts
|
||||
|
||||
41
.github/workflows/helm_manual_release.yml
vendored
41
.github/workflows/helm_manual_release.yml
vendored
@ -1,41 +0,0 @@
|
||||
name: "helm: release"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Publish Helm charts to github pages
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
helm_version: "3.18.4"
|
||||
|
||||
- name: Publish Helm charts to github container registry (ghcr.io)
|
||||
uses: bitdeps/helm-oci-charts-releaser@v0.1.5
|
||||
with:
|
||||
charts_dir: k8s/charts
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
oci_registry: ghcr.io/${{ github.repository_owner }}
|
||||
oci_username: github-actions
|
||||
oci_password: ${{ secrets.GITHUB_TOKEN }}
|
||||
skip_dependencies: true
|
||||
skip_helm_install: true
|
||||
skip_gh_release: true
|
||||
2
.github/workflows/java_integration_tests.yml
vendored
2
.github/workflows/java_integration_tests.yml
vendored
@ -111,7 +111,7 @@ jobs:
|
||||
|
||||
# Wait for S3 API
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
|
||||
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
|
||||
echo "✓ S3 API is ready"
|
||||
break
|
||||
fi
|
||||
|
||||
2
.github/workflows/java_unit_tests.yml
vendored
2
.github/workflows/java_unit_tests.yml
vendored
@ -49,7 +49,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: test-reports-java-${{ matrix.java }}
|
||||
path: |
|
||||
|
||||
4
.github/workflows/kafka-quicktest.yml
vendored
4
.github/workflows/kafka-quicktest.yml
vendored
@ -26,14 +26,14 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
16
.github/workflows/kafka-tests.yml
vendored
16
.github/workflows/kafka-tests.yml
vendored
@ -37,7 +37,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
@ -82,7 +82,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
@ -132,7 +132,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@ -311,7 +311,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@ -445,7 +445,7 @@ jobs:
|
||||
# Test consumer group functionality with explicit timeout
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
@ -473,7 +473,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@ -631,7 +631,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@ -789,7 +789,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
|
||||
140
.github/workflows/kms-tests.yml
vendored
140
.github/workflows/kms-tests.yml
vendored
@ -1,140 +0,0 @@
|
||||
name: "KMS Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/kms/**'
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'test/kms/**'
|
||||
- '.github/workflows/kms-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/kms/**'
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'test/kms/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}-kms-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
kms-provider-tests:
|
||||
name: KMS Provider Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run KMS provider integration tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
docker --version
|
||||
|
||||
make test-provider-ci
|
||||
|
||||
- name: Show OpenBao logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== OpenBao Container Logs ==="
|
||||
docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found"
|
||||
echo "=== Setup Logs ==="
|
||||
cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
make stop-openbao-ci || true
|
||||
|
||||
s3-kms-e2e-tests:
|
||||
name: S3 KMS End-to-End Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 KMS end-to-end tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
docker --version
|
||||
aws --version
|
||||
|
||||
make test-s3-kms-ci
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
echo "=== OpenBao Container Logs ==="
|
||||
cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found"
|
||||
echo "=== SeaweedFS Server Logs ==="
|
||||
tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"
|
||||
echo "=== Setup Logs ==="
|
||||
cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log"
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-kms-e2e-logs
|
||||
path: |
|
||||
/tmp/seaweedfs-kms-mini.log
|
||||
/tmp/openbao-ci-container.log
|
||||
/tmp/weed-kms-ci-setup.log
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
make stop-seaweedfs-ci || true
|
||||
make stop-openbao-ci || true
|
||||
make clean-ci || true
|
||||
@ -30,6 +30,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '10m'
|
||||
|
||||
jobs:
|
||||
@ -42,10 +43,10 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@ -69,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: metadata-subscribe-test-logs
|
||||
path: |
|
||||
|
||||
77
.github/workflows/multi-master-tests.yml
vendored
77
.github/workflows/multi-master-tests.yml
vendored
@ -1,77 +0,0 @@
|
||||
name: "Multi-Master Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/multi-master-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
multi-master-failover-tests:
|
||||
name: Multi-Master Failover Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Run multi-master failover tests
|
||||
# The tests in test/multi_master spin up their own 3-node master raft
|
||||
# cluster (using the freshly-installed `weed` binary) and exercise
|
||||
# leader-election, failover and recovery scenarios. The shared
|
||||
# test/testutil port-allocator regression test runs alongside since it
|
||||
# is a prerequisite for the cluster fixtures.
|
||||
run: |
|
||||
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
# test/multi_master/cluster.go creates per-test dirs via
|
||||
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
|
||||
# node's log into <baseDir>/logs/master*.log.
|
||||
echo "Collecting per-node master logs from temp directories..."
|
||||
mkdir -p /tmp/multi-master-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
|
||||
done
|
||||
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: multi-master-test-logs
|
||||
path: /tmp/multi-master-logs/
|
||||
retention-days: 7
|
||||
137
.github/workflows/nfs-tests.yml
vendored
137
.github/workflows/nfs-tests.yml
vendored
@ -1,137 +0,0 @@
|
||||
name: "NFS Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/nfs-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
nfs-integration:
|
||||
name: NFS Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run NFS Integration Tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running NFS integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Install test dependencies
|
||||
go mod download
|
||||
|
||||
# Run the protocol-layer tests. The kernel-mount tests require root
|
||||
# for mount(2) and are exercised in their own privileged step below;
|
||||
# skip them here so a "skipped because not root" line doesn't show
|
||||
# up as noise on every CI run.
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "NFS integration tests completed"
|
||||
|
||||
- name: Install kernel NFS client
|
||||
run: |
|
||||
# nfs-common provides mount.nfs; netbase provides /etc/protocols
|
||||
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y nfs-common netbase
|
||||
|
||||
- name: Run kernel-mount E2E tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running kernel-mount end-to-end tests..."
|
||||
echo "These mount the running 'weed nfs' subprocess via the actual"
|
||||
echo "Linux NFS client to catch protocol regressions invisible to"
|
||||
echo "the go-nfs-client-based tests above."
|
||||
echo "============================================"
|
||||
|
||||
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
|
||||
# binary that test/nfs/framework.go locates via $PATH) resolve
|
||||
# correctly under sudo, and pass through the Go module/cache dirs
|
||||
# so we don't redownload modules under root.
|
||||
sudo env "PATH=$PATH" \
|
||||
GOMODCACHE="$(go env GOMODCACHE)" \
|
||||
GOCACHE="$(go env GOCACHE)" \
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "Kernel-mount E2E tests completed"
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
|
||||
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
|
||||
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
|
||||
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
|
||||
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
|
||||
118
.github/workflows/pjdfstest.yml
vendored
118
.github/workflows/pjdfstest.yml
vendored
@ -1,118 +0,0 @@
|
||||
name: "pjdfstest POSIX Compliance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: pjdfstest/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-e2e
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build pjdfstest image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/pjdfstest
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-harness
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-harness
|
||||
|
||||
- name: Tag pjdfstest image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
|
||||
|
||||
- name: Start SeaweedFS cluster
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml up --wait
|
||||
|
||||
- name: Run pjdfstest
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
|
||||
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/pjdfstest-docker-logs
|
||||
for svc in master volume filer mount; do
|
||||
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: pjdfstest-results
|
||||
path: |
|
||||
/tmp/pjdfstest-output.log
|
||||
/tmp/pjdfstest-docker-logs/
|
||||
retention-days: 7
|
||||
39
.github/workflows/plugin-workers.yml
vendored
39
.github/workflows/plugin-workers.yml
vendored
@ -1,39 +0,0 @@
|
||||
name: "Plugin Worker Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
plugin-worker:
|
||||
name: "Plugin Worker: ${{ matrix.worker }}"
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- worker: erasure_coding
|
||||
path: test/plugin_workers/erasure_coding
|
||||
- worker: vacuum
|
||||
path: test/plugin_workers/vacuum
|
||||
- worker: volume_balance
|
||||
path: test/plugin_workers/volume_balance
|
||||
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Run plugin worker tests
|
||||
run: go test -v ./${{ matrix.path }}
|
||||
6
.github/workflows/postgres-tests.yml
vendored
6
.github/workflows/postgres-tests.yml
vendored
@ -25,14 +25,14 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: postgres-logs
|
||||
path: test/postgres/postgres-output.log
|
||||
|
||||
242
.github/workflows/rust-volume-server-tests.yml
vendored
242
.github/workflows/rust-volume-server-tests.yml
vendored
@ -1,242 +0,0 @@
|
||||
name: "Rust Volume Server Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
jobs:
|
||||
rust-unit-tests:
|
||||
name: Rust Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build Rust volume server
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
rust-integration-tests:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Run integration tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
run: |
|
||||
echo "Running Rust volume server integration tests..."
|
||||
go test -v -count=1 -timeout=15m ./test/volume_server/rust/...
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/rust-volume-server-it-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-server-integration-test-logs
|
||||
path: /tmp/rust-volume-server-it-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
rust-volume-go-tests:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-type: [grpc, http]
|
||||
shard: [1, 2, 3]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Run volume server integration tests with Rust volume
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
|
||||
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/rust-volume-go-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }}
|
||||
path: /tmp/rust-volume-go-test-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
|
||||
166
.github/workflows/rust_binaries_dev.yml
vendored
166
.github/workflows/rust_binaries_dev.yml
vendored
@ -1,166 +0,0 @@
|
||||
name: "rust: build dev volume server binaries"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- '.github/workflows/rust_binaries_dev.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
cleanup:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Delete old Rust volume dev assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
fail-if-no-assets: false
|
||||
assets: |
|
||||
weed-volume-*
|
||||
|
||||
build-rust-volume-dev-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: cleanup
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
asset_suffix: linux-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Set BUILD_TIME
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Package large disk binary
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume weed-volume-large-disk
|
||||
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --no-default-features
|
||||
|
||||
- name: Package normal binary
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume weed-volume-normal
|
||||
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
files: |
|
||||
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-rust-volume-dev-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: build-rust-volume-dev-linux
|
||||
runs-on: macos-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: aarch64-apple-darwin
|
||||
asset_suffix: darwin-arm64
|
||||
- target: x86_64-apple-darwin
|
||||
asset_suffix: darwin-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: brew install protobuf
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Set BUILD_TIME
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Package large disk binary
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package normal binary
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
files: |
|
||||
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
253
.github/workflows/rust_binaries_release.yml
vendored
253
.github/workflows/rust_binaries_release.yml
vendored
@ -1,253 +0,0 @@
|
||||
name: "rust: build versioned volume server binaries"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-rust-volume-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
asset_suffix: linux_amd64
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
asset_suffix: linux_arm64
|
||||
cross: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross-compilation tools
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-${{ matrix.target }}-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
# Large disk (default, 5bytes feature)
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
# Normal volume size
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: macos-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-apple-darwin
|
||||
asset_suffix: darwin_amd64
|
||||
- target: aarch64-apple-darwin
|
||||
asset_suffix: darwin_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: brew install protobuf
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-${{ matrix.target }}-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-windows:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: choco install protoc -y
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-windows-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-windows-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
shell: bash
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume.exe weed-volume-large-disk.exe
|
||||
7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
|
||||
rm weed-volume-large-disk.exe
|
||||
|
||||
cp seaweed-volume/target/release/weed-volume.exe weed-volume-normal.exe
|
||||
7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
|
||||
rm weed-volume-normal.exe
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip
|
||||
129
.github/workflows/s3-etag-acl-tests.yml
vendored
129
.github/workflows/s3-etag-acl-tests.yml
vendored
@ -1,129 +0,0 @@
|
||||
name: "S3 ETag and ACL Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-etag-acl-tests:
|
||||
name: S3 ETag + ACL Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-etag-acl
|
||||
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
|
||||
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-etag-acl \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
# Wait for the S3 endpoint to come up (returns 403 unauth before any
|
||||
# request is signed; that's fine — it means the server is listening).
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run ETag tests
|
||||
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
|
||||
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
|
||||
# SDK for Java v2 rejects the latter on the PutObject path.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/etag/...
|
||||
|
||||
- name: Run ACL versioning tests
|
||||
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
|
||||
# PutObjectAcl with and without versionId, modifying ACLs on different
|
||||
# versions independently.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/acl/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-etag-acl-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
@ -39,22 +39,6 @@ jobs:
|
||||
echo "=== Running S3 Integration Tests ==="
|
||||
go test -v -timeout=60s -run TestS3Integration ./...
|
||||
|
||||
- name: Run S3 DeleteBucketNotEmpty Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 DeleteBucketNotEmpty Tests ==="
|
||||
go test -v -timeout=60s -run TestS3DeleteBucketNotEmpty ./...
|
||||
|
||||
- name: Run S3 Empty Directory Marker Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 Empty Directory Marker Tests ==="
|
||||
go test -v -timeout=180s -run TestS3ListObjectsEmptyDirectoryMarkers ./...
|
||||
|
||||
- name: Run IAM Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
@ -65,7 +49,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: integration-test-logs
|
||||
path: test/s3/normal/*.log
|
||||
|
||||
2
.github/workflows/s3-filer-group-tests.yml
vendored
2
.github/workflows/s3-filer-group-tests.yml
vendored
@ -77,7 +77,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-filer-group-test-logs
|
||||
path: test/s3/filer_group/weed-test*.log
|
||||
|
||||
146
.github/workflows/s3-go-tests.yml
vendored
146
.github/workflows/s3-go-tests.yml
vendored
@ -76,7 +76,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-versioning-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@ -124,7 +124,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-versioning-compatibility-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@ -172,7 +172,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-cors-compatibility-logs
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@ -239,138 +239,12 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-retention-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/retention/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-lifecycle-tests:
|
||||
name: S3 Lifecycle Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# One job per test so each gets a fresh `weed mini` server, avoiding
|
||||
# the cross-test volume-pool exhaustion that surfaced when several
|
||||
# TTL-pinned bucket collections piled up in a single run.
|
||||
test:
|
||||
- TestLifecycleAbortIncompleteMultipartUpload
|
||||
- TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort
|
||||
- TestLifecycleBootstrapWalkOnExistingObjects
|
||||
- TestLifecycleConfigUpdateBetweenSweeps
|
||||
- TestLifecycleDeleteBucketLifecycleStopsDispatching
|
||||
- TestLifecycleDisabledRuleSkipsObject
|
||||
- TestLifecycleEmptyBucketSweepIsNoOp
|
||||
- TestLifecycleExpirationDateInThePast
|
||||
- TestLifecycleExpirationFiresOnBackdatedObject
|
||||
- TestLifecycleExpiredDeleteMarkerCleanup
|
||||
- TestLifecycleMultipleBucketsInOneSweep
|
||||
- TestLifecycleMultipleRulesInOneBucket
|
||||
- TestLifecycleNewerNoncurrentVersions
|
||||
- TestLifecycleNoncurrentVersionExpiration
|
||||
- TestLifecycleSizeFilterGreaterThan
|
||||
- TestLifecycleSkipsObjectLockedObjects
|
||||
- TestLifecycleSuspendedVersioningExpiration
|
||||
- TestLifecycleTagFilter
|
||||
- TestLifecycleVersionedBucketCreatesDeleteMarker
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run ${{ matrix.test }}
|
||||
timeout-minutes: 8
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN='^${{ matrix.test }}$$'
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Last 200 lines of server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-lifecycle-test-logs-${{ matrix.test }}
|
||||
path: test/s3/lifecycle/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Checksum Tests
|
||||
timeout-minutes: 16
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-checksum-test-logs
|
||||
path: test/s3/checksum/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-22.04
|
||||
@ -432,7 +306,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-cors-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@ -481,7 +355,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-retention-worm-logs
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@ -548,7 +422,7 @@ jobs:
|
||||
|
||||
- name: Upload stress test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-versioning-stress-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@ -587,8 +461,6 @@ jobs:
|
||||
export S3_ENDPOINT="http://localhost:8006"
|
||||
export S3_ACCESS_KEY="0555b35654ad1656d804"
|
||||
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
|
||||
export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
|
||||
export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
|
||||
|
||||
# Run the specific test that is equivalent to AWS S3 tagging behavior
|
||||
make test-with-server || {
|
||||
@ -604,7 +476,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-tagging-test-logs
|
||||
path: test/s3/tagging/weed-test*.log
|
||||
@ -657,7 +529,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-remote-cache-test-logs
|
||||
path: |
|
||||
|
||||
38
.github/workflows/s3-iam-tests.yml
vendored
38
.github/workflows/s3-iam-tests.yml
vendored
@ -5,8 +5,6 @@ on:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
push:
|
||||
@ -14,8 +12,6 @@ on:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
|
||||
@ -69,7 +65,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: iam-unit-test-results
|
||||
path: |
|
||||
@ -84,7 +80,7 @@ jobs:
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
|
||||
test-type: ["basic", "advanced", "policy-enforcement"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@ -121,7 +117,7 @@ jobs:
|
||||
"basic")
|
||||
echo "Running basic IAM functionality tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAMUserManagement|TestIAMAccessKeyManagement|TestIAMPolicyManagement" ./...
|
||||
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAM" ./...
|
||||
;;
|
||||
"advanced")
|
||||
echo "Running advanced IAM feature tests..."
|
||||
@ -133,28 +129,6 @@ jobs:
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./...
|
||||
;;
|
||||
"group")
|
||||
echo "Running IAM group management tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestIAMGroup" ./...
|
||||
;;
|
||||
"sts")
|
||||
echo "Running STS and service account tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
# SigV4-signed STS calls need admin credentials matching test_config.json.
|
||||
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
|
||||
export STS_TEST_ACCESS_KEY=test-access-key
|
||||
export STS_TEST_SECRET_KEY=test-secret-key
|
||||
# The use_service_account_credentials subtest is excluded because
|
||||
# newly-created service-account access keys are not currently
|
||||
# persisted to the filer after CreateServiceAccount — a
|
||||
# pre-existing sync issue tracked separately from the
|
||||
# GetFederationToken routing fix this PR addresses.
|
||||
go test -v -timeout 15m \
|
||||
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
|
||||
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
|
||||
./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: ${{ matrix.test-type }}"
|
||||
exit 1
|
||||
@ -188,7 +162,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-iam-integration-logs-${{ matrix.test-type }}
|
||||
path: test/s3/iam/weed-*.log
|
||||
@ -248,7 +222,7 @@ jobs:
|
||||
|
||||
- name: Upload distributed test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-iam-distributed-logs
|
||||
path: test/s3/iam/weed-*.log
|
||||
@ -300,7 +274,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-iam-performance-results
|
||||
path: |
|
||||
|
||||
4
.github/workflows/s3-keycloak-tests.yml
vendored
4
.github/workflows/s3-keycloak-tests.yml
vendored
@ -97,7 +97,7 @@ jobs:
|
||||
# Verify service accessibility
|
||||
echo "=== Verifying Service Accessibility ==="
|
||||
curl -f http://localhost:8080/realms/master
|
||||
curl -s http://localhost:8333/healthz
|
||||
curl -s http://localhost:8333
|
||||
echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
|
||||
|
||||
# Run Keycloak-specific tests
|
||||
@ -152,7 +152,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-keycloak-test-logs
|
||||
path: |
|
||||
|
||||
133
.github/workflows/s3-mutation-regression-tests.yml
vendored
133
.github/workflows/s3-mutation-regression-tests.yml
vendored
@ -1,133 +0,0 @@
|
||||
name: "S3 Mutation Regression Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/delete/**'
|
||||
- 'test/s3/distributed_lock/**'
|
||||
- 'test/s3/versioning/**'
|
||||
- 'test/volume_server/framework/**'
|
||||
- 'docker/compose/s3.json'
|
||||
- '.github/workflows/s3-mutation-regression-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-mutation-regression-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 versioning regression tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload versioning logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-regression-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 delete regression tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload delete logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-delete-regression-logs
|
||||
path: test/s3/delete/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run distributed lock regressions
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
|
||||
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
set -x
|
||||
mkdir -p "$TMPDIR"
|
||||
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
|
||||
|
||||
- name: Upload distributed lock logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-distributed-lock-regression-logs
|
||||
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
|
||||
retention-days: 3
|
||||
6
.github/workflows/s3-parquet-tests.yml
vendored
6
.github/workflows/s3-parquet-tests.yml
vendored
@ -41,7 +41,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
@ -121,7 +121,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: test-logs-python-${{ matrix.python-version }}
|
||||
path: |
|
||||
@ -148,7 +148,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
|
||||
- name: Run Go unit tests
|
||||
|
||||
8
.github/workflows/s3-policy-tests.yml
vendored
8
.github/workflows/s3-policy-tests.yml
vendored
@ -70,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: policy-unit-test-results
|
||||
path: |
|
||||
@ -178,7 +178,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-policy-variables-test-logs
|
||||
path: /tmp/weed_policy_test_server.log
|
||||
@ -299,7 +299,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
|
||||
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
@ -386,7 +386,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: trusted-proxy-test-logs
|
||||
path: /tmp/weed_proxy_test.log
|
||||
|
||||
124
.github/workflows/s3-proxy-signature-tests.yml
vendored
124
.github/workflows/s3-proxy-signature-tests.yml
vendored
@ -1,124 +0,0 @@
|
||||
name: "S3 Proxy Signature Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
proxy-signature-tests:
|
||||
name: S3 Proxy Signature Verification Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Build SeaweedFS binary for Linux
|
||||
run: |
|
||||
set -x
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -v -o test/s3/proxy_signature/weed ./weed
|
||||
|
||||
- name: Run S3 Proxy Signature Tests
|
||||
timeout-minutes: 10
|
||||
working-directory: test/s3/proxy_signature
|
||||
run: |
|
||||
set -x
|
||||
echo "Starting Docker Compose services..."
|
||||
docker compose up -d --build
|
||||
|
||||
# Check if containers are running
|
||||
echo "Checking container status..."
|
||||
docker compose ps
|
||||
|
||||
# Wait for services to be ready
|
||||
echo "Waiting for nginx proxy to be ready..."
|
||||
PROXY_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://localhost:9000/ > /dev/null 2>&1; then
|
||||
echo "Proxy is ready"
|
||||
PROXY_READY=1
|
||||
break
|
||||
fi
|
||||
echo "Waiting for proxy... ($i/30)"
|
||||
sleep 1
|
||||
done
|
||||
if [ $PROXY_READY -eq 0 ]; then
|
||||
echo "ERROR: Proxy failed to become ready after 30 seconds"
|
||||
echo "Docker compose logs:"
|
||||
docker compose logs --no-color || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait for SeaweedFS to be ready
|
||||
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
|
||||
S3_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
# Check logs first for the readiness line. weed mini's progress
|
||||
# board prints " S3 ready (Xs)"; older builds and the
|
||||
# standalone S3 binary log "S3 (gateway|service) ... ready".
|
||||
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)|S3[[:space:]]+ready"; then
|
||||
echo "SeaweedFS S3 gateway is ready"
|
||||
S3_READY=1
|
||||
break
|
||||
fi
|
||||
# Fallback: check headers via proxy (which is already ready)
|
||||
if curl -s -I http://localhost:9000/ | grep -qi "SeaweedFS"; then
|
||||
echo "SeaweedFS S3 gateway is responding via proxy"
|
||||
S3_READY=1
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 gateway... ($i/30)"
|
||||
sleep 1
|
||||
done
|
||||
if [ $S3_READY -eq 0 ]; then
|
||||
echo "ERROR: SeaweedFS S3 gateway failed to become ready after 30 seconds"
|
||||
echo "Latest seaweedfs logs:"
|
||||
docker compose logs --no-color --tail 20 seaweedfs || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run the test script inside AWS CLI container
|
||||
echo "Running test script..."
|
||||
docker run --rm --network host \
|
||||
--entrypoint bash \
|
||||
amazon/aws-cli:latest \
|
||||
-c "$(cat test.sh)"
|
||||
|
||||
TEST_RESULT=$?
|
||||
|
||||
# Cleanup
|
||||
docker compose down
|
||||
|
||||
exit $TEST_RESULT
|
||||
|
||||
- name: Cleanup on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/proxy_signature
|
||||
run: |
|
||||
echo "Cleaning up Docker containers..."
|
||||
ls -al weed || true
|
||||
ldd weed || true
|
||||
echo "Docker compose logs:"
|
||||
docker compose logs --no-color || true
|
||||
echo "Container status before cleanup:"
|
||||
docker ps -a
|
||||
echo "Stopping services..."
|
||||
docker compose down || true
|
||||
110
.github/workflows/s3-sdk-v2-routing-tests.yml
vendored
110
.github/workflows/s3-sdk-v2-routing-tests.yml
vendored
@ -1,110 +0,0 @@
|
||||
name: "S3 SDK V2 Route Disambiguation Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/sdk_v2_routing/**'
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/sdk_v2_routing/**'
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-sdk-v2-routing-tests:
|
||||
name: S3 SDK V2 Routing Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
# Pins the regression for issue #9559: AWS SDK V2 / Hadoop s3a
|
||||
# listing a bucket literally named "buckets" must get an XML
|
||||
# ListObjectsV2 response, not the JSON ListTableBuckets body
|
||||
# served by the S3 Tables REST endpoint on the same path.
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-sdk-v2-routing
|
||||
cat > /tmp/seaweedfs-sdk-v2-routing-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-sdk-v2-routing \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-sdk-v2-routing-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run SDK V2 routing tests
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/sdk_v2_routing/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sdk-v2-routing-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
81
.github/workflows/s3-spark-tests.yml
vendored
81
.github/workflows/s3-spark-tests.yml
vendored
@ -1,81 +0,0 @@
|
||||
name: "S3 Spark Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/s3/spark/**'
|
||||
- 'test/s3tables/testutil/**'
|
||||
- '.github/workflows/s3-spark-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-spark-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-spark-issue-repro-tests:
|
||||
name: S3 Spark Issue Reproduction Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: docker pull apache/spark:3.5.8
|
||||
|
||||
- name: Run S3 Spark integration tests
|
||||
working-directory: test/s3/spark
|
||||
timeout-minutes: 35
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting S3 Spark Integration Tests ==="
|
||||
|
||||
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
|
||||
echo "S3 Spark integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/spark
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|spark)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-spark-test-logs
|
||||
path: test/s3/spark/test-output.log
|
||||
retention-days: 3
|
||||
22
.github/workflows/s3-sse-tests.yml
vendored
22
.github/workflows/s3-sse-tests.yml
vendored
@ -73,12 +73,8 @@ jobs:
|
||||
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
else
|
||||
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
|
||||
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
|
||||
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
|
||||
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
|
||||
# same change that fixed them.
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
|
||||
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
fi
|
||||
|
||||
- name: Show server logs on failure
|
||||
@ -99,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@ -147,7 +143,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-compatibility-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@ -196,7 +192,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-metadata-persistence-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@ -245,7 +241,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-copy-operations-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@ -294,7 +290,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-multipart-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@ -344,7 +340,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-performance-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@ -393,7 +389,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-volume-encryption-logs
|
||||
path: /tmp/seaweedfs-sse-*.log
|
||||
|
||||
564
.github/workflows/s3-tables-tests.yml
vendored
564
.github/workflows/s3-tables-tests.yml
vendored
@ -2,6 +2,10 @@ name: "S3 Tables Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-tables-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@ -23,9 +27,6 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
@ -62,7 +63,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-tables-test-logs
|
||||
path: test/s3tables/table-buckets/test-output.log
|
||||
@ -83,9 +84,6 @@ jobs:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Run Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog
|
||||
@ -118,7 +116,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog/test-output.log
|
||||
@ -140,14 +138,11 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Pre-pull Trino image
|
||||
run: docker pull trinodb/trino:479
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
@ -184,555 +179,12 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: trino-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_trino/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
dremio-iceberg-catalog-tests:
|
||||
name: Dremio Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Dremio image
|
||||
run: docker pull dremio/dremio-oss:25.2.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Dremio Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog_dremio
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Dremio Iceberg Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Dremio Iceberg catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_dremio
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|dremio)" || true
|
||||
echo "=== Dremio containers ==="
|
||||
docker ps -a --filter "name=seaweed-dremio" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: dremio-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_dremio/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
doris-iceberg-catalog-tests:
|
||||
name: Doris Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Doris image
|
||||
run: docker pull apache/doris:doris-all-in-one-2.1.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Doris Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 30
|
||||
working-directory: test/s3tables/catalog_doris
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Doris Iceberg Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 25m . 2>&1 | tee test-output.log || {
|
||||
echo "Doris Iceberg catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_doris
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|doris)" || true
|
||||
echo "=== Doris containers ==="
|
||||
docker ps -a --filter "name=seaweed-doris" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: doris-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_doris/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Pre-pull Polaris image
|
||||
run: docker pull apache/polaris:latest
|
||||
|
||||
- name: Run Polaris Integration Tests
|
||||
timeout-minutes: 25
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Polaris Tests ==="
|
||||
|
||||
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
|
||||
echo "Polaris integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/polaris
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: polaris-test-logs
|
||||
path: test/s3tables/polaris/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
spark-iceberg-catalog-tests:
|
||||
name: Spark Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: docker pull apache/spark:3.5.1
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Spark Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog_spark
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Spark Iceberg Catalog Tests ==="
|
||||
|
||||
# Run Spark + Iceberg catalog integration tests
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Spark Iceberg catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_spark
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: spark-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_spark/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
risingwave-catalog-tests:
|
||||
name: RisingWave Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull RisingWave image
|
||||
run: |
|
||||
docker pull risingwavelabs/risingwave:v2.5.0
|
||||
docker pull postgres:16-alpine
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run RisingWave Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog_risingwave
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting RisingWave Catalog Tests ==="
|
||||
|
||||
# Run RisingWave catalog integration tests
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "RisingWave catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_risingwave
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: risingwave-catalog-test-logs
|
||||
path: test/s3tables/catalog_risingwave/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
sts-integration-tests:
|
||||
name: STS Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run STS Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/sts_integration
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting STS Integration Tests ==="
|
||||
|
||||
# Run STS integration tests
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "STS integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/sts_integration
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: sts-integration-test-logs
|
||||
path: test/s3tables/sts_integration/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
lakekeeper-integration-tests:
|
||||
name: Lakekeeper Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
|
||||
- name: Pre-pull LocalStack image (if needed)
|
||||
run: docker pull localstack/localstack:latest || true
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Lakekeeper Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/lakekeeper
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Lakekeeper Integration Tests ==="
|
||||
|
||||
# Run Lakekeeper integration tests
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Lakekeeper integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/lakekeeper
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: lakekeeper-integration-test-logs
|
||||
path: test/s3tables/lakekeeper/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
unity-catalog-integration-tests:
|
||||
name: Unity Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Unity Catalog image
|
||||
run: docker pull unitycatalog/unitycatalog:v0.4.0
|
||||
|
||||
- name: Pre-pull Python image for delta-rs writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Unity Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/unity_catalog
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Unity Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Unity Catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/unity_catalog
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
|
||||
echo "=== Unity Catalog containers ==="
|
||||
docker ps -a --filter "name=seaweed-unity-catalog" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: unity-catalog-integration-test-logs
|
||||
path: test/s3tables/unity_catalog/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
s3-tables-build-verification:
|
||||
name: S3 Tables Build Verification
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
99
.github/workflows/s3tests.yml
vendored
99
.github/workflows/s3tests.yml
vendored
@ -37,8 +37,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@ -132,49 +130,7 @@ jobs:
|
||||
done
|
||||
|
||||
echo "✅ S3 server is responding, starting tests..."
|
||||
|
||||
# Spawn the lifecycle worker so test_lifecycle_expiration etc. have
|
||||
# something driving deletions. The s3tests build tag rescales one
|
||||
# day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of
|
||||
# the upload's mtime; -dispatch / -checkpoint defaults are already
|
||||
# tightened under the same build tag.
|
||||
LC_LOG=/tmp/lifecycle-worker.log
|
||||
# -debug routes glog to stderr so the bootstrap walker's progress
|
||||
# shows up in $LC_LOG; without it weed shell silences glog.
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9333 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
# Aliveness check: a bad shell command exits in <1s and the suite
|
||||
# would otherwise just timeout the expiration tests with no signal.
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
# bash -e exits the step on the first tox failure, so move teardown
|
||||
# into a trap to guarantee the worker log + data dir reach the runner.
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@ -354,8 +310,11 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
versioning-tests:
|
||||
name: S3 Versioning & Object Lock tests
|
||||
@ -380,8 +339,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@ -550,8 +507,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@ -775,8 +730,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@ -997,39 +950,6 @@ jobs:
|
||||
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Spawn the lifecycle worker (see basic-tests block for context).
|
||||
LC_LOG=/tmp/lifecycle-worker-sql.log
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9337 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@ -1209,7 +1129,10 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
|
||||
|
||||
120
.github/workflows/samba-integration.yml
vendored
120
.github/workflows/samba-integration.yml
vendored
@ -1,120 +0,0 @@
|
||||
name: "Samba on FUSE Integration"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/samba/**'
|
||||
- '.github/workflows/samba-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/samba/**'
|
||||
- '.github/workflows/samba-integration.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: samba-integration/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samba-integration:
|
||||
name: samba-integration
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=samba-e2e
|
||||
cache-to: type=gha,mode=max,scope=samba-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build samba image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/samba
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:samba
|
||||
push: true
|
||||
cache-from: type=gha,scope=samba-harness
|
||||
cache-to: type=gha,mode=max,scope=samba-harness
|
||||
|
||||
- name: Tag samba image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:samba
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:samba chrislusf/seaweedfs:samba
|
||||
|
||||
- name: Start SeaweedFS cluster and Samba
|
||||
run: |
|
||||
docker compose -f test/samba/docker-compose.yml up --wait
|
||||
|
||||
- name: Run Samba test battery
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/samba/docker-compose.yml exec -T samba \
|
||||
/run_inside_container.sh 2>&1 | tee /tmp/samba-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/samba-docker-logs
|
||||
for svc in master volume filer samba; do
|
||||
docker compose -f test/samba/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/samba-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/samba/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: samba-integration-results
|
||||
path: |
|
||||
/tmp/samba-output.log
|
||||
/tmp/samba-docker-logs/
|
||||
retention-days: 7
|
||||
5
.github/workflows/sftp-tests.yml
vendored
5
.github/workflows/sftp-tests.yml
vendored
@ -24,6 +24,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
@ -36,10 +37,10 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@ -43,7 +43,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
@ -125,7 +125,7 @@ jobs:
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: spark-test-results
|
||||
path: test/java/spark/target/surefire-reports/
|
||||
@ -133,7 +133,7 @@ jobs:
|
||||
|
||||
- name: Publish test report
|
||||
if: always()
|
||||
uses: dorny/test-reporter@v3
|
||||
uses: dorny/test-reporter@v2
|
||||
with:
|
||||
name: Spark Test Results
|
||||
path: test/java/spark/target/surefire-reports/*.xml
|
||||
|
||||
46
.github/workflows/telemetry-integration.yml
vendored
46
.github/workflows/telemetry-integration.yml
vendored
@ -1,46 +0,0 @@
|
||||
name: Telemetry Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
telemetry-integration-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build telemetry server
|
||||
run: cd telemetry/server && go build -o telemetry-server .
|
||||
|
||||
- name: Run telemetry integration test
|
||||
run: go run telemetry/test/integration.go
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: telemetry-test-logs
|
||||
path: telemetry-server-test.log
|
||||
retention-days: 7
|
||||
@ -24,7 +24,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@ -35,7 +35,7 @@ jobs:
|
||||
set -e
|
||||
mkdir -p /tmp/data
|
||||
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
|
||||
until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
|
||||
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Setup Caddy
|
||||
run: |
|
||||
@ -54,7 +54,7 @@ jobs:
|
||||
- name: Start Caddy
|
||||
run: |
|
||||
./caddy start
|
||||
until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
|
||||
until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Create Bucket
|
||||
run: |
|
||||
@ -103,7 +103,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: seaweedfs-logs
|
||||
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
|
||||
|
||||
56
.github/workflows/tls-rotation-tests.yml
vendored
56
.github/workflows/tls-rotation-tests.yml
vendored
@ -1,56 +0,0 @@
|
||||
name: "TLS Rotation Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tls-rotation-tests:
|
||||
name: TLS Rotation Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run TLS Rotation Integration Tests
|
||||
working-directory: test/tls_rotation
|
||||
run: |
|
||||
go test -v -count=1 -timeout 5m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting master logs from temp directories..."
|
||||
mkdir -p /tmp/tls-rotation-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tls-rotation-test-logs
|
||||
path: /tmp/tls-rotation-test-logs/
|
||||
retention-days: 14
|
||||
13
.github/workflows/tus-tests.yml
vendored
13
.github/workflows/tus-tests.yml
vendored
@ -2,6 +2,17 @@ name: "TUS Protocol Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
@ -95,7 +106,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tus-test-logs
|
||||
path: |
|
||||
|
||||
56
.github/workflows/vacuum-integration-tests.yml
vendored
56
.github/workflows/vacuum-integration-tests.yml
vendored
@ -1,56 +0,0 @@
|
||||
name: "Vacuum Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run Vacuum Integration Tests
|
||||
working-directory: test/vacuum
|
||||
run: |
|
||||
go test -v -timeout 10m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/vacuum-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vacuum-integration-test-logs
|
||||
path: /tmp/vacuum-test-logs/
|
||||
retention-days: 14
|
||||
@ -1,121 +0,0 @@
|
||||
name: "Volume Server Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '30m'
|
||||
|
||||
jobs:
|
||||
volume-server-integration-tests:
|
||||
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-type: [grpc, http]
|
||||
shard: [1, 2, 3]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run volume server integration tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
|
||||
go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/volume-server-it-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-it-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: volume-server-integration-test-logs
|
||||
path: /tmp/volume-server-it-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -2,7 +2,6 @@
|
||||
vendor
|
||||
tags
|
||||
*.swp
|
||||
.claude/
|
||||
### OSX template
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
@ -142,6 +141,4 @@ test/s3/iam/.test_env
|
||||
/test/erasure_coding/admin_dockertest/tmp
|
||||
/test/erasure_coding/admin_dockertest/task_logs
|
||||
weed_bin
|
||||
telemetry/server/telemetry-server
|
||||
.aider*
|
||||
/seaweed-volume/docs
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
{
|
||||
"setup": [],
|
||||
"teardown": [],
|
||||
"run": []
|
||||
}
|
||||
6
Makefile
6
Makefile
@ -1,4 +1,4 @@
|
||||
.PHONY: test admin-generate admin-build admin-clean admin-dev admin-run admin-test admin-fmt admin-help weed-commands
|
||||
.PHONY: test admin-generate admin-build admin-clean admin-dev admin-run admin-test admin-fmt admin-help
|
||||
|
||||
BINARY = weed
|
||||
ADMIN_DIR = weed/admin
|
||||
@ -11,9 +11,6 @@ all: install
|
||||
install: admin-generate
|
||||
cd weed; go install
|
||||
|
||||
weed-commands:
|
||||
cd weed && $(MAKE) weed-db weed-sql
|
||||
|
||||
warp_install:
|
||||
go install github.com/minio/warp@v0.7.6
|
||||
|
||||
@ -42,6 +39,7 @@ test: admin-generate
|
||||
|
||||
# Admin component targets
|
||||
admin-generate:
|
||||
@echo "Generating admin component templates..."
|
||||
@cd $(ADMIN_DIR) && $(MAKE) generate
|
||||
|
||||
admin-build: admin-generate
|
||||
|
||||
73
README.md
73
README.md
@ -56,6 +56,7 @@ Table of Contents
|
||||
* [Quick Start](#quick-start)
|
||||
* [Quick Start with weed mini](#quick-start-with-weed-mini)
|
||||
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
|
||||
* [Quick Start with Single Binary](#quick-start-with-single-binary)
|
||||
* [Introduction](#introduction)
|
||||
* [Features](#features)
|
||||
* [Additional Features](#additional-features)
|
||||
@ -79,41 +80,39 @@ Table of Contents
|
||||
|
||||
|
||||
## Quick Start with weed mini ##
|
||||
The easiest way to get started with SeaweedFS for development and testing:
|
||||
|
||||
Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file, or run `go install github.com/seaweedfs/seaweedfs/weed@latest`. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
AWS_ACCESS_KEY_ID=admin \
|
||||
AWS_SECRET_ACCESS_KEY=secret \
|
||||
S3_BUCKET=my-bucket \
|
||||
# remove quarantine on macOS
|
||||
# xattr -d com.apple.quarantine ./weed
|
||||
|
||||
./weed mini -dir=/data
|
||||
```
|
||||
|
||||
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
|
||||
|
||||
The same command starts everything else too:
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
This single command starts a complete SeaweedFS setup with:
|
||||
- **Master UI**: http://localhost:9333
|
||||
- **Volume Server**: http://localhost:9340
|
||||
- **Filer UI**: http://localhost:8888
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
- **WebDAV**: http://localhost:7333
|
||||
- **Admin UI**: http://localhost:23646
|
||||
|
||||
> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
|
||||
|
||||
Perfect for development, testing, learning SeaweedFS, and single-node deployments. To scale out, add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, on another machine, or on thousands of machines.
|
||||
Perfect for development, testing, learning SeaweedFS, and single node deployments!
|
||||
|
||||
## Quick Start for S3 API on Docker ##
|
||||
|
||||
```bash
|
||||
docker run -p 8333:8333 \
|
||||
-e AWS_ACCESS_KEY_ID=admin \
|
||||
-e AWS_SECRET_ACCESS_KEY=secret \
|
||||
-e S3_BUCKET=my-bucket \
|
||||
chrislusf/seaweedfs
|
||||
```
|
||||
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
|
||||
|
||||
Same behavior as the `weed mini` command above — the S3 endpoint is at http://localhost:8333 with `my-bucket` pre-created. Drop the env vars to run anonymously for development.
|
||||
## Quick Start with Single Binary ##
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
|
||||
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. The difference with `weed mini` is that `weed mini` can auto configure based on the single host environment, while `weed server` requires manual configuration and are designed for production use.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
||||
# Introduction #
|
||||
|
||||
@ -146,11 +145,6 @@ SeaweedFS can achieve both fast local access time and elastic cloud storage capa
|
||||
What's more, the cloud storage access API cost is minimized.
|
||||
Faster and cheaper than direct cloud storage!
|
||||
|
||||
SeaweedFS also ships a built-in **Iceberg REST Catalog**, turning the same cluster into a self-contained lakehouse.
|
||||
Spark, Trino, Dremio, DuckDB, and RisingWave can query Iceberg tables directly — no Hive Metastore, Glue, or
|
||||
external catalog service required. Storage and table metadata live in one system, simplifying on-prem and
|
||||
small-team analytics stacks.
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
# Features #
|
||||
@ -187,13 +181,6 @@ small-team analytics stacks.
|
||||
* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
|
||||
* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]
|
||||
|
||||
## Data Lakehouse Features ##
|
||||
* [S3 Table Buckets][S3TableBucket] expose a dedicated namespace for Iceberg tables with strict layout validation.
|
||||
* Built-in [Iceberg REST Catalog][IcebergCatalog] runs alongside the S3 endpoint — no external metastore needed.
|
||||
* Native integrations with [Apache Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], and [RisingWave][RisingWaveIceberg].
|
||||
* [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan removal, manifest rewriting.
|
||||
* Granular IAM at the bucket, namespace, and table level via standard S3 bucket policies.
|
||||
|
||||
## Kubernetes ##
|
||||
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
|
||||
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
|
||||
@ -217,14 +204,6 @@ small-team analytics stacks.
|
||||
[KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
|
||||
[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
|
||||
[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
|
||||
[S3TableBucket]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Table-Bucket
|
||||
[IcebergCatalog]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS-Iceberg-Catalog
|
||||
[IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance
|
||||
[SparkIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Spark-Iceberg-Integration
|
||||
[TrinoIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Trino-Iceberg-Integration
|
||||
[DremioIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Dremio-Iceberg-Integration
|
||||
[DuckDBIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/DuckDB-Iceberg-Integration
|
||||
[RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration
|
||||
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
@ -509,22 +488,20 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mong
|
||||
|
||||
### Compared to MinIO ###
|
||||
|
||||
Please note, as Apr 25, 2026 MinIO ceased developement. It's strongly discouraged to use that unmaintained software with multiple security bugs.
|
||||
MinIO follows AWS S3 closely and is ideal for testing for S3 API. It has good UI, policies, versionings, etc. SeaweedFS is trying to catch up here. It is also possible to put MinIO as a gateway in front of SeaweedFS later.
|
||||
|
||||
MinIO followed AWS S3 closely and was ideal for testing for S3 API. It had good UI, policies, versionings, etc. SeaweedFS is trying to catch up here.
|
||||
MinIO metadata are in simple files. Each file write will incur extra writes to corresponding meta file.
|
||||
|
||||
MinIO metadata were in simple files. Each file write will incur extra writes to corresponding meta file.
|
||||
|
||||
MinIO did not have optimization for lots of small files. The files were simply stored as is to local disks.
|
||||
MinIO does not have optimization for lots of small files. The files are simply stored as is to local disks.
|
||||
Plus the extra meta file and shards for erasure coding, it only amplifies the LOSF problem.
|
||||
|
||||
MinIO had multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
|
||||
MinIO has multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
|
||||
|
||||
MinIO had full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
|
||||
MinIO has full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
|
||||
|
||||
MinIO did not have POSIX-like API support.
|
||||
MinIO does not have POSIX-like API support.
|
||||
|
||||
MinIO had specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
|
||||
MinIO has specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
|
||||
|
||||
## Dev Plan ##
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
24
SECURITY.md
24
SECURITY.md
@ -1,24 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you find a security issue in SeaweedFS, please report it privately:
|
||||
|
||||
- Email: support@seaweedfs.com
|
||||
- Do not open a public GitHub issue
|
||||
|
||||
Please include:
|
||||
- A clear description of the issue
|
||||
- Steps to reproduce (if possible)
|
||||
- Affected versions
|
||||
|
||||
## Response
|
||||
|
||||
- We will respond as soon as possible (usually within 1 business day)
|
||||
- We will investigate and work on a fix
|
||||
- We may coordinate disclosure with you
|
||||
|
||||
## Notes
|
||||
|
||||
- Please allow time for a fix before public disclosure
|
||||
- If you’re unsure whether something is a security issue, feel free to reach out
|
||||
@ -1,790 +0,0 @@
|
||||
# Execution Plan: SeaweedFS Volume Server — Go to Rust Port
|
||||
|
||||
## Scope Summary
|
||||
|
||||
| Component | Go Source | Lines (non-test) | Description |
|
||||
|---|---|---|---|
|
||||
| CLI & startup | `weed/command/volume.go` | 476 | ~40 CLI flags, server bootstrap |
|
||||
| HTTP server + handlers | `weed/server/volume_server*.go` | 1,517 | Struct, routes, read/write/delete handlers |
|
||||
| gRPC handlers | `weed/server/volume_grpc_*.go` | 3,073 | 40 RPC method implementations |
|
||||
| Storage engine | `weed/storage/` | 15,271 | Volumes, needles, index, compaction, EC, backend |
|
||||
| Protobuf definitions | `weed/pb/volume_server.proto` | 759 | Service + message definitions |
|
||||
| Shared utilities | `weed/security/`, `weed/stats/`, `weed/util/` | ~2,000+ | JWT, TLS, metrics, helpers |
|
||||
| **Total** | | **~23,000+** | |
|
||||
|
||||
## Rust Crate & Dependency Strategy
|
||||
|
||||
```
|
||||
seaweed-volume/
|
||||
├── Cargo.toml
|
||||
├── build.rs # protobuf codegen
|
||||
├── proto/
|
||||
│ ├── volume_server.proto # copied from Go, adapted
|
||||
│ └── remote.proto
|
||||
├── src/
|
||||
│ ├── main.rs # CLI entry point
|
||||
│ ├── config.rs # CLI flags + config
|
||||
│ ├── server/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── volume_server.rs # VolumeServer struct + lifecycle
|
||||
│ │ ├── http_handlers.rs # HTTP route dispatch
|
||||
│ │ ├── http_read.rs # GET/HEAD handlers
|
||||
│ │ ├── http_write.rs # POST/PUT handlers
|
||||
│ │ ├── http_delete.rs # DELETE handler
|
||||
│ │ ├── http_admin.rs # /status, /healthz, /ui
|
||||
│ │ ├── grpc_service.rs # gRPC trait impl dispatch
|
||||
│ │ ├── grpc_vacuum.rs
|
||||
│ │ ├── grpc_copy.rs
|
||||
│ │ ├── grpc_erasure_coding.rs
|
||||
│ │ ├── grpc_tail.rs
|
||||
│ │ ├── grpc_admin.rs
|
||||
│ │ ├── grpc_read_write.rs
|
||||
│ │ ├── grpc_batch_delete.rs
|
||||
│ │ ├── grpc_scrub.rs
|
||||
│ │ ├── grpc_tier.rs
|
||||
│ │ ├── grpc_remote.rs
|
||||
│ │ ├── grpc_query.rs
|
||||
│ │ ├── grpc_state.rs
|
||||
│ │ └── grpc_client_to_master.rs # heartbeat
|
||||
│ ├── storage/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── store.rs # Store (multi-disk manager)
|
||||
│ │ ├── volume.rs # Volume struct + lifecycle
|
||||
│ │ ├── volume_read.rs
|
||||
│ │ ├── volume_write.rs
|
||||
│ │ ├── volume_compact.rs
|
||||
│ │ ├── volume_info.rs
|
||||
│ │ ├── needle/
|
||||
│ │ │ ├── mod.rs
|
||||
│ │ │ ├── needle.rs # Needle struct + serialization
|
||||
│ │ │ ├── needle_read.rs
|
||||
│ │ │ ├── needle_write.rs
|
||||
│ │ │ ├── needle_map.rs # in-memory NeedleMap
|
||||
│ │ │ ├── needle_value.rs
|
||||
│ │ │ └── crc.rs
|
||||
│ │ ├── super_block.rs
|
||||
│ │ ├── idx/
|
||||
│ │ │ ├── mod.rs
|
||||
│ │ │ └── idx.rs # .idx file format read/write
|
||||
│ │ ├── needle_map_leveldb.rs
|
||||
│ │ ├── types.rs # NeedleId, Offset, Size, DiskType
|
||||
│ │ ├── disk_location.rs # DiskLocation per-directory
|
||||
│ │ ├── erasure_coding/
|
||||
│ │ │ ├── mod.rs
|
||||
│ │ │ ├── ec_volume.rs
|
||||
│ │ │ ├── ec_shard.rs
|
||||
│ │ │ ├── ec_encoder.rs # Reed-Solomon encoding
|
||||
│ │ │ └── ec_decoder.rs
|
||||
│ │ └── backend/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── disk.rs
|
||||
│ │ └── s3_backend.rs # tiered storage to S3
|
||||
│ ├── topology/
|
||||
│ │ └── volume_layout.rs # replication placement
|
||||
│ ├── security/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── guard.rs # whitelist + JWT gate
|
||||
│ │ ├── jwt.rs
|
||||
│ │ └── tls.rs
|
||||
│ ├── stats/
|
||||
│ │ ├── mod.rs
|
||||
│ │ └── metrics.rs # Prometheus counters/gauges
|
||||
│ └── util/
|
||||
│ ├── mod.rs
|
||||
│ ├── grpc.rs
|
||||
│ ├── http.rs
|
||||
│ └── file.rs
|
||||
└── tests/
|
||||
├── integration/
|
||||
│ ├── http_read_test.rs
|
||||
│ ├── http_write_test.rs
|
||||
│ ├── grpc_test.rs
|
||||
│ └── storage_test.rs
|
||||
└── unit/
|
||||
├── needle_test.rs
|
||||
├── idx_test.rs
|
||||
├── super_block_test.rs
|
||||
└── ec_test.rs
|
||||
```
|
||||
|
||||
### Key Rust dependencies
|
||||
|
||||
| Purpose | Crate |
|
||||
|---|---|
|
||||
| Async runtime | `tokio` |
|
||||
| gRPC | `tonic` + `prost` |
|
||||
| HTTP server | `hyper` + `axum` |
|
||||
| CLI parsing | `clap` (derive) |
|
||||
| Prometheus metrics | `prometheus` |
|
||||
| JWT | `jsonwebtoken` |
|
||||
| TLS | `rustls` + `tokio-rustls` |
|
||||
| LevelDB | `rusty-leveldb` or `rocksdb` |
|
||||
| Reed-Solomon EC | `reed-solomon-erasure` |
|
||||
| Logging | `tracing` + `tracing-subscriber` |
|
||||
| Config (security.toml) | `toml` + `serde` |
|
||||
| CRC32 | `crc32fast` |
|
||||
| Memory-mapped files | `memmap2` |
|
||||
|
||||
---
|
||||
|
||||
## Phased Execution Plan
|
||||
|
||||
### Phase 1: Project Skeleton & Protobuf Codegen
|
||||
**Goal:** Cargo project compiles, proto codegen works, CLI parses all flags.
|
||||
|
||||
**Steps:**
|
||||
|
||||
1.1. Create `seaweed-volume/Cargo.toml` with all dependencies listed above.
|
||||
|
||||
1.2. Copy `volume_server.proto` and `remote.proto` into `proto/`. Adjust package paths for Rust codegen.
|
||||
|
||||
1.3. Create `build.rs` using `tonic-build` to compile `.proto` files into Rust types.
|
||||
|
||||
1.4. Create `src/main.rs` with `clap` derive structs mirroring all 40 CLI flags from `weed/command/volume.go`:
|
||||
- `--port` (default 8080)
|
||||
- `--port.grpc` (default 0 → 10000+port)
|
||||
- `--port.public` (default 0 → same as port)
|
||||
- `--ip` (auto-detect)
|
||||
- `--id` (default empty → ip:port)
|
||||
- `--publicUrl`
|
||||
- `--ip.bind`
|
||||
- `--master` (default "localhost:9333")
|
||||
- `--mserver` (deprecated compat)
|
||||
- `--preStopSeconds` (default 10)
|
||||
- `--idleTimeout` (default 30)
|
||||
- `--dataCenter`
|
||||
- `--rack`
|
||||
- `--index` [memory|leveldb|leveldbMedium|leveldbLarge]
|
||||
- `--disk` [hdd|ssd|<tag>]
|
||||
- `--tags`
|
||||
- `--dir` (default temp dir)
|
||||
- `--dir.idx`
|
||||
- `--max` (default "8")
|
||||
- `--whiteList`
|
||||
- `--minFreeSpacePercent` (default "1")
|
||||
- `--minFreeSpace`
|
||||
- `--images.fix.orientation` (default false)
|
||||
- `--readMode` [local|proxy|redirect] (default "proxy")
|
||||
- `--cpuprofile`
|
||||
- `--memprofile`
|
||||
- `--compactionMBps` (default 0)
|
||||
- `--maintenanceMBps` (default 0)
|
||||
- `--fileSizeLimitMB` (default 256)
|
||||
- `--concurrentUploadLimitMB` (default 0)
|
||||
- `--concurrentDownloadLimitMB` (default 0)
|
||||
- `--pprof` (default false)
|
||||
- `--metricsPort` (default 0)
|
||||
- `--metricsIp`
|
||||
- `--inflightUploadDataTimeout` (default 60s)
|
||||
- `--inflightDownloadDataTimeout` (default 60s)
|
||||
- `--hasSlowRead` (default true)
|
||||
- `--readBufferSizeMB` (default 4)
|
||||
- `--index.leveldbTimeout` (default 0)
|
||||
- `--debug` (default false)
|
||||
- `--debug.port` (default 6060)
|
||||
|
||||
1.5. Implement the same flag validation logic from `startVolumeServer()`:
|
||||
- Parse comma-separated `--dir`, `--max`, `--minFreeSpace`, `--disk`, `--tags`
|
||||
- Replicate single-value-to-all-dirs expansion
|
||||
- Validate count matches between dirs and limits
|
||||
- `--mserver` backward compat
|
||||
|
||||
1.6. **Test:** `cargo build` succeeds. `cargo run -- --help` shows all flags. Proto types generated.
|
||||
|
||||
**Verification:** Run with `--port 8080 --dir /tmp --master localhost:9333` — should parse without error and print config.
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: Core Storage Types & On-Disk Format
|
||||
**Goal:** Read and write the SeaweedFS needle/volume binary format bit-for-bit compatible with Go.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/types/needle_types.go` → `src/storage/types.rs`
|
||||
- `weed/storage/needle/needle.go` → `src/storage/needle/needle.rs`
|
||||
- `weed/storage/needle/needle_read.go` → `src/storage/needle/needle_read.rs`
|
||||
- `weed/storage/needle/needle_write.go` (partial) → `src/storage/needle/needle_write.rs`
|
||||
- `weed/storage/needle/crc.go` → `src/storage/needle/crc.rs`
|
||||
- `weed/storage/needle/needle_value_map.go` → `src/storage/needle/needle_value.rs`
|
||||
- `weed/storage/super_block/super_block.go` → `src/storage/super_block.rs`
|
||||
- `weed/storage/idx/` → `src/storage/idx/`
|
||||
|
||||
**Steps:**
|
||||
|
||||
2.1. **Fundamental types** (`types.rs`):
|
||||
- `NeedleId` (u64), `Offset` (u32 or u64 depending on version), `Size` (i32, negative = deleted)
|
||||
- `Cookie` (u32)
|
||||
- `DiskType` enum (HDD, SSD, Custom)
|
||||
- Version constants (Version1=1, Version2=2, Version3=3, CurrentVersion=3)
|
||||
- Byte serialization matching Go's `binary.BigEndian` encoding
|
||||
|
||||
2.2. **SuperBlock** (`super_block.rs`):
|
||||
- 8-byte header: Version(1) + ReplicaPlacement(1) + TTL(2) + CompactRevision(2) + Reserved(2)
|
||||
- `ReplicaPlacement` struct with same/diff rack/dc counts
|
||||
- `TTL` struct with count + unit
|
||||
- Read/write from first 8 bytes of `.dat` file
|
||||
- Match exact byte layout from `super_block.go`
|
||||
|
||||
2.3. **Needle binary format** (`needle.rs`, `needle_read.rs`):
|
||||
- Version 2/3 header: Cookie(4) + NeedleId(8) + Size(4)
|
||||
- Body: Data, Flags, Name, Mime, PairsSize, Pairs, LastModified, TTL, Checksum, AppendAtNs, Padding
|
||||
- CRC32 checksum (matching Go's `crc32.ChecksumIEEE`)
|
||||
- Padding to 8-byte alignment
|
||||
- Read path: read header → compute body length → read body → verify CRC
|
||||
|
||||
2.4. **Idx file format** (`idx/`):
|
||||
- Fixed 16-byte records: NeedleId(8) + Offset(4) + Size(4)
|
||||
- Sequential append-only file
|
||||
- Walk/iterate all entries
|
||||
- Binary search not used (loaded into memory map)
|
||||
|
||||
2.5. **NeedleMap (in-memory)** (`needle_map.rs`):
|
||||
- HashMap<NeedleId, NeedleValue> where NeedleValue = {Offset, Size}
|
||||
- Load from `.idx` file on volume mount
|
||||
- Support Get, Set, Delete operations
|
||||
- Track file count, deleted count, deleted byte count
|
||||
|
||||
2.6. **Tests:**
|
||||
- Unit test: write a needle to bytes → read it back → verify fields match
|
||||
- Unit test: write/read SuperBlock round-trip
|
||||
- Unit test: write/read idx entries round-trip
|
||||
- **Cross-compat test:** Use Go volume server to create a small volume with known data. Read it from Rust and verify all needles decoded correctly. (Keep test fixture `.dat`/`.idx` files in `tests/fixtures/`)
|
||||
|
||||
---
|
||||
|
||||
### Phase 3: Volume Struct & Lifecycle
|
||||
**Goal:** Mount, read from, write to, and unmount a volume.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/volume.go` → `src/storage/volume.rs`
|
||||
- `weed/storage/volume_read.go` → `src/storage/volume_read.rs`
|
||||
- `weed/storage/volume_write.go` → `src/storage/volume_write.rs`
|
||||
- `weed/storage/volume_loading.go`
|
||||
- `weed/storage/volume_vacuum.go` → `src/storage/volume_compact.rs`
|
||||
- `weed/storage/volume_info/volume_info.go` → `src/storage/volume_info.rs`
|
||||
- `weed/storage/volume_super_block.go`
|
||||
|
||||
**Steps:**
|
||||
|
||||
3.1. **Volume struct** (`volume.rs`):
|
||||
- Fields: Id, dir, dataFile, nm (NeedleMap), SuperBlock, readOnly, lastModifiedTs, lastCompactIndexOffset, lastCompactRevision
|
||||
- `noWriteOrDelete` / `noWriteCanDelete` / `readOnly` state flags
|
||||
- File handles for `.dat` file (read + append)
|
||||
- Lock strategy: `RwLock` for concurrent reads, exclusive writes
|
||||
|
||||
3.2. **Volume loading** — exact logic from `volume_loading.go`:
|
||||
- Open `.dat` file, read SuperBlock from first 8 bytes
|
||||
- Load `.idx` file into NeedleMap
|
||||
- Handle `.vif` (VolumeInfo) JSON sidecar file
|
||||
- Set volume state based on SuperBlock + VolumeInfo
|
||||
|
||||
3.3. **Volume read** (`volume_read.rs`) — from `volume_read.go`:
|
||||
- `ReadNeedle(needleId, cookie)`: lookup in NeedleMap → seek in .dat → read needle bytes → verify cookie + CRC → return data
|
||||
- Handle deleted needles (Size < 0)
|
||||
- `ReadNeedleBlob(offset, size)`: raw blob read
|
||||
- `ReadNeedleMeta(needleId, offset, size)`: read metadata only
|
||||
|
||||
3.4. **Volume write** (`volume_write.rs`) — from `volume_write.go`:
|
||||
- `WriteNeedle(needle)`: serialize needle → append to .dat → update .idx → update NeedleMap
|
||||
- `DeleteNeedle(needleId)`: mark as deleted in NeedleMap + append tombstone to .idx
|
||||
- File size limit check
|
||||
- Concurrent write serialization (mutex on write path)
|
||||
|
||||
3.5. **Volume compaction** (`volume_compact.rs`) — from `volume_vacuum.go`:
|
||||
- `CheckCompact()`: compute garbage ratio
|
||||
- `Compact()`: create new .dat/.idx, copy only live needles, update compact revision
|
||||
- `CommitCompact()`: rename compacted files over originals
|
||||
- `CleanupCompact()`: remove temp files
|
||||
- Throttle by `compactionBytePerSecond`
|
||||
|
||||
3.6. **Volume info** (`volume_info.rs`):
|
||||
- Read/write `.vif` JSON sidecar
|
||||
- VolumeInfo protobuf struct mapping
|
||||
- Remote file references for tiered storage
|
||||
|
||||
3.7. **Tests:**
|
||||
- Mount a volume, write 100 needles, read them all back, verify content
|
||||
- Delete 50 needles, verify they return "deleted"
|
||||
- Compact, verify only 50 remain, verify content
|
||||
- Read Go-created volume fixtures
|
||||
|
||||
---
|
||||
|
||||
### Phase 4: Store (Multi-Volume, Multi-Disk Manager)
|
||||
**Goal:** Manage multiple volumes across multiple disk directories.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/store.go` → `src/storage/store.rs`
|
||||
- `weed/storage/disk_location.go` → `src/storage/disk_location.rs`
|
||||
- `weed/storage/store_ec.go`
|
||||
- `weed/storage/store_state.go`
|
||||
|
||||
**Steps:**
|
||||
|
||||
4.1. **DiskLocation** (`disk_location.rs`):
|
||||
- Directory path, max volume count, min free space, disk type, tags
|
||||
- Load all volumes from directory on startup
|
||||
- Track free space, check writable
|
||||
|
||||
4.2. **Store** (`store.rs`):
|
||||
- Vector of `DiskLocation`s
|
||||
- `GetVolume(volumeId)` → lookup across all locations
|
||||
- `HasVolume(volumeId)` check
|
||||
- `AllocateVolume(...)` — create new volume in appropriate location
|
||||
- `DeleteVolume(...)`, `MountVolume(...)`, `UnmountVolume(...)`
|
||||
- `DeleteCollection(collection)` — delete all volumes of a collection
|
||||
- Collect volume status for heartbeat
|
||||
- `SetStopping()`, `Close()`
|
||||
- Persistent state (maintenance mode) via `store_state.go`
|
||||
|
||||
4.3. **Store state** — `VolumeServerState` protobuf with maintenance flag, persisted to disk.
|
||||
|
||||
4.4. **Tests:**
|
||||
- Create store with 2 dirs, allocate volumes in each, verify load balancing
|
||||
- Mount/unmount/delete lifecycle
|
||||
- State persistence across restart
|
||||
|
||||
---
|
||||
|
||||
### Phase 5: Erasure Coding
|
||||
**Goal:** Full EC shard encode/decode/read/write/rebuild.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/erasure_coding/` (3,599 lines)
|
||||
|
||||
**Steps:**
|
||||
|
||||
5.1. **EC volume + shard structs** — `EcVolume`, `EcShard` with file handles for `.ec00`–`.ec13` shard files + `.ecx` index + `.ecj` journal.
|
||||
|
||||
5.2. **EC encoder** — Reed-Solomon 10+4 (configurable) encoding using `reed-solomon-erasure` crate:
|
||||
- `VolumeEcShardsGenerate`: read .dat → split into data shards → compute parity → write .ec00-.ec13 + .ecx
|
||||
|
||||
5.3. **EC decoder/reader** — reconstruct data from any 10 of 14 shards:
|
||||
- `EcShardRead`: read range from a specific shard
|
||||
- Locate needle in EC volume via .ecx index
|
||||
- Handle cross-shard needle reads
|
||||
|
||||
5.4. **EC shard operations:**
|
||||
- Copy, delete, mount, unmount shards
|
||||
- `VolumeEcShardsRebuild`: rebuild missing shards from remaining
|
||||
- `VolumeEcShardsToVolume`: reconstruct .dat from EC shards
|
||||
- `VolumeEcBlobDelete`: mark deleted in EC journal
|
||||
- `VolumeEcShardsInfo`: report shard metadata
|
||||
|
||||
5.5. **Tests:**
|
||||
- Encode a volume → verify 14 shards created
|
||||
- Delete 4 shards → rebuild → verify data intact
|
||||
- Read individual needles from EC volume
|
||||
- Cross-compat with Go-generated EC shards
|
||||
|
||||
---
|
||||
|
||||
### Phase 6: Backend / Tiered Storage
|
||||
**Goal:** Support tiered storage to remote backends (S3, etc).
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/backend/` (1,850 lines)
|
||||
|
||||
**Steps:**
|
||||
|
||||
6.1. **Backend trait** — abstract `BackendStorage` trait with `ReadAt`, `WriteAt`, `Truncate`, `Close`, `Name`.
|
||||
|
||||
6.2. **Disk backend** — default local disk implementation.
|
||||
|
||||
6.3. **S3 backend** — upload .dat to S3, read ranges via S3 range requests.
|
||||
|
||||
6.4. **Tier move operations:**
|
||||
- `VolumeTierMoveDatToRemote`: upload .dat to remote, optionally delete local
|
||||
- `VolumeTierMoveDatFromRemote`: download .dat from remote
|
||||
|
||||
6.5. **Tests:**
|
||||
- Disk backend read/write round-trip
|
||||
- S3 backend with mock/localstack
|
||||
|
||||
---
|
||||
|
||||
### Phase 7: Security Layer
|
||||
**Goal:** JWT authentication, whitelist guard, TLS configuration.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/security/guard.go` → `src/security/guard.rs`
|
||||
- `weed/security/jwt.go` → `src/security/jwt.rs`
|
||||
- `weed/security/tls.go` → `src/security/tls.rs`
|
||||
|
||||
**Steps:**
|
||||
|
||||
7.1. **Guard** (`guard.rs`):
|
||||
- Whitelist IP check (exact match on `r.RemoteAddr`)
|
||||
- Wrap handlers with whitelist enforcement
|
||||
- `UpdateWhiteList()` for live reload
|
||||
|
||||
7.2. **JWT** (`jwt.rs`):
|
||||
- `SeaweedFileIdClaims` with `fid` field
|
||||
- Sign with HMAC-SHA256
|
||||
- Verify + decode with expiry check
|
||||
- Separate signing keys for read vs write
|
||||
- `GetJwt(request)` — extract from `Authorization: Bearer` header or `jwt` query param
|
||||
|
||||
7.3. **TLS** (`tls.rs`):
|
||||
- Load server TLS cert/key for gRPC and HTTPS
|
||||
- Load client TLS for mutual TLS
|
||||
- Read from `security.toml` config (same format as Go's viper config)
|
||||
|
||||
7.4. **Tests:**
|
||||
- JWT sign → verify round-trip
|
||||
- JWT with wrong key → reject
|
||||
- JWT with expired token → reject
|
||||
- JWT fid mismatch → reject
|
||||
- Whitelist allow/deny
|
||||
|
||||
---
|
||||
|
||||
### Phase 8: Prometheus Metrics
|
||||
**Goal:** Export same metric names as Go for dashboard compatibility.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/stats/metrics.go` (volume server counters/gauges/histograms)
|
||||
|
||||
**Steps:**
|
||||
|
||||
8.1. Define all Prometheus metrics matching Go names:
|
||||
- `VolumeServerRequestCounter` (labels: method, status)
|
||||
- `VolumeServerRequestHistogram` (labels: method)
|
||||
- `VolumeServerInFlightRequestsGauge` (labels: method)
|
||||
- `VolumeServerInFlightUploadSize`
|
||||
- `VolumeServerInFlightDownloadSize`
|
||||
- `VolumeServerConcurrentUploadLimit`
|
||||
- `VolumeServerConcurrentDownloadLimit`
|
||||
- `VolumeServerHandlerCounter` (labels: type — UploadLimitCond, DownloadLimitCond)
|
||||
- Read/Write/Delete request counters
|
||||
|
||||
8.2. Metrics HTTP endpoint on `--metricsPort`.
|
||||
|
||||
8.3. Optional push-based metrics loop (`LoopPushingMetric`).
|
||||
|
||||
8.4. **Test:** Verify metric names and labels match Go output.
|
||||
|
||||
---
|
||||
|
||||
### Phase 9: HTTP Server & Handlers
|
||||
**Goal:** All HTTP endpoints with exact same behavior as Go.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/server/volume_server.go` → `src/server/volume_server.rs`
|
||||
- `weed/server/volume_server_handlers.go` → `src/server/http_handlers.rs`
|
||||
- `weed/server/volume_server_handlers_read.go` → `src/server/http_read.rs`
|
||||
- `weed/server/volume_server_handlers_write.go` → `src/server/http_write.rs`
|
||||
- `weed/server/volume_server_handlers_admin.go` → `src/server/http_admin.rs`
|
||||
- `weed/server/volume_server_handlers_helper.go` (URL parsing, proxy, JSON responses)
|
||||
- `weed/server/volume_server_handlers_ui.go` → `src/server/http_admin.rs`
|
||||
|
||||
**Steps:**
|
||||
|
||||
9.1. **URL path parsing** — from `handlers_helper.go`:
|
||||
- Parse `/<vid>,<fid>` and `/<vid>/<fid>` patterns
|
||||
- Extract volume ID, file ID, filename, ext
|
||||
|
||||
9.2. **Route dispatch** — from `privateStoreHandler` and `publicReadOnlyHandler`:
|
||||
- `GET /` → `GetOrHeadHandler`
|
||||
- `HEAD /` → `GetOrHeadHandler`
|
||||
- `POST /` → `PostHandler` (whitelist gated)
|
||||
- `PUT /` → `PostHandler` (whitelist gated)
|
||||
- `DELETE /` → `DeleteHandler` (whitelist gated)
|
||||
- `OPTIONS /` → CORS preflight
|
||||
- `GET /status` → JSON status
|
||||
- `GET /healthz` → health check
|
||||
- `GET /ui/index.html` → HTML UI page
|
||||
- Static resources (CSS/JS for UI)
|
||||
|
||||
9.3. **GET/HEAD handler** (`http_read.rs`) — from `handlers_read.go` (468 lines):
|
||||
- JWT read authorization check
|
||||
- Lookup needle by volume ID + needle ID + cookie
|
||||
- ETag / If-None-Match / If-Modified-Since conditional responses
|
||||
- Content-Type from stored MIME or filename extension
|
||||
- Content-Disposition header
|
||||
- Content-Encoding (gzip/zstd stored data)
|
||||
- Range request support (HTTP 206 Partial Content)
|
||||
- JPEG orientation fix (if configured)
|
||||
- Proxy to replica on local miss (readMode=proxy)
|
||||
- Redirect to replica (readMode=redirect)
|
||||
- Download tracking (in-flight size accounting)
|
||||
|
||||
9.4. **POST/PUT handler** (`http_write.rs`) — from `handlers_write.go` (170 lines):
|
||||
- JWT write authorization check
|
||||
- Multipart form parsing
|
||||
- Extract file data, filename, content type, TTL, last-modified
|
||||
- Optional gzip/zstd compression
|
||||
- Write needle to volume
|
||||
- Replicate to peers (same logic as Go's `DistributedOperation`)
|
||||
- Return JSON: {name, size, eTag, error}
|
||||
|
||||
9.5. **DELETE handler** — already in handlers.go:
|
||||
- JWT authorization
|
||||
- Delete from local volume
|
||||
- Replicate delete to peers
|
||||
- Return JSON result
|
||||
|
||||
9.6. **Admin handlers** (`http_admin.rs`):
|
||||
- `/status` → JSON with volumes, version, disk status
|
||||
- `/healthz` → 200 OK if serving
|
||||
- `/ui/index.html` → HTML dashboard
|
||||
|
||||
9.7. **Concurrency limiting** — from `handlers.go`:
|
||||
- Upload concurrency limit with `sync::Condvar` + timeout
|
||||
- Download concurrency limit with proxy fallback to replicas
|
||||
- HTTP 429 on timeout, 499 on client cancel
|
||||
- Replication traffic bypasses upload limits
|
||||
|
||||
9.8. **Public port** — if configured, separate listener with read-only routes (GET/HEAD/OPTIONS only).
|
||||
|
||||
9.9. **Request ID middleware** — generate unique request ID per request.
|
||||
|
||||
9.10. **Tests:**
|
||||
- Integration: start server → upload file via POST → GET it back → verify content
|
||||
- Integration: upload → DELETE → GET returns 404
|
||||
- Integration: conditional GET with ETag → 304
|
||||
- Integration: range request → 206 with correct bytes
|
||||
- Integration: exceed upload limit → 429
|
||||
- Integration: whitelist enforcement
|
||||
- Integration: JWT enforcement
|
||||
|
||||
---
|
||||
|
||||
### Phase 10: gRPC Service Implementation
|
||||
**Goal:** All 40 gRPC methods with exact logic.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/server/volume_grpc_admin.go` (380 lines)
|
||||
- `weed/server/volume_grpc_vacuum.go` (124 lines)
|
||||
- `weed/server/volume_grpc_copy.go` (636 lines)
|
||||
- `weed/server/volume_grpc_copy_incremental.go` (66 lines)
|
||||
- `weed/server/volume_grpc_read_write.go` (74 lines)
|
||||
- `weed/server/volume_grpc_batch_delete.go` (124 lines)
|
||||
- `weed/server/volume_grpc_tail.go` (140 lines)
|
||||
- `weed/server/volume_grpc_erasure_coding.go` (619 lines)
|
||||
- `weed/server/volume_grpc_scrub.go` (121 lines)
|
||||
- `weed/server/volume_grpc_tier_upload.go` (98 lines)
|
||||
- `weed/server/volume_grpc_tier_download.go` (85 lines)
|
||||
- `weed/server/volume_grpc_remote.go` (95 lines)
|
||||
- `weed/server/volume_grpc_query.go` (69 lines)
|
||||
- `weed/server/volume_grpc_state.go` (26 lines)
|
||||
- `weed/server/volume_grpc_read_all.go` (35 lines)
|
||||
- `weed/server/volume_grpc_client_to_master.go` (325 lines)
|
||||
|
||||
**Steps (grouped by functional area):**
|
||||
|
||||
10.1. **Implement `tonic::Service` for `VolumeServer`** — the generated trait from proto.
|
||||
|
||||
10.2. **Admin RPCs** (`grpc_admin.rs`):
|
||||
- `AllocateVolume` — create volume on appropriate disk location
|
||||
- `VolumeMount` / `VolumeUnmount` / `VolumeDelete`
|
||||
- `VolumeMarkReadonly` / `VolumeMarkWritable`
|
||||
- `VolumeConfigure` — change replication
|
||||
- `VolumeStatus` — return read-only, size, file counts
|
||||
- `VolumeServerStatus` — disk statuses, memory, version, DC, rack
|
||||
- `VolumeServerLeave` — deregister from master
|
||||
- `DeleteCollection`
|
||||
- `VolumeNeedleStatus` — get needle metadata by ID
|
||||
- `Ping` — latency measurement
|
||||
- `GetState` / `SetState` — maintenance mode
|
||||
|
||||
10.3. **Vacuum RPCs** (`grpc_vacuum.rs`):
|
||||
- `VacuumVolumeCheck` — return garbage ratio
|
||||
- `VacuumVolumeCompact` — stream progress (streaming response)
|
||||
- `VacuumVolumeCommit` — finalize compaction
|
||||
- `VacuumVolumeCleanup` — remove temp files
|
||||
|
||||
10.4. **Copy RPCs** (`grpc_copy.rs`):
|
||||
- `VolumeCopy` — stream .dat/.idx from source to create local copy
|
||||
- `VolumeSyncStatus` — return sync metadata
|
||||
- `VolumeIncrementalCopy` — stream .dat delta since timestamp (streaming)
|
||||
- `CopyFile` — generic file copy by extension (streaming)
|
||||
- `ReceiveFile` — receive streamed file (client streaming)
|
||||
- `ReadVolumeFileStatus` — return file timestamps and sizes
|
||||
|
||||
10.5. **Read/Write RPCs** (`grpc_read_write.rs`):
|
||||
- `ReadNeedleBlob` — raw needle blob read
|
||||
- `ReadNeedleMeta` — needle metadata
|
||||
- `WriteNeedleBlob` — raw needle blob write
|
||||
- `ReadAllNeedles` — stream all needles from volume(s) (streaming)
|
||||
|
||||
10.6. **Batch delete** (`grpc_batch_delete.rs`):
|
||||
- `BatchDelete` — delete multiple file IDs, return per-ID results
|
||||
|
||||
10.7. **Tail RPCs** (`grpc_tail.rs`):
|
||||
- `VolumeTailSender` — stream new needles since timestamp (streaming)
|
||||
- `VolumeTailReceiver` — connect to another volume server and tail its changes
|
||||
|
||||
10.8. **Erasure coding RPCs** (`grpc_erasure_coding.rs`):
|
||||
- `VolumeEcShardsGenerate` — generate EC shards from volume
|
||||
- `VolumeEcShardsRebuild` — rebuild missing shards
|
||||
- `VolumeEcShardsCopy` — copy shards from another server
|
||||
- `VolumeEcShardsDelete` — delete EC shards
|
||||
- `VolumeEcShardsMount` / `VolumeEcShardsUnmount`
|
||||
- `VolumeEcShardRead` — read from EC shard (streaming)
|
||||
- `VolumeEcBlobDelete` — mark blob deleted in EC volume
|
||||
- `VolumeEcShardsToVolume` — reconstruct volume from EC shards
|
||||
- `VolumeEcShardsInfo` — return shard metadata
|
||||
|
||||
10.9. **Scrub RPCs** (`grpc_scrub.rs`):
|
||||
- `ScrubVolume` — integrity check volumes (INDEX / FULL / LOCAL modes)
|
||||
- `ScrubEcVolume` — integrity check EC volumes
|
||||
|
||||
10.10. **Tier RPCs** (`grpc_tier.rs`):
|
||||
- `VolumeTierMoveDatToRemote` — upload to remote backend (streaming progress)
|
||||
- `VolumeTierMoveDatFromRemote` — download from remote (streaming progress)
|
||||
|
||||
10.11. **Remote storage** (`grpc_remote.rs`):
|
||||
- `FetchAndWriteNeedle` — fetch from remote storage, write locally, replicate
|
||||
|
||||
10.12. **Query** (`grpc_query.rs`):
|
||||
- `Query` — experimental CSV/JSON/Parquet select on stored data (streaming)
|
||||
|
||||
10.13. **Master heartbeat** (`grpc_client_to_master.rs`):
|
||||
- `heartbeat()` background task — periodic gRPC stream to master
|
||||
- Send: volume info, EC shard info, disk stats, has-no-space flags, deleted volumes
|
||||
- Receive: volume size limit, leader address, metrics config
|
||||
- Reconnect on failure with backoff
|
||||
- `StopHeartbeat()` for graceful shutdown
|
||||
|
||||
10.14. **Tests:**
|
||||
- Integration test per RPC: call via tonic client → verify response
|
||||
- Streaming RPCs: verify all chunks received
|
||||
- Error cases: invalid volume ID, non-existent volume, etc.
|
||||
- Heartbeat: mock master gRPC server, verify registration
|
||||
|
||||
---
|
||||
|
||||
### Phase 11: Startup, Lifecycle & Graceful Shutdown
|
||||
**Goal:** Full server startup matching Go's `runVolume()` and `startVolumeServer()`.
|
||||
|
||||
**Steps:**
|
||||
|
||||
11.1. **Startup sequence** (match `volume.go` exactly):
|
||||
1. Load security configuration from `security.toml`
|
||||
2. Start metrics server on metrics port
|
||||
3. Parse folder/max/minFreeSpace/diskType/tags
|
||||
4. Validate all directory writable
|
||||
5. Resolve IP, bind IP, public URL, gRPC port
|
||||
6. Create `VolumeServer` struct
|
||||
7. Check with master (initial handshake)
|
||||
8. Create `Store` (loads all existing volumes from disk)
|
||||
9. Create security `Guard`
|
||||
10. Register HTTP routes on admin mux
|
||||
11. Optionally register public mux
|
||||
12. Start gRPC server on gRPC port
|
||||
13. Start public HTTP server (if separated)
|
||||
14. Start cluster HTTP server (with optional TLS)
|
||||
15. Start heartbeat background task
|
||||
16. Start metrics push loop
|
||||
17. Register SIGHUP handler for config reload + new volume loading
|
||||
|
||||
11.2. **Graceful shutdown** (match Go exactly):
|
||||
1. On SIGINT/SIGTERM:
|
||||
2. Stop heartbeat (notify master we're leaving)
|
||||
3. Wait `preStopSeconds`
|
||||
4. Stop public HTTP server
|
||||
5. Stop cluster HTTP server
|
||||
6. Graceful stop gRPC server
|
||||
7. `volumeServer.Shutdown()` → `store.Close()` (flush all volumes)
|
||||
|
||||
11.3. **Reload** (SIGHUP):
|
||||
- Reload security config
|
||||
- Update whitelist
|
||||
- Load newly appeared volumes from disk
|
||||
|
||||
11.4. **Tests:**
|
||||
- Start server → send SIGTERM → verify clean shutdown
|
||||
- Start server → SIGHUP → verify config reloaded
|
||||
|
||||
---
|
||||
|
||||
### Phase 12: Integration & Cross-Compatibility Testing
|
||||
**Goal:** Rust volume server is a drop-in replacement for Go volume server.
|
||||
|
||||
**Steps:**
|
||||
|
||||
12.1. **Binary compatibility tests:**
|
||||
- Create volumes with Go volume server
|
||||
- Start Rust volume server on same data directory
|
||||
- Read all data → verify identical
|
||||
- Write new data with Rust → read with Go → verify
|
||||
|
||||
12.2. **API compatibility tests:**
|
||||
- Run same HTTP requests against both Go and Rust servers
|
||||
- Compare response bodies, headers, status codes
|
||||
- Test all gRPC RPCs against both
|
||||
|
||||
12.3. **Master interop test:**
|
||||
- Start Go master server
|
||||
- Register Rust volume server
|
||||
- Verify heartbeat works
|
||||
- Verify volume assignment works
|
||||
- Upload via filer → stored on Rust volume server → read back
|
||||
|
||||
12.4. **Performance benchmarks:**
|
||||
- Throughput: sequential writes, sequential reads
|
||||
- Latency: p50/p99 for read/write
|
||||
- Concurrency: parallel reads/writes
|
||||
- Compare Rust vs Go numbers
|
||||
|
||||
12.5. **Edge cases:**
|
||||
- Volume at max size
|
||||
- Disk full handling
|
||||
- Corrupt .dat file recovery
|
||||
- Network partition during replication
|
||||
- EC shard loss + rebuild
|
||||
|
||||
---
|
||||
|
||||
## Execution Order & Dependencies
|
||||
|
||||
```
|
||||
Phase 1 (Skeleton + CLI) ← no deps, start here
|
||||
↓
|
||||
Phase 2 (Storage types) ← needs Phase 1 (types used everywhere)
|
||||
↓
|
||||
Phase 3 (Volume struct) ← needs Phase 2
|
||||
↓
|
||||
Phase 4 (Store manager) ← needs Phase 3
|
||||
↓
|
||||
Phase 7 (Security) ← independent, can parallel with 3-4
|
||||
Phase 8 (Metrics) ← independent, can parallel with 3-4
|
||||
↓
|
||||
Phase 9 (HTTP server) ← needs Phase 4 + 7 + 8
|
||||
Phase 10 (gRPC server) ← needs Phase 4 + 7 + 8
|
||||
↓
|
||||
Phase 5 (Erasure coding) ← needs Phase 4, wire into Phase 10
|
||||
Phase 6 (Tiered storage) ← needs Phase 4, wire into Phase 10
|
||||
↓
|
||||
Phase 11 (Startup + shutdown) ← needs Phase 9 + 10
|
||||
↓
|
||||
Phase 12 (Integration tests) ← needs all above
|
||||
```
|
||||
|
||||
## Estimated Scope
|
||||
|
||||
| Phase | Estimated Rust Lines | Complexity |
|
||||
|---|---|---|
|
||||
| 1. Skeleton + CLI | ~400 | Low |
|
||||
| 2. Storage types | ~2,000 | High (binary compat critical) |
|
||||
| 3. Volume struct | ~2,500 | High |
|
||||
| 4. Store manager | ~1,000 | Medium |
|
||||
| 5. Erasure coding | ~3,000 | High |
|
||||
| 6. Tiered storage | ~1,500 | Medium |
|
||||
| 7. Security | ~500 | Medium |
|
||||
| 8. Metrics | ~300 | Low |
|
||||
| 9. HTTP server | ~2,000 | High |
|
||||
| 10. gRPC server | ~3,500 | High |
|
||||
| 11. Startup/shutdown | ~500 | Medium |
|
||||
| 12. Integration tests | ~2,000 | Medium |
|
||||
| **Total** | **~19,000** | |
|
||||
|
||||
## Critical Invariants to Preserve
|
||||
|
||||
1. **Binary format compatibility** — Rust must read/write `.dat`, `.idx`, `.vif`, `.ecX` files identically to Go. A single byte off = data loss.
|
||||
2. **gRPC wire compatibility** — Same proto, same field semantics. Go master must talk to Rust volume server seamlessly.
|
||||
3. **HTTP API compatibility** — Same URL patterns, same JSON response shapes, same headers, same status codes.
|
||||
4. **Replication protocol** — Write replication between Go and Rust volume servers must work bidirectionally.
|
||||
5. **Heartbeat protocol** — Rust volume server must register with Go master and maintain heartbeat.
|
||||
6. **CRC32 algorithm** — Must use IEEE polynomial (same as Go's `crc32.ChecksumIEEE`).
|
||||
7. **JWT compatibility** — Tokens signed by Go filer/master must be verifiable by Rust volume server and vice versa.
|
||||
@ -1,7 +0,0 @@
|
||||
package main
|
||||
|
||||
import "os"
|
||||
|
||||
func main() {
|
||||
os.Exit(Run(os.Args[1:]))
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
package main
|
||||
|
||||
import "os"
|
||||
|
||||
func main() {
|
||||
os.Exit(Run(os.Args[1:]))
|
||||
}
|
||||
@ -2,15 +2,13 @@ FROM ubuntu:22.04
|
||||
|
||||
LABEL author="Chris Lu"
|
||||
|
||||
# Use Azure's Ubuntu mirror — much faster than archive.ubuntu.com from GitHub-hosted runners,
|
||||
# which have been hanging long enough on Ign:/retry to trip the 10-min step timeout.
|
||||
# Use faster mirrors and optimize package installation
|
||||
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
|
||||
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
|
||||
# For testing purposes, running as root avoids permission complexities and dependency
|
||||
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
|
||||
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
|
||||
apt-get -o Acquire::http::Timeout=15 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=15 install -y \
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
curl \
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
FROM golang:1.25 AS builder
|
||||
FROM golang:1.24 AS builder
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y build-essential wget ca-certificates && \
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine as builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
@ -16,38 +16,9 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
# Rust volume server: use pre-built binary from CI when available (placed in
|
||||
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
|
||||
# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build
|
||||
# for non-native architectures.
|
||||
FROM alpine:3.23 as rust_builder
|
||||
ARG TARGETARCH
|
||||
ARG TAGS
|
||||
COPY weed-volume-prebuilt/ /prebuilt/
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
|
||||
WORKDIR /build/seaweed-volume
|
||||
RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
|
||||
echo "Using pre-built Rust binary for ${TARGETARCH}" && \
|
||||
cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
|
||||
elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
|
||||
if [ "$TAGS" = "5BytesOffset" ]; then \
|
||||
cargo build --release; \
|
||||
else \
|
||||
cargo build --release --no-default-features; \
|
||||
fi && \
|
||||
cp target/release/weed-volume /weed-volume; \
|
||||
else \
|
||||
echo "Skipping Rust build for $TARGETARCH (unsupported)" && \
|
||||
touch /weed-volume; \
|
||||
fi
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
# Copy Rust volume server binary (real binary on amd64/arm64, empty placeholder on other platforms)
|
||||
COPY --from=rust_builder /weed-volume /usr/bin/weed-volume
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
@ -56,8 +27,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
|
||||
# To disable: docker run -e GODEBUG=fips140=off ...
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@ -89,8 +59,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@ -7,8 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse curl su-exec && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@ -40,8 +39,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
FROM golang:1.25 AS builder
|
||||
FROM golang:1.24 AS builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ARG ROCKSDB_VERSION=v10.10.1
|
||||
ARG ROCKSDB_VERSION=v10.5.1
|
||||
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
|
||||
|
||||
# build RocksDB
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
FROM golang:1.25 AS builder
|
||||
FROM golang:1.24 AS builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ARG ROCKSDB_VERSION=v10.10.1
|
||||
ARG ROCKSDB_VERSION=v10.5.1
|
||||
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
|
||||
|
||||
# build RocksDB
|
||||
@ -34,8 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@ -68,8 +67,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@ -17,8 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@ -7,7 +7,6 @@
|
||||
|
||||
[master.maintenance]
|
||||
# periodically run these scripts are the same as running them from 'weed shell'
|
||||
# Scripts are skipped while an admin server is connected.
|
||||
scripts = """
|
||||
lock
|
||||
ec.encode -fullPercent=95 -quietFor=1h
|
||||
|
||||
@ -1,53 +0,0 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "steward",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "steward-key",
|
||||
"secretKey": "steward-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "le001",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "le001-key",
|
||||
"secretKey": "le001-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": [
|
||||
"Read"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,260 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Integration test: git clone & pull on a SeaweedFS FUSE mount.
|
||||
#
|
||||
# Verifies that the mount correctly supports git's file operations by:
|
||||
# 1. Creating a bare repo on the mount (acts as a remote)
|
||||
# 2. Cloning it, making commits, and pushing back to the mount
|
||||
# 3. Cloning from the mount into a working directory also on the mount
|
||||
# 4. Pushing additional commits to the bare repo
|
||||
# 5. Checking out an older revision in the on-mount clone
|
||||
# 6. Running git pull to fast-forward with real changes
|
||||
# 7. Verifying file content integrity at each step
|
||||
#
|
||||
# Usage:
|
||||
# bash test-git-on-mount.sh /path/to/mount/point
|
||||
#
|
||||
# The mount must already be running. All test artifacts are created under
|
||||
# <mount>/git-test-<pid> and cleaned up on exit (unless TEST_KEEP=1).
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
MOUNT_DIR="${1:?Usage: $0 <mount-dir>}"
|
||||
TEST_DIR="$MOUNT_DIR/git-test-$$"
|
||||
LOCAL_DIR=$(mktemp -d)
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
cleanup() {
|
||||
if [[ "${TEST_KEEP:-}" == "1" ]]; then
|
||||
echo "TEST_KEEP=1 — leaving artifacts:"
|
||||
echo " mount: $TEST_DIR"
|
||||
echo " local: $LOCAL_DIR"
|
||||
else
|
||||
rm -rf "$TEST_DIR" 2>/dev/null || true
|
||||
rm -rf "$LOCAL_DIR" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# --- helpers ---------------------------------------------------------------
|
||||
|
||||
pass() { PASS=$((PASS + 1)); echo " PASS: $1"; }
|
||||
fail() { FAIL=$((FAIL + 1)); echo " FAIL: $1"; }
|
||||
|
||||
assert_file_contains() {
|
||||
local file=$1 expected=$2 label=$3
|
||||
if [[ -f "$file" ]] && grep -qF "$expected" "$file" 2>/dev/null; then
|
||||
pass "$label"
|
||||
else
|
||||
fail "$label (expected '$expected' in $file)"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_file_exists() {
|
||||
local file=$1 label=$2
|
||||
if [[ -f "$file" ]]; then
|
||||
pass "$label"
|
||||
else
|
||||
fail "$label ($file not found)"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_file_not_exists() {
|
||||
local file=$1 label=$2
|
||||
if [[ ! -f "$file" ]]; then
|
||||
pass "$label"
|
||||
else
|
||||
fail "$label ($file should not exist)"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_eq() {
|
||||
local actual=$1 expected=$2 label=$3
|
||||
if [[ "$actual" == "$expected" ]]; then
|
||||
pass "$label"
|
||||
else
|
||||
fail "$label (expected '$expected', got '$actual')"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- setup -----------------------------------------------------------------
|
||||
|
||||
echo "========================================"
|
||||
echo " Git-on-mount integration test"
|
||||
echo "========================================"
|
||||
echo "Mount: $MOUNT_DIR"
|
||||
echo "Test: $TEST_DIR"
|
||||
echo "Local: $LOCAL_DIR"
|
||||
echo ""
|
||||
|
||||
if ! mountpoint -q "$MOUNT_DIR" 2>/dev/null && [[ ! -d "$MOUNT_DIR" ]]; then
|
||||
echo "ERROR: $MOUNT_DIR is not a valid directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$TEST_DIR"
|
||||
|
||||
# --- Phase 1: Create bare repo on mount -----------------------------------
|
||||
|
||||
echo "--- Phase 1: Create bare repo on mount ---"
|
||||
|
||||
BARE_REPO="$TEST_DIR/repo.git"
|
||||
git init --bare "$BARE_REPO" >/dev/null 2>&1
|
||||
pass "bare repo created on mount"
|
||||
|
||||
# --- Phase 2: Clone locally, make initial commits, push -------------------
|
||||
|
||||
echo "--- Phase 2: Clone locally, make initial commits, push ---"
|
||||
|
||||
LOCAL_CLONE="$LOCAL_DIR/clone1"
|
||||
git clone "$BARE_REPO" "$LOCAL_CLONE" >/dev/null 2>&1
|
||||
cd "$LOCAL_CLONE"
|
||||
git config user.email "test@seaweedfs.test"
|
||||
git config user.name "SeaweedFS Test"
|
||||
|
||||
# Commit 1: initial files
|
||||
echo "hello world" > README.md
|
||||
mkdir -p src
|
||||
echo 'package main; import "fmt"; func main() { fmt.Println("v1") }' > src/main.go
|
||||
git add -A && git commit -m "initial commit" >/dev/null 2>&1
|
||||
COMMIT1=$(git rev-parse HEAD)
|
||||
|
||||
# Commit 2: add more files
|
||||
mkdir -p data
|
||||
for i in $(seq 1 20); do
|
||||
printf "file-%03d: %s\n" "$i" "$(head -c 64 /dev/urandom | base64)" > "data/file-$(printf '%03d' $i).txt"
|
||||
done
|
||||
git add -A && git commit -m "add data files" >/dev/null 2>&1
|
||||
COMMIT2=$(git rev-parse HEAD)
|
||||
|
||||
# Commit 3: modify and add
|
||||
echo 'package main; import "fmt"; func main() { fmt.Println("v2") }' > src/main.go
|
||||
echo "# Updated readme" >> README.md
|
||||
mkdir -p docs
|
||||
echo "documentation content" > docs/guide.md
|
||||
git add -A && git commit -m "update src and add docs" >/dev/null 2>&1
|
||||
COMMIT3=$(git rev-parse HEAD)
|
||||
|
||||
git push origin master >/dev/null 2>&1 || git push origin main >/dev/null 2>&1
|
||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
pass "3 commits pushed to mount bare repo (branch=$BRANCH)"
|
||||
|
||||
# --- Phase 3: Clone from mount bare repo to mount working dir -------------
|
||||
|
||||
echo "--- Phase 3: Clone from mount bare repo to on-mount working dir ---"
|
||||
|
||||
MOUNT_CLONE="$TEST_DIR/working"
|
||||
git clone "$BARE_REPO" "$MOUNT_CLONE" >/dev/null 2>&1
|
||||
|
||||
# Verify clone integrity
|
||||
assert_file_exists "$MOUNT_CLONE/README.md" "README.md exists after clone"
|
||||
assert_file_contains "$MOUNT_CLONE/README.md" "# Updated readme" "README.md has latest content"
|
||||
assert_file_contains "$MOUNT_CLONE/src/main.go" 'v2' "src/main.go has v2"
|
||||
assert_file_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md exists"
|
||||
assert_file_exists "$MOUNT_CLONE/data/file-001.txt" "data files exist"
|
||||
assert_file_exists "$MOUNT_CLONE/data/file-020.txt" "data/file-020.txt exists"
|
||||
|
||||
CLONE_HEAD=$(cd "$MOUNT_CLONE" && git rev-parse HEAD)
|
||||
assert_eq "$CLONE_HEAD" "$COMMIT3" "on-mount clone HEAD matches commit 3"
|
||||
|
||||
# Count files
|
||||
FILE_COUNT=$(find "$MOUNT_CLONE/data" -name '*.txt' | wc -l | tr -d ' ')
|
||||
assert_eq "$FILE_COUNT" "20" "data/ has 20 files"
|
||||
|
||||
# --- Phase 4: Push more commits from local clone --------------------------
|
||||
|
||||
echo "--- Phase 4: Push more commits from local clone ---"
|
||||
|
||||
cd "$LOCAL_CLONE"
|
||||
|
||||
# Commit 4: larger changes
|
||||
for i in $(seq 21 50); do
|
||||
printf "file-%03d: %s\n" "$i" "$(head -c 128 /dev/urandom | base64)" > "data/file-$(printf '%03d' $i).txt"
|
||||
done
|
||||
echo 'package main; import "fmt"; func main() { fmt.Println("v3") }' > src/main.go
|
||||
git add -A && git commit -m "expand data and update to v3" >/dev/null 2>&1
|
||||
COMMIT4=$(git rev-parse HEAD)
|
||||
|
||||
# Commit 5: rename and delete
|
||||
git mv docs/guide.md docs/manual.md
|
||||
git rm data/file-001.txt >/dev/null 2>&1
|
||||
git commit -m "rename guide, remove file-001" >/dev/null 2>&1
|
||||
COMMIT5=$(git rev-parse HEAD)
|
||||
|
||||
git push origin "$BRANCH" >/dev/null 2>&1
|
||||
pass "2 more commits pushed (5 total)"
|
||||
|
||||
# --- Phase 5: Checkout older revision in on-mount clone -------------------
|
||||
|
||||
echo "--- Phase 5: Checkout older revision in on-mount clone ---"
|
||||
|
||||
cd "$MOUNT_CLONE"
|
||||
git checkout "$COMMIT2" >/dev/null 2>&1
|
||||
|
||||
# Verify we're at commit 2 state
|
||||
DETACHED_HEAD=$(git rev-parse HEAD)
|
||||
assert_eq "$DETACHED_HEAD" "$COMMIT2" "on-mount clone at commit 2 (detached)"
|
||||
assert_file_not_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md not in commit 2"
|
||||
assert_file_contains "$MOUNT_CLONE/src/main.go" 'v1' "src/main.go has v1 at commit 2"
|
||||
|
||||
# --- Phase 6: Return to branch and pull -----------------------------------
|
||||
|
||||
echo "--- Phase 6: Return to branch and pull with real changes ---"
|
||||
|
||||
cd "$MOUNT_CLONE"
|
||||
git checkout "$BRANCH" >/dev/null 2>&1
|
||||
# At this point the on-mount clone is at commit 3, remote is at commit 5
|
||||
OLD_HEAD=$(git rev-parse HEAD)
|
||||
assert_eq "$OLD_HEAD" "$COMMIT3" "on-mount clone at commit 3 before pull"
|
||||
|
||||
git pull >/dev/null 2>&1
|
||||
NEW_HEAD=$(git rev-parse HEAD)
|
||||
assert_eq "$NEW_HEAD" "$COMMIT5" "HEAD matches commit 5 after pull"
|
||||
|
||||
# Verify commit 5 state
|
||||
assert_file_contains "$MOUNT_CLONE/src/main.go" 'v3' "src/main.go has v3 after pull"
|
||||
assert_file_exists "$MOUNT_CLONE/docs/manual.md" "docs/manual.md exists (renamed)"
|
||||
assert_file_not_exists "$MOUNT_CLONE/docs/guide.md" "docs/guide.md gone (renamed)"
|
||||
assert_file_not_exists "$MOUNT_CLONE/data/file-001.txt" "data/file-001.txt removed"
|
||||
assert_file_exists "$MOUNT_CLONE/data/file-050.txt" "data/file-050.txt exists"
|
||||
|
||||
FINAL_COUNT=$(find "$MOUNT_CLONE/data" -name '*.txt' | wc -l | tr -d ' ')
|
||||
assert_eq "$FINAL_COUNT" "49" "data/ has 49 files (50 added, 1 removed)"
|
||||
|
||||
# --- Phase 7: Verify git log integrity -----------------------------------
|
||||
|
||||
echo "--- Phase 7: Verify git log integrity ---"
|
||||
|
||||
cd "$MOUNT_CLONE"
|
||||
GIT_LOG=$(git log --format=%s)
|
||||
LOG_COUNT=$(echo "$GIT_LOG" | wc -l | tr -d ' ')
|
||||
assert_eq "$LOG_COUNT" "5" "git log shows 5 commits"
|
||||
|
||||
# Verify commit messages
|
||||
echo "$GIT_LOG" | grep -qF "initial commit" && pass "commit 1 message in log" || fail "commit 1 message missing"
|
||||
echo "$GIT_LOG" | grep -qF "expand data" && pass "commit 4 message in log" || fail "commit 4 message missing"
|
||||
echo "$GIT_LOG" | grep -qF "rename guide" && pass "commit 5 message in log" || fail "commit 5 message missing"
|
||||
|
||||
# --- Phase 8: Verify git status is clean ----------------------------------
|
||||
|
||||
echo "--- Phase 8: Verify git status is clean ---"
|
||||
|
||||
cd "$MOUNT_CLONE"
|
||||
STATUS=$(git status --porcelain)
|
||||
if [[ -z "$STATUS" ]]; then
|
||||
pass "git status is clean"
|
||||
else
|
||||
fail "git status has changes: $STATUS"
|
||||
fi
|
||||
|
||||
# --- Results ---------------------------------------------------------------
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo " Results: $PASS passed, $FAIL failed"
|
||||
echo "========================================"
|
||||
|
||||
if [[ "$FAIL" -gt 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
@ -20,38 +20,30 @@ if [ "$(id -u)" = "0" ]; then
|
||||
|
||||
DATA_UID=$(stat -c '%u' /data 2>/dev/null)
|
||||
DATA_GID=$(stat -c '%g' /data 2>/dev/null)
|
||||
|
||||
# Only run chown -R if ownership doesn't already match (avoids expensive
|
||||
# recursive chown on subsequent starts, and is a no-op on OpenShift when
|
||||
# fsGroup has already set correct ownership on the PVC).
|
||||
if [ "$DATA_UID" != "$SEAWEED_UID" ] || [ "$DATA_GID" != "$SEAWEED_GID" ]; then
|
||||
|
||||
# Only run chown -R if ownership doesn't match (much faster for subsequent starts)
|
||||
echo "Fixing /data ownership for seaweed user (uid=$SEAWEED_UID, gid=$SEAWEED_GID)"
|
||||
if ! chown -R seaweed:seaweed /data; then
|
||||
echo "Warning: Failed to change ownership of /data. This may cause permission errors." >&2
|
||||
echo "If /data is read-only or has mount issues, the application may fail to start." >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
# Use su-exec to drop privileges and run as seaweed user
|
||||
exec su-exec seaweed "$0" "$@"
|
||||
fi
|
||||
|
||||
isArgPassed() {
|
||||
# Match both `-flag` and `--flag` (and their `=value` forms): the Go fla9
|
||||
# library accepts both, and users may pick either form on the CLI.
|
||||
arg="$1"
|
||||
argWithEqualSign="$1="
|
||||
argDouble="-$1"
|
||||
argDoubleWithEqualSign="-$1="
|
||||
shift
|
||||
while [ $# -gt 0 ]; do
|
||||
passedArg="$1"
|
||||
shift
|
||||
case $passedArg in
|
||||
"$arg"|"$argDouble")
|
||||
"$arg")
|
||||
return 0
|
||||
;;
|
||||
"$argWithEqualSign"*|"$argDoubleWithEqualSign"*)
|
||||
"$argWithEqualSign"*)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
@ -62,7 +54,7 @@ isArgPassed() {
|
||||
case "$1" in
|
||||
|
||||
'master')
|
||||
ARGS="-mdir=/data -volumeSizeLimitMB=1024"
|
||||
ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true master $ARGS $@
|
||||
;;
|
||||
@ -76,38 +68,15 @@ case "$1" in
|
||||
exec /usr/bin/weed -logtostderr=true volume $ARGS $@
|
||||
;;
|
||||
|
||||
'volume-rust')
|
||||
ARGS="-dir /data -max 0"
|
||||
if isArgPassed "-max" "$@"; then
|
||||
ARGS="-dir /data"
|
||||
fi
|
||||
shift
|
||||
if [ ! -s /usr/bin/weed-volume ]; then
|
||||
echo "Error: Rust volume server is not available on this platform ($(uname -m))." >&2
|
||||
echo "Use 'volume' for the Go volume server instead." >&2
|
||||
exit 1
|
||||
fi
|
||||
exec /usr/bin/weed-volume $ARGS $@
|
||||
;;
|
||||
|
||||
'server')
|
||||
ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
|
||||
ARGS="-dir=/data -volume.max=0 -master.volumePreallocate -master.volumeSizeLimitMB=1024"
|
||||
if isArgPassed "-volume.max" "$@"; then
|
||||
ARGS="-dir=/data -master.volumeSizeLimitMB=1024"
|
||||
ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
|
||||
fi
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true server $ARGS $@
|
||||
;;
|
||||
|
||||
'mini')
|
||||
ARGS="-dir=/data"
|
||||
if isArgPassed "-dir" "$@"; then
|
||||
ARGS=""
|
||||
fi
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true mini $ARGS $@
|
||||
;;
|
||||
|
||||
'filer')
|
||||
ARGS=""
|
||||
shift
|
||||
|
||||
@ -33,7 +33,7 @@ isArgPassed() {
|
||||
case "$1" in
|
||||
|
||||
'master')
|
||||
ARGS="-mdir=/data -volumeSizeLimitMB=1024"
|
||||
ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true master $ARGS "$@"
|
||||
;;
|
||||
@ -48,9 +48,9 @@ case "$1" in
|
||||
;;
|
||||
|
||||
'server')
|
||||
ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
|
||||
ARGS="-dir=/data -volume.max=0 -master.volumePreallocate -master.volumeSizeLimitMB=1024"
|
||||
if isArgPassed "-volume.max" "$@"; then
|
||||
ARGS="-dir=/data -master.volumeSizeLimitMB=1024"
|
||||
ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
|
||||
fi
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true server $ARGS "$@"
|
||||
|
||||
@ -42,12 +42,3 @@ services:
|
||||
- master
|
||||
- volume
|
||||
- filer
|
||||
admin:
|
||||
image: chrislusf/seaweedfs:dev # use a remote dev image
|
||||
ports:
|
||||
- 23646:23646
|
||||
command: 'admin -master=master:9333'
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
- filer
|
||||
|
||||
385
go.mod
385
go.mod
@ -1,11 +1,11 @@
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.25.0
|
||||
go 1.24.9
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.50.2
|
||||
cloud.google.com/go/storage v1.62.1
|
||||
cloud.google.com/go/pubsub v1.50.1
|
||||
cloud.google.com/go/storage v1.59.2
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@ -15,6 +15,7 @@ require (
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/eapache/go-resiliency v1.6.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
|
||||
@ -25,30 +26,30 @@ require (
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.16.0
|
||||
github.com/go-redsync/redsync/v4 v4.15.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/go-zookeeper/zk v1.0.4 // indirect
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/golang/protobuf v1.5.4
|
||||
github.com/golang/snappy v1.0.0
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/jackc/pgx/v5 v5.8.0
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.6
|
||||
github.com/klauspost/reedsolomon v1.14.0
|
||||
github.com/klauspost/compress v1.18.3
|
||||
github.com/klauspost/reedsolomon v1.13.0
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.7
|
||||
github.com/linxGnu/grocksdb v1.10.3
|
||||
github.com/mailru/easyjson v0.9.1 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
@ -60,14 +61,14 @@ require (
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/procfs v0.20.1
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.2 // indirect
|
||||
github.com/prometheus/procfs v0.19.2
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.8
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/seaweedfs/raft v1.1.6
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/viper v1.21.0
|
||||
@ -82,84 +83,85 @@ require (
|
||||
github.com/valyala/bytebufferpool v1.0.0
|
||||
github.com/viant/ptrie v1.0.1
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.10
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.etcd.io/etcd/client/v3 v3.6.6
|
||||
go.mongodb.org/mongo-driver v1.17.6
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.45.0
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.45.0
|
||||
golang.org/x/crypto v0.51.0
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/image v0.39.0
|
||||
golang.org/x/net v0.54.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.44.0
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/tools v0.44.0 // indirect
|
||||
gocloud.dev v0.44.0
|
||||
gocloud.dev/pubsub/natspubsub v0.44.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0
|
||||
golang.org/x/crypto v0.47.0
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
|
||||
golang.org/x/image v0.35.0
|
||||
golang.org/x/net v0.49.0
|
||||
golang.org/x/oauth2 v0.34.0
|
||||
golang.org/x/sys v0.40.0
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
golang.org/x/tools v0.40.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.278.0
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/api v0.258.0
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
|
||||
google.golang.org/grpc v1.78.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/mathutil v1.7.1
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.49.1
|
||||
modernc.org/sqlite v1.44.3
|
||||
modernc.org/strutil v1.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.31.0
|
||||
cloud.google.com/go/kms v1.23.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/a-h/templ v0.3.1001
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0
|
||||
github.com/apache/iceberg-go v0.5.0
|
||||
github.com/a-h/templ v0.3.977
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0
|
||||
github.com/apache/iceberg-go v0.4.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/cognusion/imaging v1.0.3
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
github.com/go-git/go-billy/v5 v5.9.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.13
|
||||
github.com/getsentry/sentry-go v0.40.0
|
||||
github.com/gin-contrib/sessions v1.0.4
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.12
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.17.0
|
||||
github.com/lib/pq v1.11.1
|
||||
github.com/linkedin/goavro/v2 v2.14.1
|
||||
github.com/mattn/go-sqlite3 v1.14.33
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.30.1
|
||||
github.com/parquet-go/parquet-go v0.26.4
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/rabbitmq/amqp091-go v1.11.0
|
||||
github.com/rclone/rclone v1.74.1
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.72.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.19.0
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3
|
||||
github.com/shirou/gopsutil/v4 v4.26.3
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.2
|
||||
github.com/testcontainers/testcontainers-go v0.40.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1
|
||||
github.com/shirou/gopsutil/v4 v4.26.1
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.1
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/willscott/go-nfs v0.0.4
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/sync v0.19.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0
|
||||
)
|
||||
@ -170,227 +172,204 @@ require (
|
||||
atomicgo.dev/cursor v0.2.0 // indirect
|
||||
atomicgo.dev/keyboard v0.2.9 // indirect
|
||||
atomicgo.dev/schedule v0.1.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.9.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
cloud.google.com/go/longrunning v0.7.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/adrg/xdg v0.5.3 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
|
||||
github.com/apache/thrift v0.23.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.4.1 // indirect
|
||||
github.com/apache/thrift v0.22.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
github.com/boombuler/barcode v1.1.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.2 // indirect
|
||||
github.com/bufbuild/protocompile v0.14.1 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
||||
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 // indirect
|
||||
github.com/containerd/console v1.0.5 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v1.0.0-rc.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/dave/dst v0.27.2 // indirect
|
||||
github.com/diskfs/go-diskfs v1.7.0 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/docker v28.5.2+incompatible // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dromara/dongle v1.0.1 // indirect
|
||||
github.com/gin-gonic/gin v1.11.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/gookit/color v1.5.4 // indirect
|
||||
github.com/gopherjs/gopherjs v1.17.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/hamba/avro/v2 v2.31.0 // indirect
|
||||
github.com/hamba/avro/v2 v2.30.0 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jaegertracing/jaeger v1.47.0 // indirect
|
||||
github.com/jhump/protoreflect/v2 v2.0.0-beta.1 // indirect
|
||||
github.com/jtolds/gls v4.20.0+incompatible // indirect
|
||||
github.com/klauspost/asmfmt v1.3.2 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lib/pq v1.11.1 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.1.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.0 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/openzipkin/zipkin-go v0.4.3 // indirect
|
||||
github.com/parquet-go/bitpack v1.0.0 // indirect
|
||||
github.com/parquet-go/jsonlite v1.0.0 // indirect
|
||||
github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
|
||||
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.82 // indirect
|
||||
github.com/pterm/pterm v0.12.81 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.2 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
github.com/smarty/assertions v1.15.0 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/substrait-io/substrait v0.81.0 // indirect
|
||||
github.com/substrait-io/substrait-go/v7 v7.4.0 // indirect
|
||||
github.com/substrait-io/substrait-protobuf/go v0.81.0 // indirect
|
||||
github.com/twpayne/go-geom v1.6.1 // indirect
|
||||
github.com/twpayne/go-kml/v3 v3.2.1 // indirect
|
||||
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
|
||||
github.com/substrait-io/substrait v0.69.0 // indirect
|
||||
github.com/substrait-io/substrait-go/v4 v4.4.0 // indirect
|
||||
github.com/substrait-io/substrait-protobuf/go v0.71.0 // indirect
|
||||
github.com/twpayne/go-geom v1.4.1 // indirect
|
||||
github.com/twpayne/go-kml v1.5.2 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.35.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
golang.org/x/mod v0.31.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc // indirect
|
||||
gonum.org/v1/gonum v0.16.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cel.dev/expr v0.24.0 // indirect
|
||||
cloud.google.com/go/auth v0.17.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.7.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
filippo.io/edwards25519 v1.1.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.2 // indirect
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
|
||||
github.com/ProtonMail/go-crypto v1.4.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.3.0 // indirect
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.11.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.10.3 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
|
||||
github.com/aws/smithy-go v1.24.0
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/bytedance/sonic v1.14.0 // indirect
|
||||
github.com/bytedance/sonic/loader v0.3.0 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/ebitengine/purego v0.9.1 // indirect
|
||||
github.com/elastic/gosigar v0.14.3 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.3 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.6 // indirect
|
||||
github.com/go-openapi/errors v0.22.4 // indirect
|
||||
github.com/go-openapi/strfmt v0.25.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.28.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.16.5 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
|
||||
github.com/gorilla/context v1.1.2 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
github.com/gorilla/sessions v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
@ -400,11 +379,13 @@ require (
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
|
||||
github.com/henrybear327/go-proton-api v1.0.0 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
|
||||
github.com/jonboulle/clockwork v0.5.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
@ -418,16 +399,16 @@ require (
|
||||
github.com/lanrat/extsort v1.4.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lpar/date v1.0.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.22 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nats.go v1.48.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.12 // indirect
|
||||
github.com/nats-io/nats.go v1.43.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.11 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
@ -435,13 +416,13 @@ require (
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.5 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.3 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.26
|
||||
github.com/pierrec/lz4/v4 v4.1.22
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
|
||||
@ -452,12 +433,12 @@ require (
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/rfjakob/eme v1.2.0 // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/samber/lo v1.52.0 // indirect
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
@ -465,20 +446,22 @@ require (
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.6.3 // indirect
|
||||
github.com/tinylib/msgp v1.5.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||
github.com/ugorji/go/codec v1.3.0 // indirect
|
||||
github.com/unknwon/goconfig v1.0.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
@ -486,40 +469,36 @@ require (
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.6 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.43.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/term v0.39.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
modernc.org/libc v1.67.6 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a // indirect
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
|
||||
storj.io/infectious v0.0.2 // indirect
|
||||
storj.io/picobuf v0.0.4 // indirect
|
||||
storj.io/uplink v1.14.0 // indirect
|
||||
storj.io/uplink v1.13.1 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
|
||||
// apache/thrift v0.23.0 uses math.MaxUint32 as an untyped int constant in
|
||||
// lib/go/thrift/framed_transport.go, which overflows int on 32-bit GOARCHes
|
||||
// (e.g. openbsd/arm, linux/arm). Pin to v0.22.0 until upstream fixes it.
|
||||
replace github.com/apache/thrift => github.com/apache/thrift v0.22.0
|
||||
|
||||
275
install.sh
275
install.sh
@ -1,275 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# SeaweedFS Installer
|
||||
# Downloads Go and/or Rust binaries from GitHub releases.
|
||||
#
|
||||
# Usage:
|
||||
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
|
||||
# curl -fsSL ... | bash -s -- --version v3.93 --dir /usr/local/bin
|
||||
#
|
||||
# Options:
|
||||
# --component COMP Which binary to install: weed, volume-rust, all (default: weed)
|
||||
# --version VER Release version tag (default: latest)
|
||||
# --large-disk Use large disk variant (5-byte offset, 8TB max volume)
|
||||
# --dir DIR Installation directory (default: /usr/local/bin)
|
||||
# --help Show this help message
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO="seaweedfs/seaweedfs"
|
||||
COMPONENT="weed"
|
||||
VERSION=""
|
||||
LARGE_DISK=false
|
||||
INSTALL_DIR="/usr/local/bin"
|
||||
|
||||
# Colors (if terminal supports them)
|
||||
if [ -t 1 ]; then
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
else
|
||||
RED='' GREEN='' YELLOW='' BLUE='' NC=''
|
||||
fi
|
||||
|
||||
info() { echo -e "${BLUE}[info]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[ok]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[warn]${NC} $*"; }
|
||||
error() { echo -e "${RED}[error]${NC} $*" >&2; exit 1; }
|
||||
|
||||
usage() {
|
||||
sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//'
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--component) COMPONENT="$2"; shift 2 ;;
|
||||
--version) VERSION="$2"; shift 2 ;;
|
||||
--large-disk) LARGE_DISK=true; shift ;;
|
||||
--dir) INSTALL_DIR="$2"; shift 2 ;;
|
||||
--help|-h) usage ;;
|
||||
*) error "Unknown option: $1. Use --help for usage." ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Detect OS and architecture
|
||||
detect_platform() {
|
||||
local os arch
|
||||
|
||||
case "$(uname -s)" in
|
||||
Linux*) os="linux" ;;
|
||||
Darwin*) os="darwin" ;;
|
||||
MINGW*|MSYS*|CYGWIN*) os="windows" ;;
|
||||
FreeBSD*) os="freebsd" ;;
|
||||
*) error "Unsupported OS: $(uname -s)" ;;
|
||||
esac
|
||||
|
||||
case "$(uname -m)" in
|
||||
x86_64|amd64) arch="amd64" ;;
|
||||
aarch64|arm64) arch="arm64" ;;
|
||||
armv7l|armv6l) arch="arm" ;;
|
||||
*) error "Unsupported architecture: $(uname -m)" ;;
|
||||
esac
|
||||
|
||||
echo "${os}" "${arch}"
|
||||
}
|
||||
|
||||
# Get latest release tag from GitHub API
|
||||
get_latest_version() {
|
||||
local url="https://api.github.com/repos/${REPO}/releases/latest"
|
||||
if command -v curl &>/dev/null; then
|
||||
curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
elif command -v wget &>/dev/null; then
|
||||
wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
else
|
||||
error "Neither curl nor wget found. Please install one."
|
||||
fi
|
||||
}
|
||||
|
||||
# Download a file
|
||||
download() {
|
||||
local url="$1" dest="$2"
|
||||
info "Downloading ${url}"
|
||||
if command -v curl &>/dev/null; then
|
||||
curl -fsSL -o "$dest" "$url"
|
||||
elif command -v wget &>/dev/null; then
|
||||
wget -qO "$dest" "$url"
|
||||
fi
|
||||
}
|
||||
|
||||
# Build Go weed binary asset name
|
||||
go_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
local suffix="${os}_${arch}"
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
suffix="${suffix}_large_disk"
|
||||
fi
|
||||
echo "${suffix}.tar.gz"
|
||||
}
|
||||
|
||||
# Build Rust volume server asset name
|
||||
rust_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
local prefix="weed-volume"
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
prefix="weed-volume_large_disk"
|
||||
else
|
||||
prefix="weed-volume"
|
||||
fi
|
||||
local suffix="${os}_${arch}"
|
||||
if [ "$os" = "windows" ]; then
|
||||
echo "${prefix}_${suffix}.zip"
|
||||
else
|
||||
echo "${prefix}_${suffix}.tar.gz"
|
||||
fi
|
||||
}
|
||||
|
||||
# Install a single component
|
||||
install_component() {
|
||||
local component="$1" os="$2" arch="$3"
|
||||
local asset_name download_url tmpdir
|
||||
|
||||
tmpdir="$(mktemp -d)"
|
||||
trap "rm -rf '$tmpdir'" EXIT
|
||||
|
||||
case "$component" in
|
||||
weed)
|
||||
asset_name="$(go_asset_name "$os" "$arch")"
|
||||
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
|
||||
download "$download_url" "${tmpdir}/${asset_name}"
|
||||
|
||||
info "Extracting ${asset_name}..."
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
|
||||
# The Go release action puts the binary inside a directory
|
||||
local weed_bin
|
||||
weed_bin="$(find "$tmpdir" -name 'weed' -type f | head -1)"
|
||||
if [ -z "$weed_bin" ]; then
|
||||
weed_bin="$(find "$tmpdir" -name 'weed.exe' -type f | head -1)"
|
||||
fi
|
||||
if [ -z "$weed_bin" ]; then
|
||||
error "Could not find weed binary in archive"
|
||||
fi
|
||||
|
||||
chmod +x "$weed_bin"
|
||||
install_binary "$weed_bin" "weed"
|
||||
ok "Installed weed to ${INSTALL_DIR}/weed"
|
||||
;;
|
||||
|
||||
volume-rust)
|
||||
# Check platform support for Rust volume server
|
||||
case "$os" in
|
||||
linux|darwin|windows) ;;
|
||||
*) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;;
|
||||
esac
|
||||
case "$arch" in
|
||||
amd64|arm64) ;;
|
||||
*) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;;
|
||||
esac
|
||||
|
||||
asset_name="$(rust_asset_name "$os" "$arch")"
|
||||
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
|
||||
download "$download_url" "${tmpdir}/${asset_name}"
|
||||
|
||||
info "Extracting ${asset_name}..."
|
||||
if [ "$os" = "windows" ]; then
|
||||
unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir"
|
||||
else
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
fi
|
||||
|
||||
local rust_bin
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
rust_bin="$(find "$tmpdir" -name 'weed-volume-large-disk*' -type f | head -1)"
|
||||
else
|
||||
rust_bin="$(find "$tmpdir" -name 'weed-volume-normal*' -type f | head -1)"
|
||||
fi
|
||||
if [ -z "$rust_bin" ]; then
|
||||
rust_bin="$(find "$tmpdir" -name 'weed-volume*' -type f | head -1)"
|
||||
fi
|
||||
if [ -z "$rust_bin" ]; then
|
||||
error "Could not find weed-volume binary in archive"
|
||||
fi
|
||||
|
||||
chmod +x "$rust_bin"
|
||||
local dest_name="weed-volume"
|
||||
if [ "$os" = "windows" ]; then
|
||||
dest_name="weed-volume.exe"
|
||||
fi
|
||||
install_binary "$rust_bin" "$dest_name"
|
||||
ok "Installed weed-volume to ${INSTALL_DIR}/${dest_name}"
|
||||
;;
|
||||
|
||||
*)
|
||||
error "Unknown component: ${component}. Use: weed, volume-rust, all"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Copy binary to install dir, using sudo if needed
|
||||
install_binary() {
|
||||
local src="$1" name="$2"
|
||||
local dest="${INSTALL_DIR}/${name}"
|
||||
|
||||
mkdir -p "$INSTALL_DIR" 2>/dev/null || true
|
||||
|
||||
if [ -w "$INSTALL_DIR" ]; then
|
||||
cp "$src" "$dest"
|
||||
else
|
||||
info "Need elevated permissions to write to ${INSTALL_DIR}"
|
||||
sudo cp "$src" "$dest"
|
||||
fi
|
||||
chmod +x "$dest" 2>/dev/null || sudo chmod +x "$dest"
|
||||
}
|
||||
|
||||
main() {
|
||||
info "SeaweedFS Installer"
|
||||
|
||||
read -r os arch <<< "$(detect_platform)"
|
||||
info "Detected platform: ${os}/${arch}"
|
||||
|
||||
if [ -z "$VERSION" ]; then
|
||||
info "Resolving latest release..."
|
||||
VERSION="$(get_latest_version)"
|
||||
if [ -z "$VERSION" ]; then
|
||||
error "Could not determine latest version. Specify with --version"
|
||||
fi
|
||||
fi
|
||||
info "Version: ${VERSION}"
|
||||
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
info "Variant: large disk (8TB max volume)"
|
||||
else
|
||||
info "Variant: normal (32GB max volume)"
|
||||
fi
|
||||
|
||||
case "$COMPONENT" in
|
||||
all)
|
||||
install_component "weed" "$os" "$arch"
|
||||
install_component "volume-rust" "$os" "$arch"
|
||||
;;
|
||||
*)
|
||||
install_component "$COMPONENT" "$os" "$arch"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
ok "Installation complete!"
|
||||
if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then
|
||||
info " weed: ${INSTALL_DIR}/weed"
|
||||
fi
|
||||
if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then
|
||||
info " weed-volume: ${INSTALL_DIR}/weed-volume"
|
||||
fi
|
||||
echo ""
|
||||
info "Quick start:"
|
||||
info " weed master # Start master server"
|
||||
info " weed volume -mserver=localhost:9333 # Start Go volume server"
|
||||
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
|
||||
}
|
||||
|
||||
main
|
||||
@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.27"
|
||||
appVersion: "4.09"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.27.0
|
||||
version: 4.0.409
|
||||
|
||||
@ -49,35 +49,6 @@ CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
#### RocksDB variant
|
||||
|
||||
The `_large_disk_rocksdb` image tag ships with RocksDB pre-configured as the filer backend.
|
||||
To use this image with the Helm chart, override the image on all three components and disable
|
||||
the chart's default `WEED_LEVELDB2_ENABLED`, which would otherwise re-enable LevelDB2 and
|
||||
override the image's built-in RocksDB configuration:
|
||||
|
||||
```yaml
|
||||
# Replace <VERSION> with the desired seaweedfs version, e.g. 3.80_large_disk_rocksdb.
|
||||
master:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
volume:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
filer:
|
||||
enablePVC: true
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
extraEnvironmentVars:
|
||||
WEED_LEVELDB2_ENABLED: "false"
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
* `master` and `volume` use the same image tag so that all components share a consistent
|
||||
SeaweedFS build; RocksDB itself is only used by the filer.
|
||||
* `filer.enablePVC: true` (or another form of persistent storage for the filer) is required
|
||||
so that the RocksDB metadata store survives pod restarts — otherwise metadata will be lost.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
|
||||
|
||||
@ -194,9 +165,8 @@ admin:
|
||||
enabled: true
|
||||
port: 23646
|
||||
grpcPort: 33646 # For worker connections
|
||||
secret:
|
||||
adminUser: "admin"
|
||||
adminPassword: "your-secure-password" # Leave empty to disable auth
|
||||
adminUser: "admin"
|
||||
adminPassword: "your-secure-password" # Leave empty to disable auth
|
||||
|
||||
# Optional: persist admin data
|
||||
data:
|
||||
@ -221,8 +191,6 @@ If `adminPassword` is set, the admin interface requires authentication:
|
||||
|
||||
If `adminPassword` is empty or not set, the admin interface runs without authentication (not recommended for production).
|
||||
|
||||
As an alternative, a kubernetes Secret can be used (`admin.secret.existingSecret`).
|
||||
|
||||
### Admin Data Persistence
|
||||
|
||||
The admin component can store configuration and maintenance data. You can configure storage in several ways:
|
||||
@ -244,9 +212,8 @@ To enable workers, add the following to your values.yaml:
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 2 # Scale based on workload
|
||||
jobType: "vacuum,volume_balance,erasure_coding" # Job types this worker can handle
|
||||
maxDetect: 1 # Maximum concurrent detection requests
|
||||
maxExecute: 4 # Maximum concurrent execution jobs per worker
|
||||
capabilities: "vacuum,balance,erasure_coding" # Tasks this worker can handle
|
||||
maxConcurrent: 3 # Maximum concurrent tasks per worker
|
||||
|
||||
# Working directory for task execution
|
||||
# Default: "/tmp/seaweedfs-worker"
|
||||
@ -281,14 +248,14 @@ worker:
|
||||
memory: "2Gi"
|
||||
```
|
||||
|
||||
### Worker Job Types
|
||||
### Worker Capabilities
|
||||
|
||||
Workers can be configured with different job types:
|
||||
Workers can be configured with different capabilities:
|
||||
- **vacuum**: Reclaim deleted file space
|
||||
- **volume_balance**: Balance volumes across volume servers
|
||||
- **balance**: Balance volumes across volume servers
|
||||
- **erasure_coding**: Handle erasure coding operations
|
||||
|
||||
You can configure workers with all job types or create specialized worker pools with specific job types.
|
||||
You can configure workers with all capabilities or create specialized worker pools with specific capabilities.
|
||||
|
||||
### Worker Deployment Strategy
|
||||
|
||||
@ -297,11 +264,11 @@ For production deployments, consider:
|
||||
1. **Multiple Workers**: Deploy 2+ worker replicas for high availability
|
||||
2. **Resource Allocation**: Workers need sufficient CPU/memory for maintenance tasks
|
||||
3. **Storage**: Workers need temporary storage for vacuum and balance operations (size depends on volume size)
|
||||
4. **Specialized Workers**: Create separate worker deployments for different job types if needed
|
||||
4. **Specialized Workers**: Create separate worker deployments for different capabilities if needed
|
||||
|
||||
Example specialized worker configuration:
|
||||
|
||||
For specialized worker pools, deploy separate Helm releases with different job types:
|
||||
For specialized worker pools, deploy separate Helm releases with different capabilities:
|
||||
|
||||
**values-worker-vacuum.yaml** (for vacuum operations):
|
||||
```yaml
|
||||
@ -320,8 +287,8 @@ admin:
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 2
|
||||
jobType: "vacuum"
|
||||
maxExecute: 2
|
||||
capabilities: "vacuum"
|
||||
maxConcurrent: 2
|
||||
# REQUIRED: Point to the admin service of your main SeaweedFS release
|
||||
# Replace <namespace> with the namespace where your main seaweedfs is deployed
|
||||
# Example: If deploying in namespace "production":
|
||||
@ -346,8 +313,8 @@ admin:
|
||||
worker:
|
||||
enabled: true
|
||||
replicas: 1
|
||||
jobType: "volume_balance"
|
||||
maxExecute: 1
|
||||
capabilities: "balance"
|
||||
maxConcurrent: 1
|
||||
# REQUIRED: Point to the admin service of your main SeaweedFS release
|
||||
# Replace <namespace> with the namespace where your main seaweedfs is deployed
|
||||
# Example: If deploying in namespace "production":
|
||||
@ -356,7 +323,6 @@ worker:
|
||||
```
|
||||
|
||||
Deploy the specialized workers as separate releases:
|
||||
### Specialized Worker Deployment
|
||||
```bash
|
||||
# Deploy vacuum workers
|
||||
helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum.yaml
|
||||
@ -365,22 +331,6 @@ helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum
|
||||
helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml
|
||||
```
|
||||
|
||||
## OpenShift Support
|
||||
|
||||
SeaweedFS can be deployed on OpenShift or any cluster enforcing the Kubernetes "restricted" Pod Security Standard. By default, OpenShift blocks containers that run as root or use `hostPath` volumes.
|
||||
|
||||
To deploy on OpenShift, use the provided `openshift-values.yaml` which overrides the default configuration to:
|
||||
1. Use `PersistentVolumeClaims` instead of `hostPath`.
|
||||
2. Enable `runAsNonRoot` and omit hardcoded UIDs to allow OpenShift to assign valid UIDs automatically.
|
||||
3. Apply appropriate `seccompProfile` and drop capabilities.
|
||||
|
||||
Usage:
|
||||
```bash
|
||||
helm install seaweedfs seaweedfs/seaweedfs \
|
||||
-n seaweedfs --create-namespace \
|
||||
-f openshift-values.yaml
|
||||
```
|
||||
|
||||
## Enterprise
|
||||
|
||||
For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition,
|
||||
|
||||
@ -1168,108 +1168,6 @@
|
||||
"title": "Filer QPS",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"id": 91,
|
||||
"links": [],
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true,
|
||||
"width": 250
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "sum by (code) (rate(SeaweedFS_upload_error_total{namespace=\"$NAMESPACE\"}[$__rate_interval]))",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{code}}",
|
||||
"refId": "A",
|
||||
"step": 30
|
||||
}
|
||||
],
|
||||
"title": "Upload Errors",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": {
|
||||
@ -1280,7 +1178,7 @@
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 35
|
||||
"y": 28
|
||||
},
|
||||
"id": 61,
|
||||
"panels": [],
|
||||
@ -1353,7 +1251,7 @@
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 36
|
||||
"y": 29
|
||||
},
|
||||
"id": 65,
|
||||
"links": [],
|
||||
@ -1459,7 +1357,7 @@
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 36
|
||||
"y": 29
|
||||
},
|
||||
"id": 56,
|
||||
"links": [],
|
||||
@ -1565,7 +1463,7 @@
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 36
|
||||
"y": 29
|
||||
},
|
||||
"id": 58,
|
||||
"links": [],
|
||||
@ -1622,7 +1520,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 43
|
||||
"y": 36
|
||||
},
|
||||
"id": 84,
|
||||
"links": [],
|
||||
@ -1641,7 +1539,7 @@
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_bucket_traffic_received_bytes_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (bucket)",
|
||||
"expr": "sum(rate(SeaweedFS_s3_bucket_traffic_received_bytes_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
@ -1667,7 +1565,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 43
|
||||
"y": 36
|
||||
},
|
||||
"id": 85,
|
||||
"links": [],
|
||||
@ -1686,7 +1584,7 @@
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_bucket_traffic_sent_bytes_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (bucket)",
|
||||
"expr": "sum(rate(SeaweedFS_s3_bucket_traffic_sent_bytes_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
@ -1763,7 +1661,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 48
|
||||
"y": 41
|
||||
},
|
||||
"id": 86,
|
||||
"options": {
|
||||
@ -1781,7 +1679,7 @@
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (bucket)",
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
@ -1858,7 +1756,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 48
|
||||
"y": 41
|
||||
},
|
||||
"id": 72,
|
||||
"links": [],
|
||||
@ -1976,7 +1874,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 57
|
||||
"y": 50
|
||||
},
|
||||
"id": 73,
|
||||
"links": [],
|
||||
@ -2132,7 +2030,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 64
|
||||
"y": 57
|
||||
},
|
||||
"id": 55,
|
||||
"links": [],
|
||||
@ -2289,7 +2187,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 71
|
||||
"y": 64
|
||||
},
|
||||
"hideTimeOverride": false,
|
||||
"id": 59,
|
||||
@ -2361,7 +2259,7 @@
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 78
|
||||
"y": 71
|
||||
},
|
||||
"id": 62,
|
||||
"panels": [],
|
||||
@ -2433,7 +2331,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 79
|
||||
"y": 72
|
||||
},
|
||||
"id": 47,
|
||||
"links": [],
|
||||
@ -2576,7 +2474,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 79
|
||||
"y": 72
|
||||
},
|
||||
"id": 40,
|
||||
"links": [],
|
||||
@ -2673,7 +2571,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 86
|
||||
"y": 79
|
||||
},
|
||||
"id": 48,
|
||||
"links": [],
|
||||
@ -2783,7 +2681,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 93
|
||||
"y": 86
|
||||
},
|
||||
"id": 50,
|
||||
"links": [],
|
||||
@ -2885,7 +2783,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 100
|
||||
"y": 93
|
||||
},
|
||||
"id": 51,
|
||||
"links": [],
|
||||
@ -2925,7 +2823,7 @@
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 100
|
||||
"y": 93
|
||||
},
|
||||
"id": 63,
|
||||
"panels": [],
|
||||
@ -2998,7 +2896,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 108
|
||||
"y": 101
|
||||
},
|
||||
"id": 12,
|
||||
"links": [],
|
||||
@ -3093,7 +2991,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 108
|
||||
"y": 101
|
||||
},
|
||||
"id": 14,
|
||||
"links": [],
|
||||
@ -3135,7 +3033,7 @@
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 115
|
||||
"y": 108
|
||||
},
|
||||
"id": 64,
|
||||
"panels": [],
|
||||
@ -3208,7 +3106,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 116
|
||||
"y": 109
|
||||
},
|
||||
"id": 52,
|
||||
"links": [],
|
||||
@ -3336,7 +3234,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 116
|
||||
"y": 109
|
||||
},
|
||||
"id": 54,
|
||||
"links": [],
|
||||
@ -3433,7 +3331,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 123
|
||||
"y": 116
|
||||
},
|
||||
"id": 53,
|
||||
"links": [],
|
||||
@ -3528,7 +3426,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 55
|
||||
"y": 48
|
||||
},
|
||||
"id": 89,
|
||||
"options": {
|
||||
@ -3635,7 +3533,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 55
|
||||
"y": 48
|
||||
},
|
||||
"id": 90,
|
||||
"options": {
|
||||
@ -3756,4 +3654,4 @@
|
||||
"uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
@ -1,132 +0,0 @@
|
||||
# openshift-values.yaml
|
||||
#
|
||||
# Example overrides for deploying SeaweedFS on OpenShift (or any cluster
|
||||
# enforcing the Kubernetes "restricted" Pod Security Standard).
|
||||
#
|
||||
# OpenShift's default "restricted" SCC blocks containers that:
|
||||
# - Run as UID 0 (root)
|
||||
# - Request privilege escalation
|
||||
# - Use hostPath volumes
|
||||
# - Omit a seccompProfile
|
||||
#
|
||||
# These overrides satisfy all four requirements by:
|
||||
# 1. Replacing hostPath volumes with PersistentVolumeClaims (or emptyDir for logs)
|
||||
# 2. Enabling runAsNonRoot: true. By omitting runAsUser, OpenShift will
|
||||
# automatically assign a valid UID from the namespace's allocated range.
|
||||
# 3. Dropping all Linux capabilities and setting allowPrivilegeEscalation: false
|
||||
# 4. Enabling RuntimeDefault seccompProfile
|
||||
#
|
||||
# Usage:
|
||||
# helm install seaweedfs seaweedfs/seaweedfs \
|
||||
# -n seaweedfs --create-namespace \
|
||||
# -f openshift-values.yaml
|
||||
#
|
||||
# Adjust storageClass and sizes to match your cluster's available StorageClasses.
|
||||
# On OpenShift you can discover them with: oc get storageclass
|
||||
global:
|
||||
enableReplication: true
|
||||
# replication type is XYZ:
|
||||
# X number of replica in other data centers
|
||||
# Y number of replica in other racks in the same data center
|
||||
# Z number of replica in other servers in the same rack
|
||||
replicationPlacement: "000" # no data replica
|
||||
master:
|
||||
replicas: 1
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "10Gi"
|
||||
storageClass: "" # leave empty to use the cluster default StorageClass
|
||||
|
||||
logs:
|
||||
type: "emptyDir" # avoids hostPath; use persistentVolumeClaim if you need log persistence
|
||||
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
|
||||
volume:
|
||||
replicas: 1
|
||||
dataDirs:
|
||||
- name: data1
|
||||
type: "persistentVolumeClaim"
|
||||
size: "100Gi"
|
||||
storageClass: "" # leave empty to use the cluster default StorageClass
|
||||
maxVolumes: 0
|
||||
|
||||
logs:
|
||||
type: "emptyDir"
|
||||
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
|
||||
filer:
|
||||
replicas: 1
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "25Gi"
|
||||
storageClass: "" # leave empty to use the cluster default StorageClass
|
||||
|
||||
logs:
|
||||
type: "emptyDir"
|
||||
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
|
||||
# S3 gateway (if enabled)
|
||||
s3:
|
||||
enabled: true
|
||||
replicas: 1
|
||||
port: 8333
|
||||
enableAuth: true
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
logs:
|
||||
type: "emptyDir"
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
@ -2,7 +2,7 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.componentName" (list . "admin") }}
|
||||
name: {{ printf "%s-admin" (include "seaweedfs.fullname" .) | trunc 63 | trimSuffix "-" }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user