gian/seaweedfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Perf CI: benchmark the Rust volume server and report memory usage (#10111)

Browse Source 
* ci: add per-process memory sampler for perf jobs

Samples VmRSS once a second into a CSV and records peak VmHWM per process
on stop. Linux only; reads /proc/<pid>/status.

* ci: run perf benchmarks on the Rust volume server and report memory

Matrix the throughput and S3 jobs over go/rust volume servers, using a
standalone master (plus filer for S3) and swapping only the volume binary
so the two are directly comparable. Sample peak RSS in every job and surface
it per impl in the run summary.

* ci: harden mem sampler arg handling and peak fallback

Guard against missing args under set -u, and fall back to the max RSS
sampled when a process exits before VmHWM can be read.
This commit is contained in:
Chris Lu 2026-06-25 10:52:23 -07:00 committed by GitHub
parent 2efc0e1656
commit 2c2df751f5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 283 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

251
.github/workflows/performance.yml vendored
View File

@ -7,6 +7,8 @@ on:
- '**/*.go'
- 'go.mod'
- 'go.sum'
- 'seaweed-volume/**'
- 'test/perf/**'
- '.github/workflows/performance.yml'
workflow_dispatch:
inputs:
@ -48,6 +50,9 @@ concurrency:
permissions:
contents: read
env:
VOL_SIZE_LIMIT: "1024"
jobs:
performance-profile:
name: CPU and Heap Profile
@ -83,6 +88,11 @@ jobs:
# give the volume server a moment to register with the master
sleep 3
- name: Start memory sampler
run: |
bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" &
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
- name: Capture profiles under load
run: |
DURATION="${{ github.event.inputs.profile_duration || '30' }}"
@ -96,6 +106,22 @@ jobs:
go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true
go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true
- name: Record memory usage
if: always()
run: |
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
sleep 2
{
echo "## Memory usage (peak RSS)"
echo '```'
if [ -f mem-profile.csv.peak ]; then
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak
else
echo "no memory samples captured"
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Profile summary
if: always()
run: |
@ -125,13 +151,21 @@ jobs:
cpu-top.txt
heap-top.txt
goroutine.txt
mem-profile.csv
mem-profile.csv.peak
weed.log
retention-days: 30
benchmark:
name: Throughput Benchmark
name: Throughput Benchmark (${{ matrix.impl }} volume)
runs-on: ubuntu-latest
timeout-minutes: 45
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
impl: [go, rust]
env:
IMPL: ${{ matrix.impl }}
steps:
- name: Check out code
uses: actions/checkout@v7
@ -141,15 +175,41 @@ jobs:
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
if: matrix.impl == 'rust'
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust'
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-
- name: Build weed
run: go build -o weed_bin ./weed
- name: Start server
- name: Build Rust volume server
if: matrix.impl == 'rust'
run: cd seaweed-volume && cargo build --release
- name: Start master and volume server
run: |
mkdir -p ./perfdata
./weed_bin -v=1 server -dir=./perfdata -volume.max=0 \
-master.volumeSizeLimitMB=1024 > weed.log 2>&1 &
echo "WEED_PID=$!" >> "$GITHUB_ENV"
mkdir -p ./perfdata/master ./perfdata/vol
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
-mdir=./perfdata/master -peers=none \
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
> master.log 2>&1 &
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
@ -157,8 +217,33 @@ jobs:
fi
sleep 1
done
if [ "${IMPL}" = "rust" ]; then
./seaweed-volume/target/release/weed-volume \
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
> volume.log 2>&1 &
else
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
-port=8080 -dir=./perfdata/vol -max=100 \
> volume.log 2>&1 &
fi
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
echo "volume server is ready"
break
fi
sleep 1
done
# let the volume server register with the master via heartbeat
sleep 3
- name: Start memory sampler
run: |
bash test/perf/mem_sample.sh mem-benchmark.csv \
"master=${MASTER_PID}" "volume=${VOLUME_PID}" &
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
- name: Run throughput benchmark
run: |
N="${{ github.event.inputs.benchmark_files || '100000' }}"
@ -168,46 +253,75 @@ jobs:
-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt
- name: Run Go micro-benchmarks
if: matrix.impl == 'go'
continue-on-error: true
run: |
go test -run='^$' -bench=. -benchmem -benchtime=10x \
./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \
2>&1 | tee go-benchmarks.txt
- name: Record memory usage
if: always()
run: |
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
sleep 2
{
echo "## Memory usage (peak RSS, ${IMPL} volume)"
echo '```'
if [ -f mem-benchmark.csv.peak ]; then
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak
else
echo "no memory samples captured"
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Benchmark summary
if: always()
run: |
{
echo "## Throughput benchmark"
echo "## Throughput benchmark (${IMPL} volume)"
echo '```'
grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \
benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Stop server
- name: Stop processes
if: always()
run: kill "${WEED_PID}" 2>/dev/null || true
run: |
kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
- name: Show server log on failure
- name: Show logs on failure
if: failure()
run: tail -200 weed.log || true
run: |
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
- name: Upload benchmark results
if: always()
uses: actions/upload-artifact@v7
with:
name: benchmark-results-${{ github.run_number }}
name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
path: |
benchmark-results.txt
go-benchmarks.txt
weed.log
mem-benchmark.csv
mem-benchmark.csv.peak
master.log
volume.log
retention-days: 7
s3-benchmark:
name: S3 Read/Write Benchmark
name: S3 Read/Write Benchmark (${{ matrix.impl }} volume)
runs-on: ubuntu-latest
timeout-minutes: 45
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
impl: [go, rust]
env:
IMPL: ${{ matrix.impl }}
steps:
- name: Check out code
uses: actions/checkout@v7
@ -217,19 +331,43 @@ jobs:
with:
go-version-file: 'go.mod'
- name: Install protobuf compiler
if: matrix.impl == 'rust'
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust'
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
restore-keys: |
rust-
- name: Build weed and S3 load tool
run: |
go build -o weed_bin ./weed
go build -o s3bench ./test/s3/benchmark
- name: Start server with S3 gateway
- name: Build Rust volume server
if: matrix.impl == 'rust'
run: cd seaweed-volume && cargo build --release
- name: Start cluster with S3 gateway
run: |
mkdir -p ./perfdata
./weed_bin -v=1 server -dir=./perfdata -s3 -filer \
-volume.max=0 -master.volumeSizeLimitMB=1024 \
-s3.port=8000 -s3.config=./docker/compose/s3.json \
> weed.log 2>&1 &
echo "WEED_PID=$!" >> "$GITHUB_ENV"
mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
-mdir=./perfdata/master -peers=none \
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
> master.log 2>&1 &
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
echo "master is ready"
@ -237,6 +375,29 @@ jobs:
fi
sleep 1
done
if [ "${IMPL}" = "rust" ]; then
./seaweed-volume/target/release/weed-volume \
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
> volume.log 2>&1 &
else
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
-port=8080 -dir=./perfdata/vol -max=100 \
> volume.log 2>&1 &
fi
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 60); do
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
echo "volume server is ready"
break
fi
sleep 1
done
sleep 3
./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \
-s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \
> filer.log 2>&1 &
echo "FILER_PID=$!" >> "$GITHUB_ENV"
for i in $(seq 1 30); do
if nc -z localhost 8000 2>/dev/null; then
echo "s3 gateway is ready"
@ -246,6 +407,12 @@ jobs:
done
sleep 2
- name: Start memory sampler
run: |
bash test/perf/mem_sample.sh mem-s3.csv \
"master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" &
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
- name: Run S3 read/write benchmark
run: |
OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"
@ -256,31 +423,55 @@ jobs:
-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \
2>&1 | tee s3-benchmark-results.txt
- name: Record memory usage
if: always()
run: |
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
sleep 2
{
echo "## Memory usage (peak RSS, ${IMPL} volume)"
echo '```'
if [ -f mem-s3.csv.peak ]; then
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak
else
echo "no memory samples captured"
fi
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: S3 benchmark summary
if: always()
run: |
{
echo "## S3 read/write benchmark"
echo "## S3 read/write benchmark (${IMPL} volume)"
echo '```'
grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \
s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Stop server
- name: Stop processes
if: always()
run: kill "${WEED_PID}" 2>/dev/null || true
run: |
kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
- name: Show server log on failure
- name: Show logs on failure
if: failure()
run: tail -200 weed.log || true
run: |
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true
- name: Upload S3 benchmark results
if: always()
uses: actions/upload-artifact@v7
with:
name: s3-benchmark-results-${{ github.run_number }}
name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
path: |
s3-benchmark-results.txt
weed.log
mem-s3.csv
mem-s3.csv.peak
master.log
volume.log
filer.log
retention-days: 7

62
test/perf/mem_sample.sh Executable file
View File

@ -0,0 +1,62 @@
#!/usr/bin/env bash
# Sample resident memory of one or more processes once per second.
# Usage: mem_sample.sh <out.csv> <label=pid> [label=pid ...]
#
# Writes a per-second time series to <out.csv> and, on stop, the peak
# resident set (VmHWM) per process to <out.csv>.peak. Stop with SIGTERM/SIGINT;
# if a process already exited, its peak falls back to the max RSS sampled.
# Linux only (reads /proc/<pid>/status).
set -u
if [ "$#" -lt 2 ]; then
echo "Usage: $0 <out.csv> <label=pid> [label=pid ...]" >&2
exit 1
fi
out="$1"; shift
labels=(); pids=(); peaks=()
for arg in "$@"; do
labels+=("${arg%%=*}")
pids+=("${arg#*=}")
peaks+=(0)
done
record_peaks() {
local total=0 i hwm
: > "${out}.peak"
for i in "${!pids[@]}"; do
hwm="$(awk '/^VmHWM:/{print $2}' "/proc/${pids[$i]}/status" 2>/dev/null)"
if [ -z "$hwm" ] || [ "$hwm" -eq 0 ]; then
hwm="${peaks[$i]}"
fi
printf '%s\t%s\n' "${labels[$i]}" "$hwm" >> "${out}.peak"
total=$((total + hwm))
done
printf 'total\t%s\n' "$total" >> "${out}.peak"
exit 0
}
trap record_peaks TERM INT
header="epoch"
for l in "${labels[@]}"; do header+=",${l}_rss_kb"; done
header+=",total_rss_kb"
echo "$header" > "$out"
while true; do
line="$(date +%s)"; total=0
for i in "${!pids[@]}"; do
p="${pids[$i]}"
rss="$(awk '/^VmRSS:/{print $2}' "/proc/$p/status" 2>/dev/null)"
rss="${rss:-0}"
if [ "$rss" -gt "${peaks[$i]}" ]; then
peaks[$i]="$rss"
fi
line+=",${rss}"
total=$((total + rss))
done
echo "${line},${total}" >> "$out"
# Sleep via a background child + wait so a caught signal interrupts promptly.
sleep 1 &
wait $! 2>/dev/null
done