gian/ceph-mirror
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2026-08-02 07:03:18 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

rgw/test/multisite: test for bilog autotrim -ENOENT race after secondary reshard

Browse Source 
Adds test_bucket_log_trim_enoent_race_after_reshard to reproduce the
exact failure from tracker #70858: when autotrim runs on the primary
zone first and fully cleans up (including removing its own bucket.instance
metadata), then an autotrim on the secondary zone asks the primary
and receives -ENOENT.

The secondary hasn't received the BucketLogType::Deleted
flag yet (metadata sync is stalled via delay injection), so the current code
left StatusShards{generation=0, shards=[]} which caused take_min_status()
to fail with -EINVAL, leaving the bucket.instance metadata permanently
orphaned.

Tests: https://tracker.ceph.com/issues/70858
Signed-off-by: Oguzhan Ozmen <oozmen@bloomberg.net>
This commit is contained in:
Oguzhan Ozmen 2026-07-05 00:45:40 +00:00
parent 36b2e02d2e
commit 43518e2f3e
2 changed files with 107 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/rgw/driver/rados/rgw_sync.cc
View File

@ -1360,6 +1360,17 @@ int RGWMetaSyncSingleEntryCR::operate(const DoutPrefixProvider *dpp) {
retcode = 0;
for (tries = 0; tries < NUM_TRANSIENT_ERROR_RETRIES; tries++) {
if (sync_status != -ENOENT) {
if (section == "bucket.instance" &&
cct->_conf->rgw_inject_delay_sec > 0 &&
std::string_view(cct->_conf->rgw_inject_delay_pattern) ==
"delay_meta_sync_bucket_instance_store") {
yield {
utime_t dur;
dur.set_from_double(cct->_conf->rgw_inject_delay_sec);
tn->log(0, SSTR("injecting a delay of " << dur << "s for bucket.instance metadata store"));
wait(dur);
}
}
tn->log(10, SSTR("storing local metadata entry: " << section << ":" << key));
yield call(new RGWMetaStoreEntryCR(sync_env, raw_key, md_bl));
} else {

96
src/test/rgw/rgw_multi/tests.py
View File

@ -2200,6 +2200,102 @@ def test_bucket_log_trim_after_delete_bucket_secondary_reshard():
assert check_bucket_instance_metadata(zone.zone, test_bucket.name)
@attr('bucket_trim')
def test_bucket_log_trim_enoent_race_after_reshard():
zonegroup = realm.master_zonegroup()
zonegroup_conns = ZonegroupConns(zonegroup)
primary = zonegroup_conns.rw_zones[0]
secondary = zonegroup_conns.rw_zones[1]
all_clusters = set()
for zg in realm.current_period.zonegroups:
for zone in zg.zones:
all_clusters.add(zone.cluster)
def set_meta_sync_delay(delay_sec):
for cluster in all_clusters:
if delay_sec > 0:
cluster.ceph_admin(
['config', 'set', 'client', 'rgw_inject_delay_sec', str(delay_sec)])
cluster.ceph_admin(
['config', 'set', 'client', 'rgw_inject_delay_pattern', 'delay_meta_sync_bucket_instance_store'])
else:
cluster.ceph_admin(
['config', 'rm', 'client', 'rgw_inject_delay_sec'])
cluster.ceph_admin(
['config', 'rm', 'client', 'rgw_inject_delay_pattern'])
def make_test_bucket():
name = gen_bucket_name()
log.info('create bucket zone=%s name=%s', primary.zone.name, name)
bucket = primary.create_bucket(name)
for objname in ('a', 'b', 'c', 'd'):
primary.s3_client.put_object(Bucket=bucket.name, Key=objname, Body='foo')
zonegroup_meta_checkpoint(zonegroup)
zonegroup_bucket_checkpoint(zonegroup_conns, name)
return bucket
test_bucket = make_test_bucket()
# reshard on secondary to create a generation mismatch
secondary.zone.cluster.admin(['bucket', 'reshard',
'--bucket', test_bucket.name,
'--num-shards', '13',
'--yes-i-really-mean-it'] + secondary.zone.zone_args())
for obj in ('a', 'b', 'c', 'd'):
cmd = ['object', 'rm'] + primary.zone.zone_args()
cmd += ['--bucket', test_bucket.name]
cmd += ['--object', obj]
primary.zone.cluster.admin(cmd + primary.zone.zone_args())
log.info('setting metadata sync delay to reproduce ENOENT trim race')
set_meta_sync_delay(30)
time.sleep(10) # let the delay config reach the radosgws before deleting
try:
primary.s3_client.delete_bucket(Bucket=test_bucket.name)
zonegroup_data_checkpoint(zonegroup_conns)
# run autotrim on primary first — primary has the Deleted flag
# so it will fully clean up including removing its own
# bucket.instance metadata
bilog_autotrim(primary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
bilog_autotrim(primary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
# now autotrim on secondary — secondary queries primary, but
# primary's instance metadata is gone so primary responds -ENOENT.
# secondary doesn't have the Deleted flag yet (metadata sync is
# stalled). In #70858, this leaves StatusShards{gen=0, shards=[]}
# causing take_min_status() to fail with -EINVAL.
bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
finally:
log.info('removing metadata sync delay')
set_meta_sync_delay(0)
for zonegroup in realm.current_period.zonegroups:
zonegroup_conns = ZonegroupConns(zonegroup)
zonegroup_meta_checkpoint(zonegroup)
for zone in zonegroup_conns.zones:
log.info('trimming on zone=%s', zone.name)
bilog_autotrim(zone.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
for zonegroup in realm.current_period.zonegroups:
zonegroup_conns = ZonegroupConns(zonegroup)
for zone in zonegroup_conns.zones:
assert check_bucket_instance_metadata(zone.zone, test_bucket.name)
@attr('bucket_reshard')
def test_bucket_reshard_incremental():
zonegroup = realm.master_zonegroup()