gian/ceph-mirror
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2026-08-02 07:03:18 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #70003 from BBoozmen/wip-oozmen-70858

Browse Source 
rgw: fix bilog autotrim of stale bucket.instance metadata after peer deletion
This commit is contained in:
Oguzhan Ozmen 2026-07-22 11:47:39 -04:00 committed by GitHub
commit 9e160aa8f1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 417 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/rgw/driver/rados/rgw_sync.cc
View File

@ -1360,6 +1360,28 @@ int RGWMetaSyncSingleEntryCR::operate(const DoutPrefixProvider *dpp) {
retcode = 0;
for (tries = 0; tries < NUM_TRANSIENT_ERROR_RETRIES; tries++) {
if (sync_status != -ENOENT) {
if (section == "bucket.instance" &&
cct->_conf->rgw_inject_delay_sec > 0 &&
std::string_view(cct->_conf->rgw_inject_delay_pattern) ==
"delay_meta_sync_bucket_instance_store") {
yield {
utime_t dur;
dur.set_from_double(cct->_conf->rgw_inject_delay_sec);
tn->log(0, SSTR("injecting a delay of " << dur << "s for bucket.instance metadata store"));
wait(dur);
}
}
if (section == "bucket" &&
cct->_conf->rgw_inject_delay_sec > 0 &&
std::string_view(cct->_conf->rgw_inject_delay_pattern) ==
"delay_meta_sync_bucket_entrypoint_store") {
yield {
utime_t dur;
dur.set_from_double(cct->_conf->rgw_inject_delay_sec);
tn->log(0, SSTR("injecting a delay of " << dur << "s for bucket entrypoint metadata store"));
wait(dur);
}
}
tn->log(10, SSTR("storing local metadata entry: " << section << ":" << key));
yield call(new RGWMetaStoreEntryCR(sync_env, raw_key, md_bl));
} else {

204
src/rgw/driver/rados/rgw_trim_bilog.cc
View File

@ -458,7 +458,6 @@ class RGWReadRemoteStatusShardsCR : public RGWCoroutine {
rgw::sal::RadosStore* const store;
CephContext *cct;
RGWHTTPManager *http;
const RGWBucketInfo* bucket_info;
std::string bucket_instance;
const rgw_zone_id zid;
const std::string& zone_id;
@ -469,13 +468,12 @@ public:
rgw::sal::RadosStore* const store,
CephContext *cct,
RGWHTTPManager *http,
const RGWBucketInfo* bucket_info,
std::string bucket_instance,
const rgw_zone_id zid,
const std::string& zone_id,
StatusShards *p)
: RGWCoroutine(cct), dpp(dpp), store(store),
cct(cct), http(http), bucket_info(bucket_info), bucket_instance(bucket_instance),
cct(cct), http(http), bucket_instance(bucket_instance),
zid(zid), zone_id(zone_id), p(p) {}
int operate(const DoutPrefixProvider *dpp) override {
@ -505,10 +503,16 @@ public:
if (retcode < 0 && retcode != -ENOENT) {
return set_cr_error(retcode);
} else if (retcode == -ENOENT && bucket_info->layout.logs.front().layout.type == rgw::BucketLogType::Deleted) {
} else if (retcode == -ENOENT) {
// the peer has no bucket-index sync status for this bucket which does
// not by itself mean it is deleted there (an uninitialized sync status,
// e.g. no data written yet, looks the same). UINT64_MAX just excludes
// this peer from the minimum trim generation; we will confirm real
// deletion via the bucket entrypoint before removing anything.
p->generation = UINT64_MAX;
ldpp_dout(dpp, 10) << "INFO: could not read shard status for bucket:" << bucket_instance
<< " from zone: " << zid.id << dendl;
ldpp_dout(dpp, 10) << "bucket=" << bucket_instance
<< ": peer zone=" << zid.id << " has no sync status (-ENOENT); "
<< "excluding it from the trim generation" << dendl;
}
return set_cr_done();
@ -518,6 +522,95 @@ public:
};
/// result of a metadata GET for a bucket ENTRYPOINT, unwrapping the
/// {"key":..., "ver":..., "mtime":..., "data":{...}} envelope returned by
/// /admin/metadata/bucket/<key>
struct RGWBucketEntrypointMetadataResult {
RGWBucketEntryPoint data;
void decode_json(JSONObj *obj) {
JSONDecoder::decode_json("data", data, obj);
}
};
/// Resolve a bucket's entrypoint to the instance id it currently points at.
///
/// The entrypoint is a reliable deletion signal: its removal is a real metadata
/// op (unlike the no-op instance removal), so its absence means a deletion has synced.
/// But a zone's *local* entrypoint copy is not authoritative during creation: the
/// instance syncs to peers before the entrypoint does (put_linked_bucket_info() in
/// rgw_rados.cc writes the instance, then the entrypoint), so a peer can hold the
/// instance without yet holding the entrypoint: i.e., indistinguishable locally from
/// a real deletion.
/// The metadata master is authoritative and has the entrypoint immediately on
/// creation, so consult it, unless this zone is the master, whose local copy
/// is itself authoritative.
///
/// On success sets *bucket_id to the resolved instance id, or empty if the
/// entrypoint no longer exists. Returns an error only if the authoritative copy
/// could not be read (the caller should then skip and retry on a later cycle).
class RGWReadMasterBucketEntrypointCR : public RGWCoroutine {
rgw::sal::RadosStore* const store;
RGWHTTPManager *http;
const rgw_bucket bucket;
std::string *bucket_id;
std::shared_ptr<rgw_get_bucket_info_result> local_result;
RGWBucketEntrypointMetadataResult master_result;
public:
RGWReadMasterBucketEntrypointCR(rgw::sal::RadosStore* store,
RGWHTTPManager *http,
const rgw_bucket& bucket,
std::string *bucket_id)
: RGWCoroutine(store->ctx()), store(store), http(http),
bucket(bucket), bucket_id(bucket_id),
local_result(make_shared<rgw_get_bucket_info_result>()) {}
int operate(const DoutPrefixProvider *dpp) override {
reenter(this) {
if (store->svc()->zone->is_meta_master()) {
yield call(new RGWGetBucketInfoCR(store->svc()->async_processor, store,
{bucket.tenant, bucket.name},
local_result, dpp));
if (retcode == -ENOENT) {
bucket_id->clear();
return set_cr_done();
}
if (retcode < 0) {
return set_cr_error(retcode);
}
*bucket_id = local_result->bucket ? local_result->bucket->get_bucket_id()
: std::string();
return set_cr_done();
}
yield {
auto *master_conn = store->svc()->zone->get_master_conn();
if (!master_conn) {
ldpp_dout(dpp, 0) << "WARNING: no connection to metadata master zone, "
"can't resolve entrypoint for bucket=" << bucket << dendl;
return set_cr_error(-ECANCELED);
}
std::string key = rgw_bucket(bucket.tenant, bucket.name).get_key();
rgw_http_param_pair params[] = { { "key", key.c_str() }, { nullptr, nullptr } };
call(new RGWReadRESTResourceCR<RGWBucketEntrypointMetadataResult>(
cct, master_conn, http, "/admin/metadata/bucket", params, &master_result));
}
if (retcode == -ENOENT) {
bucket_id->clear();
return set_cr_done();
}
if (retcode < 0) {
return set_cr_error(retcode);
}
*bucket_id = master_result.data.bucket.bucket_id;
return set_cr_done();
}
return 0;
}
};
/// trim the bilog of all of the given bucket instance's shards
class BucketTrimInstanceCR : public RGWCoroutine {
static constexpr auto MAX_RETRIES = 25u;
@ -527,6 +620,7 @@ class BucketTrimInstanceCR : public RGWCoroutine {
std::string bucket_instance;
rgw_bucket_get_sync_policy_params get_policy_params;
std::shared_ptr<rgw_bucket_get_sync_policy_result> source_policy;
std::string cur_entrypoint_bucket_id;
rgw_bucket bucket;
const std::string& zone_id; //< my zone id
RGWBucketInfo _bucket_info;
@ -561,7 +655,8 @@ class BucketTrimInstanceCR : public RGWCoroutine {
}
if (min_generation == UINT64_MAX) {
// if all peers have deleted this bucket, purge the rest of our log generations
// no peer is syncing this bucket (all returned -ENOENT); this alone does
// not confirm deletion: we will check the entrypoint before cleanup.
totrim.gen = UINT64_MAX;
return 0;
}
@ -581,24 +676,48 @@ class BucketTrimInstanceCR : public RGWCoroutine {
return 0;
}
// every peer returned -ENOENT for this bucket's sync status (no peer is
// syncing it). NOTE: this alone does not confirm deletion. A live bucket
// whose sync status is not yet initialized looks the same; the entrypoint
// check in operate() disambiguates.
bool all_peers_report_bucket_gone() const {
return totrim.gen == UINT64_MAX;
}
// the local layout already carries the Deleted flag, i.e. metadata sync has
// delivered the deletion to this zone. That is authoritative on its own, so
// the normal Deleted-driven cleanup applies and we can skip the entrypoint
// round-trip to the metadata master.
bool locally_confirmed_deleted() const {
return pbucket_info->layout.logs.back().layout.type == rgw::BucketLogType::Deleted;
}
/// If there is a generation below the minimum, prepare to clean it up.
int maybe_remove_generation() {
if (clean_info)
return 0;
bool deleted_type = (pbucket_info->layout.logs.back().layout.type == rgw::BucketLogType::Deleted);
bool deleted_type = locally_confirmed_deleted();
if (pbucket_info->layout.logs.front().gen < totrim.gen ||
(pbucket_info->layout.logs.front().gen <= totrim.gen && deleted_type)) {
clean_info = {*pbucket_info, {}};
auto log = clean_info->first.layout.logs.cbegin();
clean_info->second = *log;
// the local zone still sees this bucket as a normal live bucket
// with one generation and no Deleted flag.
if (clean_info->first.layout.logs.size() == 1 && !deleted_type) {
ldpp_dout(dpp, -1)
<< "Critical error! Attempt to remove only log generation! "
<< "log.gen=" << log->gen << ", totrim.gen=" << totrim.gen
<< dendl;
return -EIO;
if (!all_peers_report_bucket_gone()) { // unexpected: a peer reported a real generation but local has only one
ldpp_dout(dpp, -1)
<< "Critical error! Attempt to remove only log generation! "
<< "log.gen=" << log->gen << ", totrim.gen=" << totrim.gen
<< dendl;
return -EIO;
}
ldpp_dout(dpp, 10) << "bucket=" << bucket_instance
<< " all peers gone, deferring instance removal" << dendl;
clean_info = std::nullopt;
return 0;
}
clean_info->first.layout.logs.erase(log);
}
@ -687,7 +806,7 @@ inline int parse_decode_json<StatusShards>(
int BucketTrimInstanceCR::operate(const DoutPrefixProvider *dpp)
{
reenter(this) {
ldpp_dout(dpp, 4) << "starting trim on bucket=" << bucket_instance << dendl;
ldpp_dout(dpp, 1) << "starting trim on bucket=" << bucket_instance << dendl;
get_policy_params.zone = zone_id;
get_policy_params.bucket = bucket;
@ -736,7 +855,7 @@ int BucketTrimInstanceCR::operate(const DoutPrefixProvider *dpp)
auto p = peer_status.begin();
for (auto& zid : zids) {
spawn(new RGWReadRemoteStatusShardsCR(dpp, store, cct, http, pbucket_info, bucket_instance, zid, zone_id, &*p), false);
spawn(new RGWReadRemoteStatusShardsCR(dpp, store, cct, http, bucket_instance, zid, zone_id, &*p), false);
++p;
}
}
@ -756,6 +875,35 @@ int BucketTrimInstanceCR::operate(const DoutPrefixProvider *dpp)
ldpp_dout(dpp, 4) << "failed to find minimum generation" << dendl;
return set_cr_error(retcode);
}
// When all peers report -ENOENT the bucket may be deleted, or it may be a
// live bucket whose sync status is not yet initialized (both look the same).
// But, if the local layout already carries the Deleted flag, metadata sync has
// delivered the deletion and that is authoritative on its own; i.e., go through
// to the normal Deleted-driven cleanup below. Otherwise, confirm real deletion
// by resolving the authoritative entrypoint before touching the instance
// metadata; only an absent (or repointed) entrypoint means this instance is
// truly gone.
if (all_peers_report_bucket_gone() && !locally_confirmed_deleted()) {
yield call(new RGWReadMasterBucketEntrypointCR(store, http, bucket,
&cur_entrypoint_bucket_id));
if (retcode < 0) {
ldpp_dout(dpp, 4) << "bucket=" << bucket_instance
<< " could not confirm deletion via entrypoint: " << cpp_strerror(retcode)
<< ", skipping to avoid removing a live bucket" << dendl;
return set_cr_done();
}
if (cur_entrypoint_bucket_id == pbucket_info->bucket.bucket_id) {
// entrypoint still resolves to this instance: the bucket is live and its
// sync status must be uninitialized, not deleted. Leave it alone.
ldpp_dout(dpp, 10) << "bucket=" << bucket_instance
<< " still has a live entrypoint, skipping (not deleted)" << dendl;
return set_cr_done();
}
ldpp_dout(dpp, 10) << "bucket=" << bucket_instance
<< " entrypoint absent or repointed, treating as deleted" << dendl;
}
retcode = maybe_remove_generation();
if (retcode < 0) {
ldpp_dout(dpp, 4) << "error removing old generation from log: "
@ -763,6 +911,28 @@ int BucketTrimInstanceCR::operate(const DoutPrefixProvider *dpp)
return set_cr_error(retcode);
}
// The no-Deleted-flag orphan path (#70858): all peers are gone, the
// entrypoint check above confirmed the bucket is deleted, and there's no
// generation cleanup pending. This is only reachable when the Deleted flag
// never arrived. What remains is a single InIndex generation, so remove the
// orphaned instance metadata directly.
if (all_peers_report_bucket_gone() && !clean_info) {
ldpp_dout(dpp, 1) << "all peers report bucket gone, removing "
<< "orphaned bucket instance metadata" << dendl;
_bucket_info = *pbucket_info;
yield call(new RGWRemoveBucketInstanceInfoCR(
store->svc()->async_processor,
store, _bucket_info.bucket,
_bucket_info, nullptr, dpp));
if (retcode < 0 && retcode != -ENOENT) {
ldpp_dout(dpp, 0) << "failed to remove instance bucket info: "
<< cpp_strerror(retcode) << dendl;
return set_cr_error(retcode);
}
observer->on_bucket_trimmed(std::move(bucket_instance));
return set_cr_done();
}
if (clean_info) {
if (clean_info->second.layout.type != rgw::BucketLogType::InIndex) {
ldpp_dout(dpp, 0) << "Unable to convert log of unknown type "
@ -1212,7 +1382,7 @@ int BucketTrimCR::operate(const DoutPrefixProvider *dpp)
// trim bucket instances with limited concurrency
set_status("trimming buckets");
ldpp_dout(dpp, 4) << "collected " << buckets.size() << " buckets for trim" << dendl;
ldpp_dout(dpp, 1) << "collected " << buckets.size() << " buckets for trim" << dendl;
yield call(new BucketTrimInstanceCollectCR(store, http, observer, buckets,
config.concurrent_buckets, dpp));
// ignore errors from individual buckets
@ -1247,7 +1417,7 @@ int BucketTrimCR::operate(const DoutPrefixProvider *dpp)
return set_cr_error(retcode);
}
ldpp_dout(dpp, 4) << "bucket index log processing completed in "
ldpp_dout(dpp, 1) << "bucket index log processing completed in "
<< ceph::mono_clock::now() - start_time << dendl;
return set_cr_done();
}

208
src/test/rgw/rgw_multi/tests.py
View File

@ -2200,6 +2200,214 @@ def test_bucket_log_trim_after_delete_bucket_secondary_reshard():
assert check_bucket_instance_metadata(zone.zone, test_bucket.name)
@attr('bucket_trim')
def test_bucket_log_trim_enoent_race_after_reshard():
zonegroup = realm.master_zonegroup()
zonegroup_conns = ZonegroupConns(zonegroup)
primary = zonegroup_conns.rw_zones[0]
secondary = zonegroup_conns.rw_zones[1]
all_clusters = set()
for zg in realm.current_period.zonegroups:
for zone in zg.zones:
all_clusters.add(zone.cluster)
def set_meta_sync_delay(delay_sec):
for cluster in all_clusters:
if delay_sec > 0:
cluster.ceph_admin(
['config', 'set', 'client', 'rgw_inject_delay_sec', str(delay_sec)])
cluster.ceph_admin(
['config', 'set', 'client', 'rgw_inject_delay_pattern', 'delay_meta_sync_bucket_instance_store'])
else:
cluster.ceph_admin(
['config', 'rm', 'client', 'rgw_inject_delay_sec'])
cluster.ceph_admin(
['config', 'rm', 'client', 'rgw_inject_delay_pattern'])
def make_test_bucket():
name = gen_bucket_name()
log.info('create bucket zone=%s name=%s', primary.zone.name, name)
bucket = primary.create_bucket(name)
for objname in ('a', 'b', 'c', 'd'):
primary.s3_client.put_object(Bucket=bucket.name, Key=objname, Body='foo')
zonegroup_meta_checkpoint(zonegroup)
zonegroup_bucket_checkpoint(zonegroup_conns, name)
return bucket
test_bucket = make_test_bucket()
# reshard on secondary to create a generation mismatch
secondary.zone.cluster.admin(['bucket', 'reshard',
'--bucket', test_bucket.name,
'--num-shards', '13',
'--yes-i-really-mean-it'] + secondary.zone.zone_args())
for obj in ('a', 'b', 'c', 'd'):
cmd = ['object', 'rm'] + primary.zone.zone_args()
cmd += ['--bucket', test_bucket.name]
cmd += ['--object', obj]
primary.zone.cluster.admin(cmd + primary.zone.zone_args())
log.info('setting metadata sync delay to reproduce ENOENT trim race')
set_meta_sync_delay(30)
time.sleep(10) # let the delay config reach the radosgws before deleting
try:
primary.s3_client.delete_bucket(Bucket=test_bucket.name)
zonegroup_data_checkpoint(zonegroup_conns)
# run autotrim on primary first — primary has the Deleted flag
# so it will fully clean up including removing its own
# bucket.instance metadata
bilog_autotrim(primary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
bilog_autotrim(primary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
# now autotrim on secondary — secondary queries primary, but
# primary's instance metadata is gone so primary responds -ENOENT.
# secondary doesn't have the Deleted flag yet (metadata sync is
# stalled). In #70858, this leaves StatusShards{gen=0, shards=[]}
# causing take_min_status() to fail with -EINVAL.
bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
finally:
log.info('removing metadata sync delay')
set_meta_sync_delay(0)
for zonegroup in realm.current_period.zonegroups:
zonegroup_conns = ZonegroupConns(zonegroup)
zonegroup_meta_checkpoint(zonegroup)
for zone in zonegroup_conns.zones:
log.info('trimming on zone=%s', zone.name)
bilog_autotrim(zone.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'],)
time.sleep(config.checkpoint_delay)
for zonegroup in realm.current_period.zonegroups:
zonegroup_conns = ZonegroupConns(zonegroup)
for zone in zonegroup_conns.zones:
assert check_bucket_instance_metadata(zone.zone, test_bucket.name)
@attr('bucket_trim')
def test_bucket_log_trim_live_empty_bucket_not_removed():
# A live bucket with no data written yet also returns -ENOENT for its bucket sync
# status on every peer as sync status is not initialized until data flows.
# autotrim must NOT mistake that for a deletion and remove the bucket
# instance metadata. This is a guard testcase for the -ENOENT disambiguation
# (tracker #70858): i.e., if the entrypoint still exists, so the bucket is live.
zonegroup = realm.master_zonegroup()
zonegroup_conns = ZonegroupConns(zonegroup)
primary = zonegroup_conns.rw_zones[0]
# create a bucket but write no objects, so no bucket sync status is created
name = gen_bucket_name()
log.info('create empty bucket zone=%s name=%s', primary.zone.name, name)
bucket = primary.create_bucket(name)
# make sure the entrypoint and instance metadata reach every zone
zonegroup_meta_checkpoint(zonegroup)
# run autotrim everywhere; peers return -ENOENT for sync status, but the
# bucket is live (entrypoint present) and must be left intact.
for zg in realm.current_period.zonegroups:
zg_conns = ZonegroupConns(zg)
for zone in zg_conns.zones:
log.info('trimming on zone=%s', zone.name)
bilog_autotrim(zone.zone, ['--rgw-sync-log-trim-max-buckets', '50'])
time.sleep(config.checkpoint_delay)
# the bucket instance metadata must still be present on every zone.
# check_bucket_instance_metadata() returns False when the instance is still
# present (i.e. it was NOT trimmed away).
for zg in realm.current_period.zonegroups:
zg_conns = ZonegroupConns(zg)
for zone in zg_conns.zones:
assert not check_bucket_instance_metadata(zone.zone, bucket.name), \
'live bucket %s instance wrongly removed on zone %s' % (bucket.name, zone.name)
@attr('bucket_trim')
def test_bucket_log_trim_new_bucket_entrypoint_not_synced():
# A bucket freshly created on the primary syncs its INSTANCE metadata to the
# secondary before its ENTRYPOINT (creation writes the instance first, see
# put_linked_bucket_info() in rgw_rados.cc). If autotrim runs on the secondary
# in that window, every peer returns -ENOENT (new, data-less bucket => sync
# status uninitialized). autotrim must confirm the deletion against the
# metadata master (which already has the entrypoint) rather than the secondary's
# not-yet-synced local copy, and so must leave the live bucket's instance intact.
#
# Force the window described above by delaying entrypoint ("bucket" section) metadata
# sync on the secondary while letting the instance ("bucket.instance") go through.
zonegroup = realm.master_zonegroup()
zonegroup_conns = ZonegroupConns(zonegroup)
primary = zonegroup_conns.rw_zones[0]
secondary = zonegroup_conns.rw_zones[1]
def set_entrypoint_sync_delay(delay_sec):
cluster = secondary.zone.cluster
if delay_sec > 0:
cluster.ceph_admin(['config', 'set', 'client', 'rgw_inject_delay_sec', str(delay_sec)])
cluster.ceph_admin(['config', 'set', 'client', 'rgw_inject_delay_pattern', 'delay_meta_sync_bucket_entrypoint_store'])
else:
cluster.ceph_admin(['config', 'rm', 'client', 'rgw_inject_delay_sec'])
cluster.ceph_admin(['config', 'rm', 'client', 'rgw_inject_delay_pattern'])
# metadata list is per-zone, so this reads the secondary's LOCAL entrypoints
def secondary_has_entrypoint(name):
cmd = ['metadata', 'list', 'bucket'] + secondary.zone.zone_args()
out, _ = secondary.zone.cluster.admin(cmd, check_retcode=False, read_only=True)
try:
return name in json.loads(out)
except ValueError:
return False
with override_config(checkpoint_retries=30, checkpoint_delay=2):
set_entrypoint_sync_delay(config.checkpoint_retries * config.checkpoint_delay)
try:
time.sleep(10) # let the delay config reach the radosgws before creating
name = gen_bucket_name()
log.info('create bucket zone=%s name=%s (no objects)', primary.zone.name, name)
primary.create_bucket(name)
# wait until the INSTANCE has synced to the secondary
# while its ENTRYPOINT is still delayed (absent)
window_hit = False
for _ in range(config.checkpoint_retries):
time.sleep(config.checkpoint_delay)
inst_present = not check_bucket_instance_metadata(secondary.zone, name)
ep_present = secondary_has_entrypoint(name)
log.info('secondary sync state: instance=%s entrypoint=%s', inst_present, ep_present)
if inst_present and not ep_present:
window_hit = True
break
if inst_present and ep_present:
break # entrypoint arrived too; window missed
assert window_hit, \
'could not reproduce instance-synced-but-entrypoint-not window on secondary'
# autotrim on the secondary while its local entrypoint is still absent
log.info('running autotrim on secondary while entrypoint is not yet synced')
bilog_autotrim(secondary.zone, ['--rgw-sync-log-trim-max-buckets', '50'])
time.sleep(config.checkpoint_delay)
# the instance metadata must NOT have been removed since the bucket is live
assert not check_bucket_instance_metadata(secondary.zone, name), \
'live (mid-sync) bucket %s instance wrongly removed on secondary zone' % name
finally:
set_entrypoint_sync_delay(0)
# let metadata sync catch up so the entrypoint lands and state is consistent
zonegroup_meta_checkpoint(zonegroup)
@attr('bucket_reshard')
def test_bucket_reshard_incremental():
zonegroup = realm.master_zonegroup()