diff --git a/qa/suites/fs/libcephfs/tasks/client_full.yaml b/qa/suites/fs/libcephfs/tasks/client_full.yaml new file mode 100644 index 00000000000..a7e8b1e33f5 --- /dev/null +++ b/qa/suites/fs/libcephfs/tasks/client_full.yaml @@ -0,0 +1,38 @@ +overrides: + ceph: + cephfs: + ec_profile: + - disabled + log-ignorelist: + - OSD full dropping all updates + - OSD near full + - pausewr flag + - failsafe engaged, dropping updates + - failsafe disengaged, no longer dropping + - is full \(reached quota + - POOL_FULL + - POOL_BACKFILLFULL + - PG_RECOVERY_FULL + - PG_DEGRADED + conf: + mon: + mon osd nearfull ratio: 0.6 + mon osd backfillfull ratio: 0.6 + mon osd full ratio: 0.7 + osd: + osd mon report interval: 5 + osd objectstore: memstore + osd failsafe full ratio: 1.0 + memstore device bytes: 200000000 + client: + debug client: 20 + debug objecter: 20 + debug objectcacher: 20 + mds: + debug ms: 1 + debug mds: 20 +tasks: +- workunit: + clients: + client.0: + - client/test_full.sh diff --git a/qa/workunits/client/test_full.sh b/qa/workunits/client/test_full.sh new file mode 100755 index 00000000000..e562383e0e7 --- /dev/null +++ b/qa/workunits/client/test_full.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -ex + +ceph_test_client_async_full diff --git a/src/test/client/CMakeLists.txt b/src/test/client/CMakeLists.txt index 8b1f4a8f782..46892188777 100644 --- a/src/test/client/CMakeLists.txt +++ b/src/test/client/CMakeLists.txt @@ -40,4 +40,21 @@ if(${WITH_CEPHFS}) ) install(TARGETS ceph_test_client_fscrypt DESTINATION ${CMAKE_INSTALL_BINDIR}) + + add_executable(ceph_test_client_async_full + main.cc + fscrypt_conf.cc + nonblocking_full.cc + ) + target_link_libraries(ceph_test_client_async_full + client + global + ceph-common + cephfs + ${UNITTEST_LIBS} + ${EXTRALIBS} + ${CMAKE_DL_LIBS} + ) + install(TARGETS ceph_test_client_async_full + DESTINATION ${CMAKE_INSTALL_BINDIR}) endif(${WITH_CEPHFS}) diff --git a/src/test/client/TestClient.h b/src/test/client/TestClient.h index ad79baa6186..326478f3e5a 100644 --- a/src/test/client/TestClient.h +++ b/src/test/client/TestClient.h @@ -224,6 +224,104 @@ public: #endif return 0; } + + void ll_write_n_bytes(struct Fh *fh, size_t to_write, size_t block_size, + int iov_cnt, off_t *offset) { + /// @brief Write N bytes of data asynchronously. + /// @param fh - File handle + /// @param to_write - bytes to write + /// @param block_size - Total size of each iovec structs array + /// @param iov_cnt - Number of elements in iovec structs array + /// @param offset - location to start writing from + + const size_t DATA_PER_BLOCK = size_t(block_size / iov_cnt); + + // create a single large buffer upto block_size + std::vector buffer(block_size, 'X'); + struct iovec iov_buf_out[iov_cnt]; + for (int i = 0; i < iov_cnt; ++i) { + iov_buf_out[i].iov_base = buffer.data() + (i * DATA_PER_BLOCK); + iov_buf_out[i].iov_len = DATA_PER_BLOCK; + } + + int64_t rc = 0, bytes_written = 0, num_iov = iov_cnt, bytes_expected = block_size; + iovec *input_buffer_ptr = iov_buf_out; + + // small buffer for writing bytes less than block_size + std::vector small_buf; + struct iovec iov_small_buf_out[1]; + + while (to_write > 0) { + if (to_write < block_size) { + // fill the small buffer for writing the remaining bytes + small_buf.assign(to_write, 'Y'); + iov_small_buf_out[0] = {small_buf.data(), to_write}; + input_buffer_ptr = iov_small_buf_out; + bytes_expected = to_write; + num_iov = 1; + } + C_SaferCond writefinish("writefinish-upto-blocksize"); + rc = ll_preadv_pwritev(fh, input_buffer_ptr, num_iov, *offset, + true, &writefinish, nullptr); + ASSERT_EQ(rc, 0); + bytes_written = writefinish.wait(); + ASSERT_EQ(bytes_written, bytes_expected); + *offset += bytes_written; + to_write -= bytes_written; + if (num_iov == 1) { + ASSERT_EQ(to_write, 0); + } + } + } + + bool is_data_pool_full(int64_t data_pool) { + /* check if data pool is reported as full with a specified timeout + and period to sleep */ + return objecter->with_osdmap([&](const OSDMap &o) { + for (const auto& kv : o.get_pools()) { + if (kv.first == data_pool && kv.second.has_flag(pg_pool_t::FLAG_FULL)) { + return true; + } + } + return false; + }); + } + + bool wait_until_true(std::function condition, int32_t timeout = 600, + int32_t period = 5) { + // wait for the condition to be true until timeout + while(true) { + if (condition()) { + return true; + } + timeout -= period; + timeout = std::max(timeout, 0); + if (!timeout) { + return false; + } + sleep(period); + } + } + + bool wait_for_osdmap_epoch_update(const epoch_t initial_osd_epoch, + int32_t timeout = 10, + int32_t period = 5) { + // wait for timeout and check for osdmap epoch increment + + bs::error_code ec; + objecter->wait_for_latest_osdmap(ca::use_blocked[ec]); + ldout(cct, 20) << __func__ << ": latest osdmap: " << ec << dendl; + + while(timeout) { + if (objecter->with_osdmap(std::mem_fn(&OSDMap::get_epoch)) > initial_osd_epoch) { + return true; + } else { + sleep(period); + timeout -= period; + } + } + return false; + } }; class TestClient : public ::testing::Test { diff --git a/src/test/client/nonblocking.cc b/src/test/client/nonblocking.cc index 8a877c6b03b..11b6400b332 100644 --- a/src/test/client/nonblocking.cc +++ b/src/test/client/nonblocking.cc @@ -795,3 +795,470 @@ TEST_F(TestClient, LlreadvLlwritevLargeBuffers) { client->ll_release(fh); ASSERT_EQ(0, client->ll_unlink(root, filename, myperm)); } + +TEST_F(TestClient, LlreadvLlwritevOverlimit) { + /* + POSIX.1 allows an implementation to place a limit on the number of + items that can be passed in iov however going through the libcephfs or + the client code, there is no such limit imposed in cephfs, therefore + async I/O over the limit should succeed. + + For more info: https://manpages.ubuntu.com/manpages/xenial/man2/readv.2.html + */ + + Inode *root = nullptr, *file = nullptr; + Fh *fh = nullptr; + struct ceph_statx stx; + root = client->get_root(); + ASSERT_NE(root, (Inode *)NULL); + + int mypid = getpid(); + char fname[256]; + sprintf(fname, "test_llreadvllwritevoverlimitfile%u", mypid); + ASSERT_EQ(0, client->ll_createx(root, fname, 0666, + O_RDWR | O_CREAT | O_TRUNC, + &file, &fh, &stx, 0, 0, myperm)); + + // setup buffer array + const int IOV_SEG_OVERLIMIT = 1500; + ssize_t bytes_to_write = 0; + struct iovec iov_out_overlimit[IOV_SEG_OVERLIMIT]; + struct iovec iov_in_overlimit[IOV_SEG_OVERLIMIT]; + for(int i = 0; i < IOV_SEG_OVERLIMIT; ++i) { + char out_str[] = "foo"; + + // fill iovec structures for write op + iov_out_overlimit[i].iov_base = out_str; + iov_out_overlimit[i].iov_len = sizeof(out_str); + bytes_to_write += iov_out_overlimit[i].iov_len; + + // set up iovec structures for read op + char in_str[sizeof(out_str)]; + iov_in_overlimit[i].iov_base = in_str; + iov_in_overlimit[i].iov_len = sizeof(in_str); + } + + C_SaferCond writefinish; + C_SaferCond readfinish; + + int64_t rc; + bufferlist bl; + + rc = client->ll_preadv_pwritev(fh, iov_out_overlimit, IOV_SEG_OVERLIMIT, + 0, true, &writefinish, nullptr); + ASSERT_EQ(rc, 0); + ssize_t bytes_written = writefinish.wait(); + ASSERT_EQ(bytes_written, bytes_to_write); + + rc = client->ll_preadv_pwritev(fh, iov_in_overlimit, IOV_SEG_OVERLIMIT, + 0, false, &readfinish, &bl); + ASSERT_EQ(rc, 0); + ssize_t bytes_read = readfinish.wait(); + ASSERT_EQ(bytes_read, bytes_to_write); + copy_bufferlist_to_iovec(iov_in_overlimit, IOV_SEG_OVERLIMIT, &bl, + bytes_read); + + for(int i = 0; i < IOV_SEG_OVERLIMIT; ++i) { + ASSERT_EQ(strncmp((const char*)iov_in_overlimit[i].iov_base, + (const char*)iov_out_overlimit[i].iov_base, + iov_out_overlimit[i].iov_len), 0); + } + + client->ll_release(fh); + ASSERT_EQ(0, client->ll_unlink(root, fname, myperm)); +} + +TEST_F(TestClient, LlreadvLlwritevNonContiguous) { + /* Test writing at non-contiguous memory locations, and make sure read at + the exact locations where the buffers are written and the bytes read should + be equal to the bytes written. */ + + Inode *root = nullptr, *file = nullptr; + Fh *fh = nullptr; + struct ceph_statx stx; + root = client->get_root(); + ASSERT_NE(root, (Inode *)NULL); + + int mypid = getpid(); + char fname[256]; + sprintf(fname, "test_llreadvllwritevnoncontiguousfile%u", mypid); + + ASSERT_EQ(0, client->ll_createx(root, fname, 0666, + O_RDWR | O_CREAT | O_TRUNC, + &file, &fh, &stx, 0, 0, myperm)); + + const int NUM_BUF = 5; + char out_buf_0[] = "hello "; + char out_buf_1[] = "world\n"; + char out_buf_2[] = "Ceph - "; + char out_buf_3[] = "a scalable distributed "; + char out_buf_4[] = "storage system\n"; + + struct iovec iov_out_non_contiguous[NUM_BUF] = { + {out_buf_0, sizeof(out_buf_0)}, + {out_buf_1, sizeof(out_buf_1)}, + {out_buf_2, sizeof(out_buf_2)}, + {out_buf_3, sizeof(out_buf_3)}, + {out_buf_4, sizeof(out_buf_4)} + }; + + char in_buf_0[sizeof(out_buf_0)]; + char in_buf_1[sizeof(out_buf_1)]; + char in_buf_2[sizeof(out_buf_2)]; + char in_buf_3[sizeof(out_buf_3)]; + char in_buf_4[sizeof(out_buf_4)]; + + struct iovec iov_in_non_contiguous[NUM_BUF] = { + {in_buf_0, sizeof(in_buf_0)}, + {in_buf_1, sizeof(in_buf_1)}, + {in_buf_2, sizeof(in_buf_2)}, + {in_buf_3, sizeof(in_buf_3)}, + {in_buf_4, sizeof(in_buf_4)} + }; + + ssize_t bytes_to_write = 0, total_bytes_written = 0, total_bytes_read = 0; + for(int i = 0; i < NUM_BUF; ++i) { + bytes_to_write += iov_out_non_contiguous[i].iov_len; + } + + int64_t rc; + bufferlist bl, tmpbl; + + struct iovec *current_iov = iov_out_non_contiguous; + + for(int i = 0; i < NUM_BUF; ++i) { + C_SaferCond writefinish("test-nonblocking-writefinish-non-contiguous"); + rc = client->ll_preadv_pwritev(fh, current_iov++, 1, i * NUM_BUF * 10, + true, &writefinish, nullptr); + ASSERT_EQ(rc, 0); + rc = writefinish.wait(); + ASSERT_EQ(rc, iov_out_non_contiguous[i].iov_len); + total_bytes_written += rc; + } + ASSERT_EQ(total_bytes_written, bytes_to_write); + + current_iov = iov_in_non_contiguous; + + for(int i = 0; i < NUM_BUF; ++i) { + ssize_t bytes_read = 0; + C_SaferCond readfinish("test-nonblocking-readfinish-non-contiguous"); + rc = client->ll_preadv_pwritev(fh, current_iov++, 1, i * NUM_BUF * 10, + false, &readfinish, &tmpbl); + ASSERT_EQ(rc, 0); + bytes_read = readfinish.wait(); + ASSERT_EQ(bytes_read, iov_out_non_contiguous[i].iov_len); + total_bytes_read += bytes_read; + bl.append(tmpbl); + tmpbl.clear(); + } + ASSERT_EQ(total_bytes_read, bytes_to_write); + + copy_bufferlist_to_iovec(iov_in_non_contiguous, NUM_BUF, &bl, total_bytes_read); + for(int i = 0; i < NUM_BUF; ++i) { + ASSERT_EQ(0, strncmp((const char*)iov_in_non_contiguous[i].iov_base, + (const char*)iov_out_non_contiguous[i].iov_base, + iov_out_non_contiguous[i].iov_len)); + } + + client->ll_release(fh); + ASSERT_EQ(0, client->ll_unlink(root, fname, myperm)); +} + +TEST_F(TestClient, LlreadvLlwritevWriteOnlyFile) { + /* Test async I/O with a file that has only "w" perms.*/ + + Inode *root = nullptr, *file = nullptr; + Fh *fh = nullptr; + struct ceph_statx stx; + root = client->get_root(); + ASSERT_NE(root, (Inode *)NULL); + + int mypid = getpid(); + char fname[256]; + sprintf(fname, "test_llreadvllwritevwriteonlyfile%u", mypid); + ASSERT_EQ(0, client->ll_createx(root, fname, 0666, + O_WRONLY | O_CREAT | O_TRUNC, + &file, &fh, &stx, 0, 0, myperm)); + + char out_buf_0[] = "hello "; + char out_buf_1[] = "world\n"; + struct iovec iov_out[2] = { + {out_buf_0, sizeof(out_buf_0)}, + {out_buf_1, sizeof(out_buf_1)}, + }; + + char in_buf_0[sizeof(out_buf_0)]; + char in_buf_1[sizeof(out_buf_1)]; + struct iovec iov_in[2] = { + {in_buf_0, sizeof(in_buf_0)}, + {in_buf_1, sizeof(in_buf_1)}, + }; + + ssize_t bytes_to_write = iov_out[0].iov_len + iov_out[1].iov_len; + + C_SaferCond writefinish; + C_SaferCond readfinish; + + int64_t rc; + bufferlist bl; + + rc = client->ll_preadv_pwritev(fh, iov_out, 2, 0, true, &writefinish, + nullptr); + ASSERT_EQ(rc, 0); + ssize_t total_bytes_written = writefinish.wait(); + ASSERT_EQ(total_bytes_written, bytes_to_write); + + rc = client->ll_preadv_pwritev(fh, iov_in, 2, 0, false, &readfinish, + &bl); + ASSERT_EQ(rc, 0); + ssize_t total_bytes_read = readfinish.wait(); + ASSERT_EQ(total_bytes_read, -EBADF); + ASSERT_EQ(bl.length(), 0); + + client->ll_release(fh); + ASSERT_EQ(0, client->ll_unlink(root, fname, myperm)); +} + +TEST_F(TestClient, LlreadvLlwritevFsync) { + /*Test two scenarios: + a) async I/O with fsync enabled and sync metadata+data + b) asynx I/O with fsync enabled and sync data only */ + + Inode *root = nullptr, *file = nullptr; + Fh *fh = nullptr; + struct ceph_statx stx; + root = client->get_root(); + ASSERT_NE(root, (Inode *)NULL); + + int mypid = getpid(); + char fname[256]; + sprintf(fname, "test_llreadvllwritevfsyncfile%u", mypid); + ASSERT_EQ(0, client->ll_createx(root, fname, 0666, + O_RDWR | O_CREAT | O_TRUNC, + &file, &fh, &stx, 0, 0, myperm)); + + char out_buf_0[] = "hello "; + char out_buf_1[] = "world b is much longer\n"; + + char in_buf_0[sizeof(out_buf_0)]; + char in_buf_1[sizeof(out_buf_1)]; + + struct iovec iov_out_fsync_sync_all[2] = { + {out_buf_0, sizeof(out_buf_0)}, + {out_buf_1, sizeof(out_buf_1)}, + }; + + struct iovec iov_in_fsync_sync_all[2] = { + {in_buf_0, sizeof(in_buf_0)}, + {in_buf_1, sizeof(in_buf_1)}, + }; + + struct iovec iov_out_dataonly_fysnc[2] = { + {out_buf_0, sizeof(out_buf_0)}, + {out_buf_1, sizeof(out_buf_1)} + }; + + struct iovec iov_in_dataonly_fysnc[2] = { + {in_buf_0, sizeof(in_buf_0)}, + {in_buf_1, sizeof(in_buf_1)} + }; + + // fsync - true, syncdataonly - false + C_SaferCond writefinish_fsync("test-nonblocking-writefinish-fsync-sync-all"); + C_SaferCond readfinish_fsync("test-nonblocking-readfinish-fsync-sync-all"); + + int64_t rc; + bufferlist bl; + ssize_t bytes_to_write = 0, bytes_written = 0, bytes_read = 0; + + bytes_to_write = iov_out_fsync_sync_all[0].iov_len + + iov_out_fsync_sync_all[1].iov_len; + + rc = client->ll_preadv_pwritev(fh, iov_out_fsync_sync_all, 2, 1000, true, + &writefinish_fsync, nullptr, true, false); + ASSERT_EQ(rc, 0); + bytes_written = writefinish_fsync.wait(); + ASSERT_EQ(bytes_written, bytes_to_write); + + rc = client->ll_preadv_pwritev(fh, iov_in_fsync_sync_all, 2, 1000, false, + &readfinish_fsync, &bl); + ASSERT_EQ(rc, 0); + bytes_read = readfinish_fsync.wait(); + ASSERT_EQ(bytes_read, bytes_to_write); + + copy_bufferlist_to_iovec(iov_in_fsync_sync_all, 2, &bl, bytes_read); + for(int i = 0 ; i < 2; ++i) { + ASSERT_EQ(strncmp((const char*)iov_in_fsync_sync_all[i].iov_base, + (const char*)iov_out_fsync_sync_all[i].iov_base, + iov_out_fsync_sync_all[i].iov_len), 0); + } + + // fsync - true, syncdataonly - true + C_SaferCond writefinish_fsync_data_only("test-nonblocking-writefinish-fsync-syncdataonly"); + C_SaferCond readfinish_fsync_data_only("test-nonblocking-readfinish-fsync-syndataonly"); + + + bytes_to_write = iov_out_dataonly_fysnc[0].iov_len + + iov_out_dataonly_fysnc[1].iov_len; + + rc = client->ll_preadv_pwritev(fh, iov_out_dataonly_fysnc, 2, 100, + true, &writefinish_fsync_data_only, + nullptr, true, true); + ASSERT_EQ(rc, 0); + bytes_written = writefinish_fsync_data_only.wait(); + ASSERT_EQ(bytes_written, bytes_to_write); + + bl.clear(); + rc = client->ll_preadv_pwritev(fh, iov_in_dataonly_fysnc, 2, 100, + false, &readfinish_fsync_data_only, &bl); + ASSERT_EQ(rc, 0); + bytes_read = readfinish_fsync_data_only.wait(); + ASSERT_EQ(bytes_read, bytes_to_write); + + copy_bufferlist_to_iovec(iov_in_dataonly_fysnc, 2, &bl, rc); + for(int i = 0; i < 2; ++i) { + ASSERT_EQ(0, strncmp((const char*)iov_in_dataonly_fysnc[i].iov_base, + (const char*)iov_out_dataonly_fysnc[i].iov_base, + iov_out_dataonly_fysnc[i].iov_len)); + } + + client->ll_release(fh); + ASSERT_EQ(0, client->ll_unlink(root, fname, myperm)); +} + +TEST_F(TestClient, LlreadvLlwritevBufferOverflow) { + /* Provide empty read buffers to see how the function behaves + when there is no space to store the data*/ + + Inode *root = nullptr, *file = nullptr; + Fh *fh = nullptr; + struct ceph_statx stx; + root = client->get_root(); + ASSERT_NE(root, (Inode *)NULL); + + int mypid = getpid(); + char fname[256]; + sprintf(fname, "test_llreadvllwritevbufferoverflowfile%u", mypid); + ASSERT_EQ(0, client->ll_createx(root, fname, 0666, + O_RDWR | O_CREAT | O_TRUNC, + &file, &fh, &stx, 0, 0, myperm)); + + char out_buf_0[] = "hello "; + char out_buf_1[] = "world\n"; + struct iovec iov_out[2] = { + {out_buf_0, sizeof(out_buf_0)}, + {out_buf_1, sizeof(out_buf_1)}, + }; + + char in_buf_0[0]; + char in_buf_1[0]; + struct iovec iov_in[2] = { + {in_buf_0, sizeof(in_buf_0)}, + {in_buf_1, sizeof(in_buf_1)}, + }; + + ssize_t bytes_to_write = iov_out[0].iov_len + iov_out[1].iov_len; + + C_SaferCond writefinish; + C_SaferCond readfinish; + + int64_t rc; + bufferlist bl; + + rc = client->ll_preadv_pwritev(fh, iov_out, 2, 0, true, &writefinish, + nullptr); + ASSERT_EQ(rc, 0); + ssize_t bytes_written = writefinish.wait(); + ASSERT_EQ(bytes_written, bytes_to_write); + + rc = client->ll_preadv_pwritev(fh, iov_in, 2, 0, false, &readfinish, + &bl); + ASSERT_EQ(rc, 0); + ssize_t bytes_read = readfinish.wait(); + ASSERT_EQ(bytes_read, 0); + ASSERT_EQ(bl.length(), 0); + + client->ll_release(fh); + ASSERT_EQ(0, client->ll_unlink(root, fname, myperm)); +} + +TEST_F(TestClient, LlreadvLlwritevQuotaFull) { + /*Test that if the max_bytes quota is exceeded, async I/O code path handles + the case gracefully*/ + + Inode *root = nullptr, *diri = nullptr; + struct ceph_statx stx; + root = client->get_root(); + ASSERT_NE(root, nullptr); + + int pid = getpid(); + + char dirname[256]; + sprintf(dirname, "testdirquotaufull%u", pid); + ASSERT_EQ(0, client->ll_mkdirx(root, dirname, 0777, &diri, &stx, 0, 0, + myperm)); + + // set quota.max_bytes + char xattrk[128]; + sprintf(xattrk, "ceph.quota.max_bytes"); + char setxattrv[128], getxattrv[128]; + int32_t len = sprintf(setxattrv, "8388608"); // 8MiB + int64_t rc = client->ll_setxattr(diri, xattrk, setxattrv, len, + CEPH_XATTR_CREATE, myperm); + ASSERT_EQ(rc, 0); + rc = client->ll_getxattr(diri, xattrk, (void *)getxattrv, len, myperm); + ASSERT_EQ(rc, 7); + ASSERT_STREQ(setxattrv, getxattrv); + + // create a file inside the dir + char filename[256]; + Inode *file = nullptr; + Fh *fh = nullptr; + sprintf(filename, "testllreadvllwritevquotafull%u", pid); + ASSERT_EQ(0, client->ll_createx(diri, filename, 0666, + O_RDWR | O_CREAT | O_TRUNC, + &file, &fh, &stx, 0, 0, myperm)); + + // try async I/O of 64MiB + const size_t BLOCK_SIZE = 32 * 1024 * 1024; + auto out_buf_0 = std::make_unique(BLOCK_SIZE); + memset(out_buf_0.get(), 0xDD, BLOCK_SIZE); + auto out_buf_1 = std::make_unique(BLOCK_SIZE); + memset(out_buf_1.get(), 0xFF, BLOCK_SIZE); + + struct iovec iov_out[2] = { + {out_buf_0.get(), BLOCK_SIZE}, + {out_buf_1.get(), BLOCK_SIZE} + }; + + auto in_buf_0 = std::make_unique(sizeof(out_buf_0)); + auto in_buf_1 = std::make_unique(sizeof(out_buf_0)); + struct iovec iov_in[2] = { + {in_buf_0.get(), sizeof(in_buf_0)}, + {in_buf_1.get(), sizeof(in_buf_1)}, + }; + + // write should fail with EDQUOT + C_SaferCond writefinish; + rc = client->ll_preadv_pwritev(fh, iov_out, 2, 0, true, + &writefinish, nullptr); + ASSERT_EQ(rc, 0); + int64_t bytes_written = writefinish.wait(); + ASSERT_EQ(bytes_written, -EDQUOT); + + // since there was no write, nothing sould be read + C_SaferCond readfinish; + bufferlist bl; + rc = client->ll_preadv_pwritev(fh, iov_in, 2, 0, false, + &readfinish, &bl); + ASSERT_EQ(rc, 0); + int64_t bytes_read = readfinish.wait(); + ASSERT_EQ(bytes_read, 0); + ASSERT_EQ(bl.length(), 0); + + client->ll_release(fh); + ASSERT_EQ(client->ll_unlink(diri, filename, myperm), 0); + + client->ll_rmdir(diri, dirname, myperm); + ASSERT_TRUE(client->ll_put(diri)); +} diff --git a/src/test/client/nonblocking_full.cc b/src/test/client/nonblocking_full.cc new file mode 100644 index 00000000000..ca7bd5b3662 --- /dev/null +++ b/src/test/client/nonblocking_full.cc @@ -0,0 +1,114 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2024 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include + +#include +#include + +#include +#include + +#include "test/client/TestClient.h" + +TEST_F(TestClient, LlreadvLlwritevDataPoolFull) { + /* Test perfoming async I/O after filling the fs and make sure it handles + the write gracefully */ + + Inode *root = nullptr, *file_a = nullptr; + Fh *fh_a = nullptr; + struct ceph_statx stx_a; + root = client->get_root(); + ASSERT_NE(root, (Inode *)NULL); + + int mypid = getpid(); + char fname_a[256]; + sprintf(fname_a, "test_llreadvllwritevdatapoolfullfile_a%u", mypid); + ASSERT_EQ(0, client->ll_createx(root, fname_a, 0666, + O_RDWR | O_CREAT | O_TRUNC, + &file_a, &fh_a, &stx_a, 0, 0, myperm)); + + int64_t rc = 0, bytes_written = 0; + + // this test case cannot handle multiple data pools + const std::vector &data_pools = client->mdsmap->get_data_pools(); + ASSERT_EQ(data_pools.size(), 1); + + struct statvfs stbuf; + rc = client->ll_statfs(root, &stbuf, myperm); + ASSERT_EQ(rc, 0); + // available size = num of free blocks * size of a block + size_t data_pool_available_space = stbuf.f_bfree * stbuf.f_bsize; + ASSERT_GT(data_pool_available_space, 0); + + off_t offset = 0; + // writing blocks of 1GiB + const size_t BLOCK_SIZE = 1024 * 1024 * 1024; + client->ll_write_n_bytes(fh_a, size_t(data_pool_available_space / 2), + BLOCK_SIZE, 4, &offset); + + // get a new file + mypid = getpid(); + char fname_b[256]; + Inode *file_b = nullptr; + Fh *fh_b = nullptr; + struct ceph_statx stx_b; + sprintf(fname_b, "test_llreadvllwritevdatapoolfullfile_b%u", mypid); + ASSERT_EQ(0, client->ll_createx(root, fname_b, 0666, + O_RDWR | O_CREAT | O_TRUNC, + &file_b, &fh_b, &stx_b, 0, 0, myperm)); + + client->ll_write_n_bytes(fh_b, size_t((data_pool_available_space * 1.1) / 2), + BLOCK_SIZE, 4, &offset); + + // if we're here then it means the write succeeded but the cluster is full + // so let us get a new osdmap epoch + const epoch_t osd_epoch = objecter->with_osdmap(std::mem_fn(&OSDMap::get_epoch)); + + objecter->maybe_request_map(); + + // wait till we have a new osdmap epoch + ASSERT_TRUE(client->wait_for_osdmap_epoch_update(osd_epoch)); + + // with the new osdmap epoch, the pools should return full flag + bool data_pool_full = client->wait_until_true([&]() + { return client->is_data_pool_full(data_pools[0]); }); + ASSERT_TRUE(data_pool_full); + + // write here should fail since the cluster is full + const size_t TINY_BLOCK_SIZE = 256 * 1024 * 1024; + auto out_buf_0 = std::make_unique(TINY_BLOCK_SIZE); + memset(out_buf_0.get(), 0xDD, TINY_BLOCK_SIZE); + auto out_buf_1 = std::make_unique(TINY_BLOCK_SIZE); + memset(out_buf_1.get(), 0xFF, TINY_BLOCK_SIZE); + + struct iovec iov_out[2] = { + {out_buf_0.get(), TINY_BLOCK_SIZE}, + {out_buf_1.get(), TINY_BLOCK_SIZE} + }; + + std::unique_ptr writefinish = nullptr; + writefinish.reset(new C_SaferCond("test-nonblocking-writefinish-datapool-full")); + rc = client->ll_preadv_pwritev(fh_b, iov_out, 2, + size_t(data_pool_available_space / 2), + true, writefinish.get(), nullptr); + ASSERT_EQ(rc, 0); + bytes_written = writefinish->wait(); + ASSERT_EQ(bytes_written, -ENOSPC); + + client->ll_release(fh_a); + ASSERT_EQ(0, client->ll_unlink(root, fname_a, myperm)); + client->ll_release(fh_b); + ASSERT_EQ(0, client->ll_unlink(root, fname_b, myperm)); +}