diff --git a/src/client/cluster_client.h b/src/client/cluster_client.h index 05d218d7..59c35ab2 100644 --- a/src/client/cluster_client.h +++ b/src/client/cluster_client.h @@ -136,8 +136,9 @@ public: bool get_immediate_commit(uint64_t inode); void continue_ops(int time_passed = 0); + // FIXME: list_inode_start/list_inode_next - not an ideal interface :) inode_list_t *list_inode_start(inode_t inode, - std::function&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback); + std::function&& objects, pg_num_t pg_num, osd_num_t primary_osd, int errcode, int status)> callback); int list_pg_count(inode_list_t *lst); const std::vector & list_inode_get_inactive_osds(inode_list_t *lst); const std::vector & list_inode_get_inactive_pgs(inode_list_t *lst); diff --git a/src/client/cluster_client_list.cpp b/src/client/cluster_client_list.cpp index 2383aefa..214d66ce 100644 --- a/src/client/cluster_client_list.cpp +++ b/src/client/cluster_client_list.cpp @@ -20,6 +20,7 @@ struct inode_list_pg_t { inode_list_t *lst = NULL; int pos = 0; + int errcode = 0; pg_num_t pg_num; osd_num_t cur_primary; bool has_unstable = false; @@ -40,11 +41,11 @@ struct inode_list_t std::vector inactive_osds; std::vector inactive_pgs; std::vector pgs; - std::function&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback; + std::function&& objects, pg_num_t pg_num, osd_num_t primary_osd, int errcode, int status)> callback; }; inode_list_t* cluster_client_t::list_inode_start(inode_t inode, - std::function&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback) + std::function&& objects, pg_num_t pg_num, osd_num_t primary_osd, int errcode, int status)> callback) { init_msgr(); pool_id_t pool_id = INODE_POOL(inode); @@ -211,6 +212,7 @@ void cluster_client_t::continue_listing(inode_list_t *lst) } else if (!lst->pgs[i]->list_osds.size()) { + lst->pgs[i]->errcode = -EIO; finish_list_pg(lst->pgs[i]); if (check_finish_listing(lst)) { @@ -259,6 +261,7 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list) { fprintf(stderr, "Failed to get PG %u/%u object list from OSD %ju (retval=%jd), skipping\n", cur_list->pg->lst->pool_id, cur_list->pg->pg_num, cur_list->osd_num, op->reply.hdr.retval); + cur_list->pg->errcode = op->reply.hdr.retval; } else { @@ -317,7 +320,7 @@ void cluster_client_t::finish_list_pg(inode_list_pg_t *pg) status |= INODE_LIST_HAS_UNSTABLE; } lst->pgs[pg->pos] = NULL; - lst->callback(lst, std::move(pg->objects), pg->pg_num, pg->cur_primary, status); + lst->callback(lst, std::move(pg->objects), pg->pg_num, pg->cur_primary, pg->errcode, status); delete pg; } else diff --git a/src/cmd/cli_merge.cpp b/src/cmd/cli_merge.cpp index 65677e92..bcb6f93e 100644 --- a/src/cmd/cli_merge.cpp +++ b/src/cmd/cli_merge.cpp @@ -51,6 +51,7 @@ struct snap_merger_t btree::safe_btree_set merge_offsets; btree::safe_btree_set::iterator oit; std::map> layer_lists; + std::map list_errcode; std::map layer_block_size; std::map layer_list_pos; std::vector continue_rwo, continue_rwo2; @@ -251,6 +252,7 @@ struct snap_merger_t // Get parents and so on start_merge(); // First list lower layers + list_errcode.clear(); list_layers(true); state = 1; resume_1: @@ -259,6 +261,15 @@ struct snap_merger_t // Wait for lists return; } + if (list_errcode.size()) + { + result = (cli_result_t){ + .err = EIO, + .text = "Failed to list lower layer(s) in some PGs, merging would be incorrect", + }; + state = 100; + return; + } if (merge_offsets.size() > 0) { state = 2; @@ -295,6 +306,7 @@ struct snap_merger_t state = 3; resume_3: // Then list upper layers + list_errcode.clear(); list_layers(false); state = 4; resume_4: @@ -303,6 +315,15 @@ struct snap_merger_t // Wait for lists return; } + if (list_errcode.size() > 0) + { + result = (cli_result_t){ + .err = EIO, + .text = "Failed to list upper layer(s) in some PGs, merging would be incorrect", + }; + state = 100; + return; + } state = 5; processed = 0; to_process = merge_offsets.size(); @@ -369,8 +390,12 @@ struct snap_merger_t { lists_todo++; inode_list_t* lst = parent->cli->list_inode_start(src, [this, src]( - inode_list_t *lst, std::set&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status) + inode_list_t *lst, std::set&& objects, pg_num_t pg_num, osd_num_t primary_osd, int errcode, int status) { + if (errcode) + { + list_errcode[src] = errcode; + } uint64_t layer_block = layer_block_size.at(src); for (object_id obj: objects) { @@ -394,9 +419,15 @@ struct snap_merger_t if (status & INODE_LIST_DONE) { auto & name = parent->cli->st_cli.inode_config.at(src).name; - if (parent->progress) + if (list_errcode.find(src) != list_errcode.end()) { - printf("Got listing of layer %s (inode %ju in pool %u)\n", name.c_str(), INODE_NO_POOL(src), INODE_POOL(src)); + fprintf(stderr, "Failed to get listing of layer %s (inode %ju in pool %u): %s (code %d)\n", + name.c_str(), INODE_NO_POOL(src), INODE_POOL(src), strerror(-list_errcode[src]), list_errcode[src]); + } + else if (parent->progress) + { + fprintf(stderr, "Got listing of layer %s (inode %ju in pool %u)\n", + name.c_str(), INODE_NO_POOL(src), INODE_POOL(src)); } if (delete_source) { @@ -428,7 +459,7 @@ struct snap_merger_t { if (op->retval < 0) { - fprintf(stderr, "error reading target bitmap at offset %jx: %s\n", op->offset, strerror(-op->retval)); + fprintf(stderr, "Warning: failed to read target bitmap at offset %jx: %s\n", op->offset, strerror(-op->retval)); } else { diff --git a/src/cmd/cli_rm_data.cpp b/src/cmd/cli_rm_data.cpp index 6a89368f..8ab6aca6 100644 --- a/src/cmd/cli_rm_data.cpp +++ b/src/cmd/cli_rm_data.cpp @@ -32,7 +32,7 @@ struct rm_inode_t inode_list_t *lister = NULL; std::vector lists; std::vector inactive_osds; - std::vector inactive_pgs; + std::set inactive_pgs; uint64_t total_count = 0, total_done = 0, total_prev_pct = 0; uint64_t pgs_to_list = 0; bool lists_done = false; @@ -44,8 +44,12 @@ struct rm_inode_t void start_delete() { lister = parent->cli->list_inode_start(inode, [this](inode_list_t *lst, - std::set&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status) + std::set&& objects, pg_num_t pg_num, osd_num_t primary_osd, int errcode, int status) { + if (errcode) + { + inactive_pgs.insert(pg_num); + } rm_pg_t *rm = new rm_pg_t((rm_pg_t){ .pg_num = pg_num, .rm_osd_num = primary_osd, @@ -102,17 +106,6 @@ struct rm_inode_t } fprintf(stderr, "\n"); } - auto inactive_pgs = parent->cli->list_inode_get_inactive_pgs(lister); - if (inactive_pgs.size() && !parent->json_output) - { - fprintf(stderr, "Some data may remain after delete in PGs which are currently inactive: "); - for (int i = 0; i < inactive_pgs.size(); i++) - { - this->inactive_pgs.push_back((uint64_t)inactive_pgs[i]); - fprintf(stderr, i > 0 ? ", %u" : "%u", inactive_pgs[i]); - } - fprintf(stderr, "\n"); - } pgs_to_list = parent->cli->list_pg_count(lister); parent->cli->list_inode_next(lister, parent->parallel_osds); } @@ -237,6 +230,16 @@ struct rm_inode_t { fprintf(stderr, "\n"); } + if (inactive_pgs.size() && !parent->json_output) + { + fprintf(stderr, "Failed to list some PGs, deletion is not complete: PG "); + int i = 0; + for (auto pg_num: inactive_pgs) + { + fprintf(stderr, (i++) > 0 ? ", %u" : "%u", pg_num); + } + fprintf(stderr, "\n"); + } bool is_error = (total_done < total_count || inactive_osds.size() > 0 || inactive_pgs.size() > 0 || error_count > 0); if (parent->progress && is_error) { @@ -246,6 +249,11 @@ struct rm_inode_t pool_id, INODE_NO_POOL(inode), pool_id, INODE_NO_POOL(inode) ); } + json11::Json::array inactive_pgs_json; + for (auto pg_num: inactive_pgs) + { + inactive_pgs_json.push_back((uint64_t)pg_num); + } result = (cli_result_t){ .err = is_error && !down_ok ? EIO : 0, .text = is_error ? "Some blocks were not removed" : ( @@ -255,7 +263,7 @@ struct rm_inode_t { "removed_objects", total_done }, { "total_objects", total_count }, { "inactive_osds", inactive_osds }, - { "inactive_pgs", inactive_pgs }, + { "inactive_pgs", inactive_pgs_json }, }, }; state = 100;