diff --git a/src/cmd/CMakeLists.txt b/src/cmd/CMakeLists.txt index 100b7c27..13e28cfc 100644 --- a/src/cmd/CMakeLists.txt +++ b/src/cmd/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(vitastor_cli STATIC cli_flatten.cpp cli_merge.cpp cli_rm_data.cpp + cli_rm_wildcard.cpp cli_rm.cpp cli_rm_osd.cpp cli_pool_cfg.cpp diff --git a/src/cmd/cli.cpp b/src/cmd/cli.cpp index 00e156ea..a92dfaf4 100644 --- a/src/cmd/cli.cpp +++ b/src/cmd/cli.cpp @@ -54,14 +54,18 @@ static const char* help_text = " -f|--force Proceed with shrinking or setting readwrite flag even if the image has children.\n" " --down-ok Proceed with shrinking even if some data will be left on unavailable OSDs.\n" "\n" - "vitastor-cli rm [] [--writers-stopped] [--down-ok]\n" - " Remove or all layers between and ( must be a child of ),\n" - " rebasing all their children accordingly. --writers-stopped allows merging to be a bit\n" - " more effective in case of a single 'slim' read-write child and 'fat' removed parent:\n" - " the child is merged into parent and parent is renamed to child in that case.\n" - " In other cases parent layers are always merged into children.\n" - " Other options:\n" - " --down-ok Continue deletion/merging even if some data will be left on unavailable OSDs.\n" + "vitastor-cli rm []\n" + "vitastor-cli rm (--exact|--matching) ...\n" + " Remove layer(s) and rebase all their children accordingly.\n" + " In the first form, remove or layers between and its child .\n" + " In the second form, remove all images with exact or pattern-matched names.\n" + " --writers-stopped allows optimised removal in case of a single 'slim' read-write\n" + " child and 'fat' removed parent: the child is merged into parent and parent is renamed\n" + " to child in that case. In other cases parent layers are always merged into children.\n" + " --exact Remove multiple images with names matching given glob patterns.\n" + " --matching Remove multiple images with given names\n" + " --writers-stopped Allow renaming inodes over their read/write children.\n" + " --down-ok Continue deletion/merging even if some data will be left on unavailable OSDs.\n" "\n" "vitastor-cli dd [iimg= | if=] [oimg= | of=] [bs=1M]\n" " [count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress]\n" @@ -276,7 +280,8 @@ static json11::Json::object parse_args(int narg, const char *args[]) !strcmp(opt, "allow-data-loss") || !strcmp(opt, "allow_data_loss") || !strcmp(opt, "down-ok") || !strcmp(opt, "down_ok") || !strcmp(opt, "dry-run") || !strcmp(opt, "dry_run") || - !strcmp(opt, "help") || !strcmp(opt, "all") || !strcmp(opt, "exact") || + !strcmp(opt, "help") || !strcmp(opt, "all") || + !strcmp(opt, "exact") || !strcmp(opt, "matching") || !strcmp(opt, "writers-stopped") || !strcmp(opt, "writers_stopped")) { cfg[opt] = "1"; diff --git a/src/cmd/cli.h b/src/cmd/cli.h index 2c02f2bc..c392c9a5 100644 --- a/src/cmd/cli.h +++ b/src/cmd/cli.h @@ -75,6 +75,7 @@ public: std::function start_rm(json11::Json); std::function start_rm_data(json11::Json); std::function start_rm_osd(json11::Json); + std::function start_rm_wildcard(json11::Json); std::function start_status(json11::Json); std::function start_dd(json11::Json); diff --git a/src/cmd/cli_rm.cpp b/src/cmd/cli_rm.cpp index 277f8aea..1da1f279 100644 --- a/src/cmd/cli_rm.cpp +++ b/src/cmd/cli_rm.cpp @@ -689,6 +689,10 @@ resume_100: std::function cli_tool_t::start_rm(json11::Json cfg) { + if (cfg["exact"].bool_value() || cfg["matching"].bool_value()) + { + return start_rm_wildcard(cfg); + } auto snap_remover = new snap_remover_t(); snap_remover->parent = this; snap_remover->from_name = cfg["from"].string_value(); diff --git a/src/cmd/cli_rm_wildcard.cpp b/src/cmd/cli_rm_wildcard.cpp new file mode 100644 index 00000000..fcd62f11 --- /dev/null +++ b/src/cmd/cli_rm_wildcard.cpp @@ -0,0 +1,208 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include +#include +#include "cli.h" +#include "cluster_client.h" +#include "str_util.h" + +struct inode_rev_t +{ + inode_t inode_num; + uint64_t meta_rev; +}; + +// Remove multiple images in correct order +struct wildcard_remover_t +{ + cli_tool_t *parent; + + json11::Json cfg; + std::vector globs; + bool exact = false; + + json11::Json::array deleted_ids, deleted_images, rebased_images; + std::map chains; // child => parent pairs + std::vector> versioned_chains; + json11::Json::object sub_cfg; + size_t i = 0; + int state = 0; + std::function sub_cb; + + cli_result_t result; + + bool is_done() + { + return state == 100; + } + + void join_chains() + { + bool changed = true; + while (changed) + { + changed = false; + auto ino_it = chains.begin(); + while (ino_it != chains.end()) + { + auto child_id = ino_it->first; + auto parent_id = ino_it->second; + auto & parent_cfg = parent->cli->st_cli.inode_config.at(parent_id); + if (parent_cfg.parent_id) + { + auto chain_it = chains.find(parent_cfg.parent_id); + if (chain_it != chains.end()) + { + changed = true; + ino_it->second = chain_it->second; + chains.erase(chain_it); + } + } + ino_it = chains.upper_bound(child_id); + } + } + // Remember metadata modification revisions to check for parallel changes + versioned_chains.clear(); + for (auto cp: chains) + { + auto child_id = cp.first; + auto parent_id = cp.second; + std::vector ver_chain; + do + { + auto & inode_cfg = parent->cli->st_cli.inode_config.at(child_id); + ver_chain.push_back((inode_rev_t){ .inode_num = child_id, .meta_rev = inode_cfg.mod_revision }); + child_id = inode_cfg.parent_id; + } while (child_id && child_id != parent_id); + versioned_chains.push_back(std::move(ver_chain)); + } + // Sort chains based on parent inode rank to first delete child-most layers + std::map ranks; + for (auto cp: chains) + { + auto parent_id = cp.second, cur_id = parent_id; + uint64_t rank = 0; + do + { + rank++; + cur_id = parent->cli->st_cli.inode_config.at(cur_id).parent_id; + } while (cur_id && cur_id != parent_id); + ranks[parent_id] = rank; + } + std::sort(versioned_chains.begin(), versioned_chains.end(), [&](const std::vector & a, const std::vector & b) + { + return ranks[a.back().inode_num] > ranks[b.back().inode_num]; + }); + } + + void loop() + { + if (state == 0) + goto resume_0; + if (state == 1) + goto resume_1; + else if (state == 100) + goto resume_100; + resume_0: + state = 0; + chains.clear(); + // Select images to delete + for (auto & ic: parent->cli->st_cli.inode_config) + { + for (auto & glob: globs) + { + if (exact ? (ic.second.name == glob) : stupid_glob(ic.second.name, glob)) + { + chains[ic.first] = ic.first; + break; + } + } + } + // Arrange them into chains + join_chains(); + // Delete each chain + i = 0; + while (i < versioned_chains.size()) + { + // Check for parallel changes + for (auto & irev: versioned_chains[i]) + { + auto inode_it = parent->cli->st_cli.inode_config.find(irev.inode_num); + if (inode_it == parent->cli->st_cli.inode_config.end() || + inode_it->second.mod_revision > irev.meta_rev) + { + if (inode_it != parent->cli->st_cli.inode_config.end()) + fprintf(stderr, "Warning: image %s modified by someone else during deletion, restarting wildcard deletion\n", inode_it->second.name.c_str()); + else + fprintf(stderr, "Warning: inode %lx modified by someone else during deletion, retrying wildcard deletion\n", irev.inode_num); + goto resume_0; + } + } + // Delete + { + auto from_cfg = parent->cli->st_cli.inode_config.at(versioned_chains[i].back().inode_num); + auto to_cfg = parent->cli->st_cli.inode_config.at(versioned_chains[i].front().inode_num); + sub_cfg = cfg.object_items(); + sub_cfg.erase("globs"); + sub_cfg.erase("exact"); + sub_cfg["from"] = from_cfg.name; + sub_cfg["to"] = to_cfg.name; + sub_cb = parent->start_rm(sub_cfg); + } +resume_1: + while (!sub_cb(result)) + { + state = 1; + return; + } + sub_cb = NULL; + i++; + merge_result(); + if (result.err) + { + break; + } + } + state = 100; + result.data = json11::Json::object{ + { "deleted_ids", deleted_ids }, + { "deleted_images", deleted_images }, + { "rebased_images", rebased_images }, + }; +resume_100: + // Done + return; + } + + void merge_result() + { + for (auto & item: result.data["deleted_ids"].array_items()) + deleted_ids.push_back(item); + for (auto & item: result.data["deleted_images"].array_items()) + deleted_images.push_back(item == result.data["renamed_to"] ? result.data["renamed_from"] : item); + for (auto & item: result.data["rebased_images"].array_items()) + rebased_images.push_back(item); + } +}; + +std::function cli_tool_t::start_rm_wildcard(json11::Json cfg) +{ + auto wildcard_remover = new wildcard_remover_t(); + wildcard_remover->parent = this; + wildcard_remover->cfg = cfg; + for (auto & glob: cfg["globs"].array_items()) + wildcard_remover->globs.push_back(glob.string_value()); + wildcard_remover->exact = cfg["exact"].bool_value(); + return [wildcard_remover](cli_result_t & result) + { + wildcard_remover->loop(); + if (wildcard_remover->is_done()) + { + result = wildcard_remover->result; + delete wildcard_remover; + return true; + } + return false; + }; +}