thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add wildcard removal command

Browse Source
This commit is contained in:
Vitaliy Filippov
2024-08-31 14:13:09 +03:00
parent 30eaa1a8e6
commit ba9f263b75
5 changed files with 228 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cmd/CMakeLists.txt
+1
View File
@@ -19,6 +19,7 @@ add_library(vitastor_cli STATIC
cli_flatten.cpp
cli_merge.cpp
cli_rm_data.cpp
cli_rm_wildcard.cpp
cli_rm.cpp
cli_rm_osd.cpp
cli_pool_cfg.cpp
src/cmd/cli.cpp
+14 -9
View File
@@ -54,14 +54,18 @@ static const char* help_text =
" -f|--force Proceed with shrinking or setting readwrite flag even if the image has children.\n"
" --down-ok Proceed with shrinking even if some data will be left on unavailable OSDs.\n"
"\n"
"vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]\n"
" Remove <from> or all layers between <from> and <to> (<to> must be a child of <from>),\n"
" rebasing all their children accordingly. --writers-stopped allows merging to be a bit\n"
" more effective in case of a single 'slim' read-write child and 'fat' removed parent:\n"
" the child is merged into parent and parent is renamed to child in that case.\n"
" In other cases parent layers are always merged into children.\n"
" Other options:\n"
" --down-ok Continue deletion/merging even if some data will be left on unavailable OSDs.\n"
"vitastor-cli rm <from> [<to>]\n"
"vitastor-cli rm (--exact|--matching) <glob> ...\n"
" Remove layer(s) and rebase all their children accordingly.\n"
" In the first form, remove <from> or layers between <from> and its child <to>.\n"
" In the second form, remove all images with exact or pattern-matched names.\n"
" --writers-stopped allows optimised removal in case of a single 'slim' read-write\n"
" child and 'fat' removed parent: the child is merged into parent and parent is renamed\n"
" to child in that case. In other cases parent layers are always merged into children.\n"
" --exact Remove multiple images with names matching given glob patterns.\n"
" --matching Remove multiple images with given names\n"
" --writers-stopped Allow renaming inodes over their read/write children.\n"
" --down-ok Continue deletion/merging even if some data will be left on unavailable OSDs.\n"
"\n"
"vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M]\n"
" [count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress]\n"
@@ -276,7 +280,8 @@ static json11::Json::object parse_args(int narg, const char *args[])
!strcmp(opt, "allow-data-loss") || !strcmp(opt, "allow_data_loss") ||
!strcmp(opt, "down-ok") || !strcmp(opt, "down_ok") ||
!strcmp(opt, "dry-run") || !strcmp(opt, "dry_run") ||
!strcmp(opt, "help") || !strcmp(opt, "all") || !strcmp(opt, "exact") ||
!strcmp(opt, "help") || !strcmp(opt, "all") ||
!strcmp(opt, "exact") || !strcmp(opt, "matching") ||
!strcmp(opt, "writers-stopped") || !strcmp(opt, "writers_stopped"))
{
cfg[opt] = "1";
src/cmd/cli.h
+1
View File
@@ -75,6 +75,7 @@ public:
std::function<bool(cli_result_t &)> start_rm(json11::Json);
std::function<bool(cli_result_t &)> start_rm_data(json11::Json);
std::function<bool(cli_result_t &)> start_rm_osd(json11::Json);
std::function<bool(cli_result_t &)> start_rm_wildcard(json11::Json);
std::function<bool(cli_result_t &)> start_status(json11::Json);
std::function<bool(cli_result_t &)> start_dd(json11::Json);
src/cmd/cli_rm.cpp
+4
View File
@@ -689,6 +689,10 @@ resume_100:
std::function<bool(cli_result_t &)> cli_tool_t::start_rm(json11::Json cfg)
{
if (cfg["exact"].bool_value() || cfg["matching"].bool_value())
{
return start_rm_wildcard(cfg);
}
auto snap_remover = new snap_remover_t();
snap_remover->parent = this;
snap_remover->from_name = cfg["from"].string_value();
src/cmd/cli_rm_wildcard.cpp
+208
View File
@@ -0,0 +1,208 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include <fcntl.h>
#include <algorithm>
#include "cli.h"
#include "cluster_client.h"
#include "str_util.h"
struct inode_rev_t
{
inode_t inode_num;
uint64_t meta_rev;
};
// Remove multiple images in correct order
struct wildcard_remover_t
{
cli_tool_t *parent;
json11::Json cfg;
std::vector<std::string> globs;
bool exact = false;
json11::Json::array deleted_ids, deleted_images, rebased_images;
std::map<inode_t, inode_t> chains; // child => parent pairs
std::vector<std::vector<inode_rev_t>> versioned_chains;
json11::Json::object sub_cfg;
size_t i = 0;
int state = 0;
std::function<bool(cli_result_t &)> sub_cb;
cli_result_t result;
bool is_done()
{
return state == 100;
}
void join_chains()
{
bool changed = true;
while (changed)
{
changed = false;
auto ino_it = chains.begin();
while (ino_it != chains.end())
{
auto child_id = ino_it->first;
auto parent_id = ino_it->second;
auto & parent_cfg = parent->cli->st_cli.inode_config.at(parent_id);
if (parent_cfg.parent_id)
{
auto chain_it = chains.find(parent_cfg.parent_id);
if (chain_it != chains.end())
{
changed = true;
ino_it->second = chain_it->second;
chains.erase(chain_it);
}
}
ino_it = chains.upper_bound(child_id);
}
}
// Remember metadata modification revisions to check for parallel changes
versioned_chains.clear();
for (auto cp: chains)
{
auto child_id = cp.first;
auto parent_id = cp.second;
std::vector<inode_rev_t> ver_chain;
do
{
auto & inode_cfg = parent->cli->st_cli.inode_config.at(child_id);
ver_chain.push_back((inode_rev_t){ .inode_num = child_id, .meta_rev = inode_cfg.mod_revision });
child_id = inode_cfg.parent_id;
} while (child_id && child_id != parent_id);
versioned_chains.push_back(std::move(ver_chain));
}
// Sort chains based on parent inode rank to first delete child-most layers
std::map<inode_t, uint64_t> ranks;
for (auto cp: chains)
{
auto parent_id = cp.second, cur_id = parent_id;
uint64_t rank = 0;
do
{
rank++;
cur_id = parent->cli->st_cli.inode_config.at(cur_id).parent_id;
} while (cur_id && cur_id != parent_id);
ranks[parent_id] = rank;
}
std::sort(versioned_chains.begin(), versioned_chains.end(), [&](const std::vector<inode_rev_t> & a, const std::vector<inode_rev_t> & b)
{
return ranks[a.back().inode_num] > ranks[b.back().inode_num];
});
}
void loop()
{
if (state == 0)
goto resume_0;
if (state == 1)
goto resume_1;
else if (state == 100)
goto resume_100;
resume_0:
state = 0;
chains.clear();
// Select images to delete
for (auto & ic: parent->cli->st_cli.inode_config)
{
for (auto & glob: globs)
{
if (exact ? (ic.second.name == glob) : stupid_glob(ic.second.name, glob))
{
chains[ic.first] = ic.first;
break;
}
}
}
// Arrange them into chains
join_chains();
// Delete each chain
i = 0;
while (i < versioned_chains.size())
{
// Check for parallel changes
for (auto & irev: versioned_chains[i])
{
auto inode_it = parent->cli->st_cli.inode_config.find(irev.inode_num);
if (inode_it == parent->cli->st_cli.inode_config.end() ||
inode_it->second.mod_revision > irev.meta_rev)
{
if (inode_it != parent->cli->st_cli.inode_config.end())
fprintf(stderr, "Warning: image %s modified by someone else during deletion, restarting wildcard deletion\n", inode_it->second.name.c_str());
else
fprintf(stderr, "Warning: inode %lx modified by someone else during deletion, retrying wildcard deletion\n", irev.inode_num);
goto resume_0;
}
}
// Delete
{
auto from_cfg = parent->cli->st_cli.inode_config.at(versioned_chains[i].back().inode_num);
auto to_cfg = parent->cli->st_cli.inode_config.at(versioned_chains[i].front().inode_num);
sub_cfg = cfg.object_items();
sub_cfg.erase("globs");
sub_cfg.erase("exact");
sub_cfg["from"] = from_cfg.name;
sub_cfg["to"] = to_cfg.name;
sub_cb = parent->start_rm(sub_cfg);
}
resume_1:
while (!sub_cb(result))
{
state = 1;
return;
}
sub_cb = NULL;
i++;
merge_result();
if (result.err)
{
break;
}
}
state = 100;
result.data = json11::Json::object{
{ "deleted_ids", deleted_ids },
{ "deleted_images", deleted_images },
{ "rebased_images", rebased_images },
};
resume_100:
// Done
return;
}
void merge_result()
{
for (auto & item: result.data["deleted_ids"].array_items())
deleted_ids.push_back(item);
for (auto & item: result.data["deleted_images"].array_items())
deleted_images.push_back(item == result.data["renamed_to"] ? result.data["renamed_from"] : item);
for (auto & item: result.data["rebased_images"].array_items())
rebased_images.push_back(item);
}
};
std::function<bool(cli_result_t &)> cli_tool_t::start_rm_wildcard(json11::Json cfg)
{
auto wildcard_remover = new wildcard_remover_t();
wildcard_remover->parent = this;
wildcard_remover->cfg = cfg;
for (auto & glob: cfg["globs"].array_items())
wildcard_remover->globs.push_back(glob.string_value());
wildcard_remover->exact = cfg["exact"].bool_value();
return [wildcard_remover](cli_result_t & result)
{
wildcard_remover->loop();
if (wildcard_remover->is_done())
{
result = wildcard_remover->result;
delete wildcard_remover;
return true;
}
return false;
};
}