diff --git a/src/client/cluster_client.cpp b/src/client/cluster_client.cpp index e9126667..5e0d94b9 100644 --- a/src/client/cluster_client.cpp +++ b/src/client/cluster_client.cpp @@ -51,7 +51,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd msgr.stop_client(op->client_id); delete op; }; - msgr.parse_config(config); + msgr.parse_config(config, true); st_cli.tfd = tfd; st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); }; @@ -481,7 +481,7 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co } // vault vault_parse_config(); - msgr.parse_config(config); + msgr.parse_config(config, false); st_cli.parse_config(config); st_cli.load_pgs(); } @@ -1600,6 +1600,9 @@ static inline void mem_or(void *res, const void *r2, unsigned int len) } } +// Error priority: others > EPERM > EIO > ENOSPC > ETIMEDOUT > EPIPE +#define ERR_PRIO(e) (((e) == -EPERM ? 5 : ((e) == -EIO ? 4 : ((e) == -ENOSPC ? 3 : ((e) == -ETIMEDOUT ? 2 : ((e) == -EPIPE ? 1 : (!(e) ? 0 : 10))))))) + void cluster_client_t::handle_op_part(cluster_op_part_t *part) { cluster_op_t *op = part->parent; @@ -1608,15 +1611,10 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part) { // Operation failed, retry part->flags |= PART_ERROR; - if (!op->retval || op->retval == -EPIPE || - part->op.reply.hdr.retval == -ENOSPC && op->retval == -ETIMEDOUT || - part->op.reply.hdr.retval == -EIO) - { - // Error priority: EIO > ENOSPC > ETIMEDOUT > EPIPE + if (ERR_PRIO(part->op.reply.hdr.retval) > ERR_PRIO(op->retval)) op->retval = part->op.reply.hdr.retval; - } uint64_t stop_client_id = 0; - if (op->retval != -EINTR && op->retval != -EIO && op->retval != -ENOSPC) + if (op->retval != -EINTR && op->retval != -EIO && op->retval != -ENOSPC && op->retval != -EPERM) { stop_client_id = part->op.client_id; if (op->retval != -EPIPE || log_level > 0) diff --git a/src/client/etcd_state_client.cpp b/src/client/etcd_state_client.cpp index 9fcc5e8a..65c4b3f2 100644 --- a/src/client/etcd_state_client.cpp +++ b/src/client/etcd_state_client.cpp @@ -12,6 +12,7 @@ #include "http_client.h" #endif #include "str_util.h" +#include "json_util.h" etcd_state_client_t::~etcd_state_client_t() { @@ -80,6 +81,46 @@ std::vector etcd_state_client_t::get_addresses() return addrs; } +std::shared_ptr etcd_state_client_t::get_user(const std::string & username) +{ + auto user_it = user_info.find(username); + if (user_it != user_info.end()) + { + return user_it->second; + } + auto inf = std::make_shared(); + inf->name = username; + return inf; +} + +bool etcd_state_client_t::check_image_perm(const std::shared_ptr & user_info, inode_t inode_num, bool write) +{ + if (user_info->type == user_type_t::ADMIN) + { + return true; + } + auto cache_it = user_info->perm_cache.find(inode_num); + if (cache_it != user_info->perm_cache.end() && + cache_it->second.mod_revision == user_perm_cache_revision) + { + return write ? (cache_it->second.perm == user_perm_t::OWNER) : (cache_it->second.perm != user_perm_t::DENY); + } + auto inode_it = inode_config.find(inode_num); + if (inode_it == inode_config.end()) + { + return false; + } + // FIXME Implement cache reset after reworking etcd interaction to not keep everything in memory + auto & perm_item = user_info->perm_cache[inode_num]; + perm_item.mod_revision = user_perm_cache_revision; + perm_item.perm = (user_info->name == inode_it->second.owner || inode_it->second.owner_group != "" && + user_info->groups.find(inode_it->second.owner_group) != user_info->groups.end() + ? user_perm_t::OWNER : (inode_it->second.reader_group != "" && + user_info->groups.find(inode_it->second.reader_group) != user_info->groups.end() + ? user_perm_t::READER : user_perm_t::DENY)); + return write ? (perm_item.perm == user_perm_t::OWNER) : (perm_item.perm != user_perm_t::DENY); +} + http_context_t *etcd_state_client_t::get_http_ctx() { if (!http_ctx) @@ -252,17 +293,21 @@ void etcd_state_client_t::parse_config(const json11::Json & config) } if (this->osd_num) { - this->etcd_client_cert = config["osd_etcd_client_cert"].string_value(); - this->etcd_client_key = config["osd_etcd_client_key"].string_value(); + this->etcd_client_cert = config["osd_cert"].string_value(); + this->etcd_client_key = config["osd_pkey"].string_value(); } else + { + this->etcd_client_cert = config["cert"].string_value(); + this->etcd_client_key = config["pkey"].string_value(); + } + if (this->etcd_client_cert == "") { this->etcd_client_cert = config["etcd_client_cert"].string_value(); this->etcd_client_key = config["etcd_client_key"].string_value(); } this->etcd_ca = config["etcd_ca"].string_value(); this->etcd_prefix = config["etcd_prefix"].string_value(); - this->use_auth = config["use_auth"].bool_value(); if (this->etcd_prefix == "") { this->etcd_prefix = "/vitastor"; @@ -1339,6 +1384,10 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv) { on_inode_change_hook(inode_num, true); } + if (this->inode_config.find(inode_num) != this->inode_config.end()) + { + user_perm_cache_revision = kv.mod_revision; + } this->inode_config.erase(inode_num); } else @@ -1354,13 +1403,39 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv) if (on_change_node_placement_hook) on_change_node_placement_hook(); } - else if (use_auth && key.substr(0, etcd_prefix.length()+13) == etcd_prefix+"/config/user/") + else if (key.substr(0, etcd_prefix.length()+13) == etcd_prefix+"/config/user/") { // /config/user/ + auto name = key.substr(etcd_prefix.length()+13); + auto & inf = user_info[name]; if (!value.is_object()) - user_info.erase(key.substr(etcd_prefix.length()+13)); + { + if (inf) + { + inf->type = user_type_t::CLIENT; + inf->groups.clear(); + inf->perm_cache.clear(); + } + user_info.erase(name); + } else - user_info[key.substr(etcd_prefix.length()+13)] = value; + { + if (!inf) + { + inf = std::make_shared(); + inf->name = name; + } + inf->type = value["type"] == "admin" ? user_type_t::ADMIN : + (value["type"] == "mon" ? user_type_t::MON : + (value["type"] == "osd" ? user_type_t::OSD : user_type_t::CLIENT)); + inf->groups.clear(); + for (auto & group: value["groups"].array_items()) + { + if (group.string_value() != "") + inf->groups.insert(group.string_value()); + } + inf->perm_cache.clear(); + } } } @@ -1384,7 +1459,12 @@ uint32_t etcd_state_client_t::parse_scheme(const std::string & scheme) void etcd_state_client_t::insert_inode_config(const inode_config_t & cfg) { - this->inode_config[cfg.num] = cfg; + auto & cfg_ref = this->inode_config[cfg.num]; + if (cfg_ref.mod_revision != cfg.mod_revision) + { + user_perm_cache_revision = cfg.mod_revision; + } + cfg_ref = cfg; if (cfg.name != "") { this->inode_by_name[cfg.name] = cfg.num; diff --git a/src/client/etcd_state_client.h b/src/client/etcd_state_client.h index 109044ee..288670a6 100644 --- a/src/client/etcd_state_client.h +++ b/src/client/etcd_state_client.h @@ -9,6 +9,7 @@ #include "json11/json11.hpp" #include "object_id.h" #include "timerfd_manager.h" +#include "../util/robin_hood.h" #define ETCD_CONFIG_WATCH_ID 1 #define ETCD_OSD_STATE_WATCH_ID 2 @@ -110,6 +111,32 @@ struct http_url_t std::string path; }; +enum class user_type_t +{ + CLIENT = 0, + ADMIN = 1, + MON = 2, + OSD = 3, +}; + +struct user_perm_t +{ + enum class perm_type_t: uint8_t; + constexpr static perm_type_t DENY = (perm_type_t)0; + constexpr static perm_type_t READER = (perm_type_t)1; + constexpr static perm_type_t OWNER = (perm_type_t)2; + uint64_t mod_revision = 0; + perm_type_t perm = DENY; +}; + +struct user_info_t +{ + std::string name; + user_type_t type; + robin_hood::unordered_flat_set groups; + robin_hood::unordered_flat_map perm_cache; +}; + struct http_co_t; struct http_context_t; @@ -145,7 +172,6 @@ public: int etcd_slow_timeout = 5000; int etcd_min_reload_interval = 1000; bool infinite_start = true; - bool use_auth = false; uint64_t global_block_size = DEFAULT_BLOCK_SIZE; uint32_t global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY; uint32_t global_immediate_commit = IMMEDIATE_NONE; @@ -171,7 +197,8 @@ public: std::set seen_peers; std::map inode_config; std::map inode_by_name; - std::map user_info; + robin_hood::unordered_flat_map> user_info; + uint64_t user_perm_cache_revision = 0; json11::Json node_placement; std::function &)> on_change_hook; @@ -193,6 +220,8 @@ public: inode_config_t deserialize_inode_cfg(uint64_t inode_num, json11::Json value, uint64_t mod_revision); etcd_kv_t parse_etcd_kv(const json11::Json & kv_json); std::vector get_addresses(); + std::shared_ptr get_user(const std::string & username); + bool check_image_perm(const std::shared_ptr & user_info, inode_t inode_num, bool write); http_context_t *get_http_ctx(); void etcd_call_oneshot(const std::string & etcd_address, const std::string & api, json11::Json payload, int timeout, std::function callback); void etcd_call(const std::string & api, json11::Json payload, int timeout, int retries, int interval, std::function callback); diff --git a/src/client/messenger.cpp b/src/client/messenger.cpp index bb2b86d4..2fdf801a 100644 --- a/src/client/messenger.cpp +++ b/src/client/messenger.cpp @@ -308,8 +308,41 @@ osd_messenger_t::~osd_messenger_t() destroy_tls(); } -void osd_messenger_t::parse_config(const json11::Json & config) +void osd_messenger_t::parse_config(const json11::Json & config, bool init) { + this->max_cipher_pool_size = config["max_cipher_pool_size"].uint64_value(); + if (!this->max_cipher_pool_size) + this->max_cipher_pool_size = 256; + if (config["proto_checksums"].is_null()) + this->use_proto_checksums = MSGR_CSUM_PAYLOAD; + else if (config["proto_checksums"].is_bool()) + this->use_proto_checksums = config["proto_checksums"].bool_value() ? MSGR_CSUM_FULL : 0; + else if (config["proto_checksums"].string_value() != "") + this->use_proto_checksums = config["proto_checksums"].string_value() == "full" ? MSGR_CSUM_FULL : MSGR_CSUM_PAYLOAD; + else + this->use_proto_checksums = 0; + this->receive_buffer_size = (uint32_t)config["tcp_header_buffer_size"].uint64_value(); + if (!this->receive_buffer_size || this->receive_buffer_size > 1024*1024*1024) + this->receive_buffer_size = 65536; + this->min_zerocopy_send_size = config["min_zerocopy_send_size"].is_null() + ? DEFAULT_MIN_ZEROCOPY_SEND_SIZE + : (int)config["min_zerocopy_send_size"].int64_value(); + this->peer_connect_interval = config["peer_connect_interval"].uint64_value(); + if (!this->peer_connect_interval) + this->peer_connect_interval = 5; + this->peer_connect_timeout = config["peer_connect_timeout"].uint64_value(); + if (!this->peer_connect_timeout) + this->peer_connect_timeout = 5; + this->osd_idle_timeout = config["osd_idle_timeout"].uint64_value(); + if (!this->osd_idle_timeout) + this->osd_idle_timeout = 5; + this->osd_ping_timeout = config["osd_ping_timeout"].uint64_value(); + if (!this->osd_ping_timeout) + this->osd_ping_timeout = 5; + this->log_level = config["log_level"].uint64_value(); + // All other parameters are only set on init + if (!init) + return; #ifdef WITH_RDMA if (!config["use_rdma"].is_null()) { @@ -340,17 +373,6 @@ void osd_messenger_t::parse_config(const json11::Json & config) if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024) this->rdma_max_msg = 129*1024; #endif - this->max_cipher_pool_size = config["max_cipher_pool_size"].uint64_value(); - if (!this->max_cipher_pool_size) - this->max_cipher_pool_size = 256; - if (config["proto_checksums"].is_null()) - this->use_proto_checksums = MSGR_CSUM_PAYLOAD; - else if (config["proto_checksums"].is_bool()) - this->use_proto_checksums = config["proto_checksums"].bool_value() ? MSGR_CSUM_FULL : 0; - else if (config["proto_checksums"].string_value() != "") - this->use_proto_checksums = config["proto_checksums"].string_value() == "full" ? MSGR_CSUM_FULL : MSGR_CSUM_PAYLOAD; - else - this->use_proto_checksums = 0; if (!osd_num) { tls_cert = config["cert"].string_value(); @@ -368,27 +390,8 @@ void osd_messenger_t::parse_config(const json11::Json & config) this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value(); else this->iothread_count = (uint32_t)config["osd_iothread_count"].uint64_value(); - this->receive_buffer_size = (uint32_t)config["tcp_header_buffer_size"].uint64_value(); - if (!this->receive_buffer_size || this->receive_buffer_size > 1024*1024*1024) - this->receive_buffer_size = 65536; this->use_sync_send_recv = config["use_sync_send_recv"].bool_value() || config["use_sync_send_recv"].uint64_value() || !ringloop; - this->min_zerocopy_send_size = config["min_zerocopy_send_size"].is_null() - ? DEFAULT_MIN_ZEROCOPY_SEND_SIZE - : (int)config["min_zerocopy_send_size"].int64_value(); - this->peer_connect_interval = config["peer_connect_interval"].uint64_value(); - if (!this->peer_connect_interval) - this->peer_connect_interval = 5; - this->peer_connect_timeout = config["peer_connect_timeout"].uint64_value(); - if (!this->peer_connect_timeout) - this->peer_connect_timeout = 5; - this->osd_idle_timeout = config["osd_idle_timeout"].uint64_value(); - if (!this->osd_idle_timeout) - this->osd_idle_timeout = 5; - this->osd_ping_timeout = config["osd_ping_timeout"].uint64_value(); - if (!this->osd_ping_timeout) - this->osd_ping_timeout = 5; - this->log_level = config["log_level"].uint64_value(); // OSD public & cluster networks this->osd_networks.clear(); if (config["osd_network"].is_string()) @@ -729,6 +732,11 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl) err = true; fprintf(stderr, "Failed to get config from OSD %ju (retval=%jd), disconnecting peer\n", cl->osd_num, op->reply.hdr.retval); } + else if (cl->gcm_enabled && !cl->hs_result.peer_is_osd) + { + err = true; + fprintf(stderr, "Client %ju is not authenticated as an OSD, disconnecting peer\n", cl->client_id); + } else { config = json11::Json::parse(std::string((char*)op->buf), json_err); @@ -879,6 +887,11 @@ bool osd_messenger_t::is_use_rdmacm() } #endif +bool osd_messenger_t::is_encryption_enabled() +{ + return tls_cert != "" || tls_key != "" || osd_tls_ca != ""; +} + json11::Json::object osd_messenger_t::read_config(const json11::Json & config) { json11::Json::object file_config; diff --git a/src/client/messenger.h b/src/client/messenger.h index 4687ae65..411c358a 100644 --- a/src/client/messenger.h +++ b/src/client/messenger.h @@ -66,6 +66,8 @@ struct op_aes_xts_decrypt_t; void destroy_aes_xts_encrypt(op_aes_xts_encrypt_t *encrypt_ctx); void destroy_aes_xts_decrypt(op_aes_xts_decrypt_t *decrypt_ctx); +struct user_info_t; + struct osd_client_t { uint64_t client_id = 0; @@ -94,6 +96,7 @@ struct osd_client_t bool gcm_enabled = false; msgr_handshake_i *hs = NULL; msgr_handshake_result_t hs_result; + std::shared_ptr user_info; std::vector my_key, peer_key; uint64_t my_iv_ctr = 0, peer_iv_ctr = 0; #ifdef WITH_ISAL_CRYPTO @@ -322,7 +325,7 @@ public: osd_op_stats_t stats, recovery_stats; void init(); - void parse_config(const json11::Json & config); + void parse_config(const json11::Json & config, bool init); void connect_peer(uint64_t osd_num, json11::Json peer_state); void stop_client(uint64_t client_id, bool force_delete = false); void destroy_client(osd_client_t *cl); @@ -331,6 +334,7 @@ public: std::function repeer_pgs; std::function break_pg_locks; std::function check_config_hook; + std::function handshake_hook; void read_requests(); void send_replies(); void accept_connections(int listen_fd); @@ -351,6 +355,7 @@ public: rdma_cm_id *rdmacm_listen(const std::string & bind_address, int rdmacm_port, int *bound_port, int log_level); void rdmacm_destroy_listener(rdma_cm_id *listener); #endif + bool is_encryption_enabled(); void inc_op_stats(osd_op_stats_t & stats, uint64_t opcode, timespec & tv_begin, timespec & tv_end, uint64_t len); void measure_exec(osd_op_t *cur_op); diff --git a/src/client/msgr_encrypt.cpp b/src/client/msgr_encrypt.cpp index 144ffdf4..20d66dbd 100644 --- a/src/client/msgr_encrypt.cpp +++ b/src/client/msgr_encrypt.cpp @@ -491,6 +491,10 @@ bool osd_messenger_t::derive_aes_keys(osd_client_t *cl, bool update_my, bool upd if (!cl->hs_result.shared_secret.size()) { cl->hs_result = cl->hs->get_result(); + if (handshake_hook) + { + handshake_hook(cl); + } } std::vector old_my = cl->my_key, old_peer = cl->peer_key; // Both keys include AES key and iv + xxhash3 secret diff --git a/src/cmd/cli.cpp b/src/cmd/cli.cpp index 3d94e40d..986e1332 100644 --- a/src/cmd/cli.cpp +++ b/src/cmd/cli.cpp @@ -259,9 +259,9 @@ static const char* help_text = " Start HTTP server able to handle CLI commands over a REST API. Options:\n" " --bind_address ADDR Specify server IP address or addresses, separated by space. Default is 127.0.0.1.\n" " --port 8080 Specify server port.\n" - " --ssl_cert FILE Path to server SSL certificate file (PEM format).\n" - " --ssl_key FILE Path to server SSL private key file.\n" - " --ssl_ca FILE Path to file with SSL CA certificates used to validate client connections.\n" + " --server_cert FILE Path to server TLS certificate file (PEM format).\n" + " --server_key FILE Path to server TLS private key file.\n" + " --client_ca FILE Path to file with TLS CA certificates used to validate client connections.\n" "\n" "Use vitastor-cli --help for command details or vitastor-cli --help --all for all details.\n" "\n" diff --git a/src/cmd/cli.h b/src/cmd/cli.h index 5c5e1696..3e9d33f8 100644 --- a/src/cmd/cli.h +++ b/src/cmd/cli.h @@ -27,12 +27,7 @@ struct cli_result_t json11::Json data; }; -struct cli_user_t -{ - std::string name; - std::string type; - std::set groups; -}; +struct user_info_t; class cli_tool_t { @@ -45,7 +40,7 @@ public: bool is_command_line = false; bool color = false; - std::unique_ptr user; // for http mode + std::shared_ptr user; // for http mode ring_loop_t *ringloop = NULL; epoll_manager_t *epmgr = NULL; diff --git a/src/cmd/cli_common.cpp b/src/cmd/cli_common.cpp index 75a9faab..b4e2ad8b 100644 --- a/src/cmd/cli_common.cpp +++ b/src/cmd/cli_common.cpp @@ -9,7 +9,7 @@ bool cli_tool_t::check_image_perm(const inode_config_t & cfg, bool write) { return !user || - user->type == "admin" || + user->type == user_type_t::ADMIN || user->name == cfg.owner || cfg.owner_group != "" && user->groups.find(cfg.owner_group) != user->groups.end() || !write && cfg.reader_group != "" && user->groups.find(cfg.reader_group) != user->groups.end(); diff --git a/src/cmd/cli_create.cpp b/src/cmd/cli_create.cpp index 6e96a1a1..5e4ae6f1 100644 --- a/src/cmd/cli_create.cpp +++ b/src/cmd/cli_create.cpp @@ -121,7 +121,7 @@ struct image_creator_t bool check_pool_permission() { - if (!parent->user || parent->user->type == "admin") + if (!parent->user || parent->user->type == user_type_t::ADMIN) { return true; } diff --git a/src/cmd/cli_ls.cpp b/src/cmd/cli_ls.cpp index a4021c7a..5835109c 100644 --- a/src/cmd/cli_ls.cpp +++ b/src/cmd/cli_ls.cpp @@ -147,7 +147,7 @@ resume_1: inode_t inode_num = INODE_WITH_POOL(pool_id, only_inode_num); uint64_t used_size = kv.value["raw_used"].uint64_value(); auto stat_it = stats.find(inode_num); - if (parent->user && parent->user->type != "admin" && stat_it == stats.end()) + if (parent->user && parent->user->type != user_type_t::ADMIN && stat_it == stats.end()) { continue; } diff --git a/src/cmd/cli_serve.cpp b/src/cmd/cli_serve.cpp index e85d8e33..1bf69768 100644 --- a/src/cmd/cli_serve.cpp +++ b/src/cmd/cli_serve.cpp @@ -76,6 +76,7 @@ struct cli_serve_t int port = 0; int listen_backlog = 0; bool ssl = false; + bool use_auth = false; std::vector listen_fds; http_context_t *http_ctx = NULL; std::set connections; @@ -111,19 +112,34 @@ struct cli_serve_t listen_backlog = options["listen_backlog"].uint64_value(); if (!listen_backlog) listen_backlog = 128; - ssl = json_is_true(options["ssl"]); - if (ssl) { - std::string ssl_cert = options["ssl_cert"].string_value(); - std::string ssl_key = options["ssl_key"].string_value(); - std::string ssl_ca = options["ssl_ca"].string_value(); - std::string error; - http_ctx = http_context_init(parent->epmgr->tfd, ssl_cert, ssl_key, ssl_ca, ssl_ca != "", error); - if (error != "") + std::string tls_cert = (parent->cli->config.find("server_cert") != parent->cli->config.end() + ? parent->cli->config["server_cert"].string_value() : ""); + std::string tls_key = (parent->cli->config.find("server_key") != parent->cli->config.end() + ? parent->cli->config["server_key"].string_value() : ""); + std::string tls_ca = (parent->cli->config.find("client_ca") != parent->cli->config.end() + ? parent->cli->config["client_ca"].string_value() : ""); + if (tls_cert != "" || tls_key != "" || tls_ca != "") { - result = (cli_result_t){ .err = EINVAL, .text = error }; - state = 100; - return; + ssl = true; + if (tls_cert == "" || tls_key == "") + { + result = (cli_result_t){ .err = EINVAL, .text = "server_cert and server_key are required to serve HTTPS" }; + state = 100; + return; + } + // use_auth is enabled by default when client_ca is set + use_auth = (parent->cli->config["use_auth"].is_null() + ? (tls_ca != "") + : json_is_true(parent->cli->config["use_auth"])); + std::string error; + http_ctx = http_context_init(parent->epmgr->tfd, tls_cert, tls_key, tls_ca, tls_ca != "", error); + if (error != "") + { + result = (cli_result_t){ .err = EINVAL, .text = error }; + state = 100; + return; + } } } for (auto & bind_address: bind_addresses) @@ -344,17 +360,9 @@ struct cli_serve_t conn->request_path = std::move(req_line[1]); conn->request_body = std::move(msg->body); conn->response_type = ""; - if (parent->cli->st_cli.use_auth) + if (use_auth) { - auto user = std::make_unique(); - user->name = msg->headers["_tls_common_name"]; - auto user_it = parent->cli->st_cli.user_info.find(user->name); - auto userinfo = user_it == parent->cli->st_cli.user_info.end() ? user_it->second : json11::Json(); - user->type = user->name == "root" ? "admin" : userinfo["type"].string_value(); - for (auto & gr: userinfo["groups"].array_items()) - { - user->groups.insert(gr.string_value()); - } + conn->p->user = parent->cli->st_cli.get_user(msg->headers["_tls_common_name"]); } auto ctype = msg->headers["content-type"]; if (conn->request_method != "GET" && conn->request_method != "POST") @@ -383,10 +391,10 @@ struct cli_serve_t { conn->response_type = "application/json"; conn->result = { .text = openapi_description }; - if (parent->cli->st_cli.use_auth) + if (use_auth) { // Filter available paths by privileges - if (conn->p->user->type == "client") + if (conn->p->user->type == user_type_t::CLIENT) { std::string error; auto openapi = json11::Json::parse(openapi_description, error).object_items(); @@ -403,7 +411,7 @@ struct cli_serve_t conn->response_type = "application/json"; conn->result = { .text = json11::Json(openapi).dump() }; } - else if (conn->p->user->type != "admin") + else if (conn->p->user->type != user_type_t::ADMIN) { conn->response_type = ""; conn->result = { .err = EACCES, .text = "Access denied" }; @@ -418,7 +426,7 @@ struct cli_serve_t { conn->result = { .err = ENOSYS, .text = "method /"+uri[0]+" only allows POST requests" }; } - else if (parent->cli->st_cli.use_auth && conn->p->user->type == "client" && !cmd_it->second.allow_client) + else if (use_auth && conn->p->user->type == user_type_t::CLIENT && !cmd_it->second.allow_client) { conn->result = { .err = EACCES, .text = "Access denied" }; } diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index fbf5220d..3e90c17c 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -14,6 +14,7 @@ #include "http_client.h" #include "str_util.h" #include "json_util.h" +#include "openssl_util.h" osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop) { @@ -67,6 +68,11 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop) msgr.repeer_pgs = [this](osd_num_t peer_osd) { repeer_pgs(peer_osd); }; msgr.break_pg_locks = [this](osd_num_t peer_osd) { break_pg_locks(peer_osd); }; msgr.check_config_hook = [this](osd_client_t *cl, json11::Json conf) { return check_peer_config(cl, conf); }; + msgr.handshake_hook = [this](osd_client_t *cl) + { + if (!cl->hs_result.peer_is_osd) + cl->user_info = st_cli.get_user(openssl_get_cn(cl->hs_result.peer_cert)); + }; msgr.init(); init_cluster(); @@ -170,9 +176,13 @@ void osd_t::parse_config(bool init) bs->parse_config(bs_cfg); } st_cli.parse_config(config); - msgr.parse_config(config); + msgr.parse_config(config, init); if (init) { + // use_auth is enabled by default when encryption is enabled + use_auth = (config["use_auth"].is_null() + ? msgr.is_encryption_enabled() + : json_is_true(config["use_auth"])); // Vital Blockstore parameters bs_block_size = config["block_size"].uint64_value(); if (!bs_block_size) diff --git a/src/osd/osd.h b/src/osd/osd.h index 59948e55..ae3871ff 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -30,6 +30,8 @@ #define OSD_RECOVERING 0x10 #define OSD_SCRUBBING 0x20 +#define SELF_CLIENT 0 + #define MAX_AUTOSYNC_INTERVAL 3600 #define DEFAULT_AUTOSYNC_INTERVAL 5 #define DEFAULT_AUTOSYNC_WRITES 128 @@ -155,6 +157,7 @@ class osd_t etcd_state_client_t st_cli; osd_messenger_t msgr; + bool use_auth = false; int etcd_failed_attempts = 0; std::string etcd_lease_id; json11::Json self_state; diff --git a/src/osd/osd_flush.cpp b/src/osd/osd_flush.cpp index 319667d5..c3b1ec53 100644 --- a/src/osd/osd_flush.cpp +++ b/src/osd/osd_flush.cpp @@ -4,7 +4,6 @@ #include "osd.h" #define FLUSH_BATCH 512 -#define SELF_CLIENT 0 void osd_t::submit_pg_flush_ops(pg_t & pg) { diff --git a/src/osd/osd_peering.cpp b/src/osd/osd_peering.cpp index 83243b12..7c634e25 100644 --- a/src/osd/osd_peering.cpp +++ b/src/osd/osd_peering.cpp @@ -9,8 +9,6 @@ #include "str_util.h" #include "osd.h" -#define SELF_CLIENT 0 - // Peering loop void osd_t::handle_peers() { diff --git a/src/osd/osd_primary.cpp b/src/osd/osd_primary.cpp index 1be7a0d2..d54fed0d 100644 --- a/src/osd/osd_primary.cpp +++ b/src/osd/osd_primary.cpp @@ -71,6 +71,21 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op) finish_op(cur_op, -EINVAL); return false; } + if (use_auth && cur_op->client_id != SELF_CLIENT) + { + osd_client_t *cl = msgr.clients.at(cur_op->client_id); + if (cl->hs_result.peer_is_osd) + { + // OSDs are not allowed to execute "primary" operations + finish_op(cur_op, -EPERM); + return false; + } + if (!st_cli.check_image_perm(cl->user_info, cur_op->req.rw.inode, (cur_op->req.hdr.opcode != OSD_OP_READ))) + { + finish_op(cur_op, -EPERM); + return false; + } + } int stripe_count = (cur_op->req.hdr.opcode == OSD_OP_SCRUB ? 0 : (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg_it->second.pg_size)); int chain_size = 0; diff --git a/src/osd/osd_primary_describe.cpp b/src/osd/osd_primary_describe.cpp index 80ec92c7..aecf03cd 100644 --- a/src/osd/osd_primary_describe.cpp +++ b/src/osd/osd_primary_describe.cpp @@ -91,6 +91,21 @@ static void scan_lists(std::vector & lists, uint64_t limit, desc // Describe unclean objects void osd_t::continue_primary_describe(osd_op_t *cur_op) { + if (use_auth) + { + osd_client_t *cl = msgr.clients.at(cur_op->client_id); + if (cl->hs_result.peer_is_osd) + { + // OSDs are not allowed to execute "primary" operations + finish_op(cur_op, -EPERM); + return; + } + if (cl->user_info->type != user_type_t::ADMIN) + { + finish_op(cur_op, -EPERM); + return; + } + } auto & desc = cur_op->req.describe; if (!desc.object_state) desc.object_state = ~desc.object_state; diff --git a/src/osd/osd_primary_subops.cpp b/src/osd/osd_primary_subops.cpp index e3314b44..5b5273b2 100644 --- a/src/osd/osd_primary_subops.cpp +++ b/src/osd/osd_primary_subops.cpp @@ -3,8 +3,6 @@ #include "osd_primary.h" -#define SELF_CLIENT 0 - void osd_t::autosync() { if (immediate_commit != IMMEDIATE_ALL && !autosync_op) diff --git a/src/osd/osd_primary_sync.cpp b/src/osd/osd_primary_sync.cpp index 3a8905f1..08d8d911 100644 --- a/src/osd/osd_primary_sync.cpp +++ b/src/osd/osd_primary_sync.cpp @@ -8,6 +8,16 @@ void osd_t::continue_primary_sync(osd_op_t *cur_op) { if (!cur_op->op_data) { + if (use_auth && cur_op->client_id != SELF_CLIENT) + { + osd_client_t *cl = msgr.clients.at(cur_op->client_id); + if (cl->hs_result.peer_is_osd) + { + // OSDs are not allowed to execute "primary" operations + finish_op(cur_op, -EPERM); + return; + } + } cur_op->op_data = (osd_primary_op_data_t*)calloc_or_die(1, sizeof(osd_primary_op_data_t)); } osd_primary_op_data_t *op_data = cur_op->op_data; diff --git a/src/osd/osd_scrub.cpp b/src/osd/osd_scrub.cpp index 4ec3817a..acdeac3a 100644 --- a/src/osd/osd_scrub.cpp +++ b/src/osd/osd_scrub.cpp @@ -3,8 +3,6 @@ #include "osd_primary.h" -#define SELF_CLIENT 0 - void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oid) { pool_id_t pool_id = pg_id.pool_id; diff --git a/src/osd/osd_secondary.cpp b/src/osd/osd_secondary.cpp index a428498f..a61e68cf 100644 --- a/src/osd/osd_secondary.cpp +++ b/src/osd/osd_secondary.cpp @@ -1,12 +1,13 @@ // Copyright (c) Vitaliy Filippov, 2019+ // License: VNPL-1.1 (see README.md for details) +#include "json11/json11.hpp" + #include "osd.h" #ifdef WITH_RDMA #include "msgr_rdma.h" #endif - -#include "json11/json11.hpp" +#include "openssl_util.h" void osd_t::secondary_op_callback(osd_op_t *op) { @@ -112,6 +113,31 @@ bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid, uint3 void osd_t::exec_secondary_real(osd_op_t *cur_op) { + osd_client_t *cl = msgr.clients.at(cur_op->client_id); + if (use_auth && !cl->hs_result.peer_is_osd) + { + // Non-OSDs are not allowed to execute "secondary" operations except LIST + bool allowed = false; + if (cur_op->req.hdr.opcode == OSD_OP_SEC_LIST) + { + if (cl->user_info->type == user_type_t::ADMIN) + { + // Admin is allowed to execute arbitrary listings + allowed = true; + } + else if (cl->user_info->type == user_type_t::CLIENT && cur_op->req.sec_list.min_inode && + cur_op->req.sec_list.min_inode == cur_op->req.sec_list.max_inode) + { + // Clients are only allowed to execute listings for readable inodes + allowed = st_cli.check_image_perm(cl->user_info, cur_op->req.sec_list.min_inode, false); + } + } + if (!allowed) + { + finish_op(cur_op, -EPERM); + return; + } + } if (cur_op->req.hdr.opcode == OSD_OP_SEC_LIST && (cur_op->req.sec_list.flags & OSD_LIST_PRIMARY)) { @@ -128,7 +154,6 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op) exec_sec_lock(cur_op); return; } - osd_client_t *cl = msgr.clients.at(cur_op->client_id); cur_op->bs_op = new blockstore_op_t(); cur_op->bs_op->callback = [this, cur_op](blockstore_op_t* bs_op) { secondary_op_callback(cur_op); }; cur_op->bs_op->opcode = (cur_op->req.hdr.opcode == OSD_OP_SEC_READ ? BS_OP_READ diff --git a/src/test/mock/messenger.cpp b/src/test/mock/messenger.cpp index bfcb5e90..4d3ef8b6 100644 --- a/src/test/mock/messenger.cpp +++ b/src/test/mock/messenger.cpp @@ -26,7 +26,7 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op) cl->sent_ops[cur_op->req.hdr.id] = cur_op; } -void osd_messenger_t::parse_config(const json11::Json & config) +void osd_messenger_t::parse_config(const json11::Json & config, bool init) { } diff --git a/src/test/stub_uring_osd.cpp b/src/test/stub_uring_osd.cpp index 1a8b614f..5f387220 100644 --- a/src/test/stub_uring_osd.cpp +++ b/src/test/stub_uring_osd.cpp @@ -39,7 +39,7 @@ int main(int narg, char *args[]) msgr->repeer_pgs = [](osd_num_t) {}; msgr->exec_op = [msgr](osd_op_t *op) { stub_exec_op(msgr, op); }; json11::Json config = json11::Json::object { { "log_level", 1 } }; - msgr->parse_config(config); + msgr->parse_config(config, true); // Accept new connections int listen_fd = create_and_bind_socket("0.0.0.0", 11203, 128, NULL); fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK); diff --git a/tests/common.sh b/tests/common.sh index 126f9f1b..0ed521ac 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -141,6 +141,7 @@ if [[ "$OSD_TLS" = "1" ]]; then VITASTOR_CFG="$VITASTOR_CFG"',"client_ca":"'$(pwd)'/testdata/client_ca.crt"' VITASTOR_CFG="$VITASTOR_CFG"',"cert":"'$(pwd)'/testdata/cli.crt"' VITASTOR_CFG="$VITASTOR_CFG"',"pkey":"'$(pwd)'/testdata/cli.key"' + VITASTOR_CFG="$VITASTOR_CFG"',"use_auth":false' fi echo "{$VITASTOR_CFG}" > ./testdata/vitastor.conf VITASTOR_CFG=./testdata/vitastor.conf