From d84b84f58d7b60b2dfafd371fbfa5b5d84162b10 Mon Sep 17 00:00:00 2001
From: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Sat, 23 Nov 2024 01:08:13 +0300
Subject: [PATCH] Fix new backfillfull feature, add more logs

---
 mon/mon.js   | 31 ++++++++++++++++++++++++-------
 mon/stats.js | 15 +++++++++++----
 2 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/mon/mon.js b/mon/mon.js
index 34d91aaa..294cd6c4 100644
--- a/mon/mon.js
+++ b/mon/mon.js
@@ -659,7 +659,13 @@ class Mon
                 this.etcd_watch_revision, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history);
         }
         new_pg_config.hash = tree_hash;
-        const { backfillfull_pools } = sum_object_counts({ ...this.state, pg: { ...this.state.pg, config: new_pg_config } }, this.config);
+        const { backfillfull_pools, backfillfull_osds } = sum_object_counts(
+            { ...this.state, pg: { ...this.state.pg, config: new_pg_config } }, this.config
+        );
+        if (backfillfull_pools.join(',') != ((this.state.pg.config||{}).backfillfull_pools||[]).join(','))
+        {
+            this.log_backfillfull(backfillfull_osds, backfillfull_pools);
+        }
         new_pg_config.backfillfull_pools = backfillfull_pools.length ? backfillfull_pools : undefined;
         if (!await this.save_pg_config(new_pg_config, etcd_request))
         {
@@ -737,7 +743,7 @@ class Mon
     async update_total_stats()
     {
         const txn = [];
-        const { object_counts, object_bytes, backfillfull_pools } = sum_object_counts(this.state, this.config);
+        const { object_counts, object_bytes, backfillfull_pools, backfillfull_osds } = sum_object_counts(this.state, this.config);
         let stats = sum_op_stats(this.state.osd, this.prev_stats);
         let { inode_stats, seen_pools } = sum_inode_stats(this.state, this.prev_stats);
         stats.object_counts = object_counts;
@@ -791,17 +797,28 @@ class Mon
             await this.etcd.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0);
         }
         if (!this.recheck_pgs_active &&
-            backfillfull_pools.join(',') != ((this.state.pg.config||{}).no_rebalance_pools||[]).join(','))
+            backfillfull_pools.join(',') != ((this.state.pg.config||{}).backfillfull_pools||[]).join(','))
         {
-            console.log(
-                (backfillfull_pools.length ? 'Pool(s) '+backfillfull_pools.join(', ') : 'No pools')+
-                ' are backfillfull, applying rebalance configuration'
-            );
+            this.log_backfillfull(backfillfull_osds, backfillfull_pools);
             const new_pg_config = { ...this.state.pg.config, backfillfull_pools: backfillfull_pools.length ? backfillfull_pools : undefined };
             await this.save_pg_config(new_pg_config);
         }
     }
 
+    log_backfillfull(osds, pools)
+    {
+        for (const osd in osds)
+        {
+            const bf = osds[osd];
+            console.log('OSD '+osd+' may fill up during rebalance: capacity '+(bf.cap/1024n/1024n)+
+                ' MB, target user data '+(bf.clean/1024n/1024n)+' MB');
+        }
+        console.log(
+            (pools.length ? 'Pool(s) '+pools.join(', ') : 'No pools')+
+            ' are backfillfull now, applying rebalance configuration'
+        );
+    }
+
     schedule_update_stats()
     {
         if (this.stopped || this.stats_timer)
diff --git a/mon/stats.js b/mon/stats.js
index 6fec1f0e..cd0c347d 100644
--- a/mon/stats.js
+++ b/mon/stats.js
@@ -147,7 +147,7 @@ function sum_object_counts(state, global_config)
                 }
                 if (st.object_count)
                 {
-                    for (const pg_osd in (((state.pg.config.items||{})[pool_id]||{})[pg_num]||{}).osd_set||[])
+                    for (const pg_osd of (((state.pg.config.items||{})[pool_id]||{})[pg_num]||{}).osd_set||[])
                     {
                         if (!(pg_osd in clean_per_osd))
                         {
@@ -163,12 +163,19 @@ function sum_object_counts(state, global_config)
     }
     // If clean_per_osd[osd] is larger than osd capacity then it will fill up during rebalance
     let backfillfull_pools = {};
+    let backfillfull_osds = {};
     for (const osd in clean_per_osd)
     {
         const st = state.osd.stats[osd];
-        if (st && st.size && st.data_block_size && (BigInt(st.size)/BigInt(st.data_block_size)*
-            BigInt((global_config.osd_backfillfull_ratio||0.99)*1000000)/1000000n) < clean_per_osd[osd])
+        if (!st || !st.size || !st.data_block_size)
         {
+            continue;
+        }
+        let cap = BigInt(st.size)/BigInt(st.data_block_size);
+        cap = cap * BigInt((global_config.osd_backfillfull_ratio||0.99)*1000000) / 1000000n;
+        if (cap < clean_per_osd[osd])
+        {
+            backfillfull_osds[osd] = { cap: BigInt(st.size), clean: clean_per_osd[osd]*BigInt(st.data_block_size) };
             for (const pool_id in pool_per_osd[osd])
             {
                 backfillfull_pools[pool_id] = true;
@@ -176,7 +183,7 @@ function sum_object_counts(state, global_config)
         }
     }
     backfillfull_pools = Object.keys(backfillfull_pools).sort();
-    return { object_counts, object_bytes, backfillfull_pools };
+    return { object_counts, object_bytes, backfillfull_pools, backfillfull_osds };
 }
 
 // sum_inode_stats(this.state, this.prev_stats)