Report slow ops in OSD stats in etcd and show them in vitastor-cli status

src/osd/osd.cpp

@@ -535,10 +535,12 @@ void osd_t::print_stats()
 
 void osd_t::print_slow()
 {
-    bool has_slow = false;
+    cur_slow_op_primary = 0;
+    cur_slow_op_secondary = 0;
     char alloc[1024];
     timespec now;
     clock_gettime(CLOCK_REALTIME, &now);
+    // FIXME: Also track slow local blockstore ops and recovery/flush/scrub ops
     for (auto & kv: msgr.clients)
     {
         for (auto op: kv.second->received_ops)
@@ -608,6 +610,7 @@ void osd_t::print_slow()
                     op->req.hdr.opcode == OSD_OP_SEC_STABILIZE || op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK ||
                     op->req.hdr.opcode == OSD_OP_SEC_READ_BMP)
                 {
+                    cur_slow_op_secondary++;
                     bufprintf(" state=%d", op->bs_op ? PRIV(op->bs_op)->op_state : -1);
                     int wait_for = op->bs_op ? PRIV(op->bs_op)->wait_for : 0;
                     if (wait_for)
@@ -618,15 +621,19 @@ void osd_t::print_slow()
                 else if (op->req.hdr.opcode == OSD_OP_READ || op->req.hdr.opcode == OSD_OP_WRITE ||
                     op->req.hdr.opcode == OSD_OP_SYNC || op->req.hdr.opcode == OSD_OP_DELETE)
                 {
+                    cur_slow_op_primary++;
                     bufprintf(" state=%d", !op->op_data ? -1 : op->op_data->st);
                 }
+                else
+                {
+                    cur_slow_op_primary++;
+                }
 #undef bufprintf
                 printf("%s\n", alloc);
-                has_slow = true;
             }
         }
     }
-    if (has_slow && bs)
+    if ((cur_slow_op_primary+cur_slow_op_secondary) > 0 && bs)
     {
         bs->dump_diagnostics();
     }

src/osd/osd.h

@@ -151,6 +151,8 @@ class osd_t
     bool etcd_reporting_pg_state = false;
     bool etcd_reporting_stats = false;
     int print_stats_timer_id = -1, slow_log_timer_id = -1;
+    uint64_t cur_slow_op_primary = 0;
+    uint64_t cur_slow_op_secondary = 0;
 
     // peers and PGs
 

src/osd/osd_cluster.cpp

@@ -201,6 +201,14 @@ json11::Json osd_t::get_statistics()
     st["immediate_commit"] = immediate_commit == IMMEDIATE_ALL ? "all" : (immediate_commit == IMMEDIATE_SMALL ? "small" : "none");
     st["host"] = self_state["host"];
     st["version"] = VITASTOR_VERSION;
+    if (cur_slow_op_primary > 0)
+    {
+        st["slow_ops_primary"] = cur_slow_op_primary;
+    }
+    if (cur_slow_op_secondary > 0)
+    {
+        st["slow_ops_secondary"] = cur_slow_op_secondary;
+    }
     json11::Json::object op_stats, subop_stats;
     for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
     {