From edac0339163a3ba00749d23a3b277f985a944df0 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Wed, 19 May 2021 15:23:26 -0400
Subject: [PATCH 1/2] msg/async: configurable threshold for reaping dead
 connections

It is helpful to set this to 1 for tests.

Signed-off-by: Sage Weil <sage@newdream.net>
(cherry picked from commit 8129d6bb953015cc05db458afa6aa9b8f5f62614)
Signed-off-by: Gerald Yang <gerald.yang.tw@gmail.com>
---
 src/common/legacy_config_opts.h | 1 +
 src/common/options.cc           | 5 +++++
 src/msg/async/AsyncMessenger.h  | 4 +---
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index fcba93ce41f..c46a4b6ab96 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -143,6 +143,7 @@ OPTION(ms_dump_on_send, OPT_BOOL)           // hexdump msg to log on send
 OPTION(ms_dump_corrupt_message_level, OPT_INT)  // debug level to hexdump undecodeable messages at
 OPTION(ms_async_op_threads, OPT_U64)            // number of worker processing threads for async messenger created on init
 OPTION(ms_async_max_op_threads, OPT_U64)        // max number of worker processing threads for async messenger
+OPTION(ms_async_reap_threshold, OPT_U32)        // number of deleted connections before we reap
 OPTION(ms_async_rdma_device_name, OPT_STR)
 OPTION(ms_async_rdma_enable_hugepage, OPT_BOOL)
 OPTION(ms_async_rdma_buffer_size, OPT_INT)
diff --git a/src/common/options.cc b/src/common/options.cc
index 85f4203be36..0df5aec4ccd 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1135,6 +1135,11 @@ std::vector<Option> get_global_options() {
     .set_description("Maximum threadpool size of AsyncMessenger")
     .add_see_also("ms_async_op_threads"),
 
+    Option("ms_async_reap_threshold", Option::TYPE_UINT, Option::LEVEL_DEV)
+    .set_default(5)
+    .set_min(1)
+    .set_description("number of deleted connections before we reap"),
+
     Option("ms_async_rdma_device_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("")
     .set_description(""),
diff --git a/src/msg/async/AsyncMessenger.h b/src/msg/async/AsyncMessenger.h
index 3ce1b61ce60..140ad7fca98 100644
--- a/src/msg/async/AsyncMessenger.h
+++ b/src/msg/async/AsyncMessenger.h
@@ -209,8 +209,6 @@ private:
   entity_addrvec_t _filter_addrs(const entity_addrvec_t& addrs);
 
  private:
-  static const uint64_t ReapDeadConnectionThreshold = 5;
-
   NetworkStack *stack;
   std::vector<Processor*> processors;
   friend class Processor;
@@ -404,7 +402,7 @@ public:
     deleted_conns.emplace(std::move(conn));
     conn->unregister();
 
-    if (deleted_conns.size() >= ReapDeadConnectionThreshold) {
+    if (deleted_conns.size() >= cct->_conf->ms_async_reap_threshold) {
       local_worker->center.dispatch_event_external(reap_handler);
     }
   }
-- 
2.25.1