From 236d79b3f8c7f372de5d83aa67b87d468c679d26 Mon Sep 17 00:00:00 2001
From: Duke Leto <duke@leto.net>
Date: Thu, 5 Aug 2021 12:44:30 -0400
Subject: [PATCH] Add rpcinfo rpc to debug deadlocks

We seem to have deadlock bugs in our RPC system, most likely inherited from ZEC or BTC.
Since some Hush RPC's take longer (such as anything with Sietch protections), the deadlocks
are more likely to occur. Eventually all RPC slots are used up and no more RPC commands
can be sent to hushd. This is why the "plz_stop" feature was implemented, but that is just
a workaround to restart the server. We must find and fix the root cause.

This rpc will allow us to see when we are getting close to our maximum work queue depth
and hopefully help us learn exactly what is happening.
---
 src/httpserver.cpp | 15 +++++++++++++++
 src/httpserver.h   |  3 +++
 src/rpc/misc.cpp   | 21 +++++++++++++++++++++
 3 files changed, 39 insertions(+)

diff --git a/src/httpserver.cpp b/src/httpserver.cpp
index 0e4e42c2b..59b8eeb1d 100644
--- a/src/httpserver.cpp
+++ b/src/httpserver.cpp
@@ -157,6 +157,11 @@ public:
         boost::unique_lock<boost::mutex> lock(cs);
         return queue.size();
     }
+    size_t MaxDepth()
+    {
+        boost::unique_lock<boost::mutex> lock(cs);
+        return maxDepth;
+    }
 };
 
 struct HTTPPathHandler
@@ -186,6 +191,16 @@ std::vector<HTTPPathHandler> pathHandlers;
 //! Bound listening sockets
 std::vector<evhttp_bound_socket *> boundSockets;
 
+
+int getWorkQueueDepth()
+{
+    return workQueue->Depth();
+}
+int getWorkQueueMaxDepth()
+{
+    return workQueue->MaxDepth();
+}
+
 /** Check if a network address is allowed to access the HTTP server */
 static bool ClientAllowed(const CNetAddr& netaddr)
 {
diff --git a/src/httpserver.h b/src/httpserver.h
index b903f91cd..7000bd45f 100644
--- a/src/httpserver.h
+++ b/src/httpserver.h
@@ -24,6 +24,9 @@ struct event_base;
 class CService;
 class HTTPRequest;
 
+int getWorkQueueDepth();
+int getWorkQueueMaxDepth();
+
 /** Initialize HTTP server.
  * Call this before RegisterHTTPHandler or EventBase().
  */
diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp
index fb1b3116c..37c27831a 100644
--- a/src/rpc/misc.cpp
+++ b/src/rpc/misc.cpp
@@ -178,6 +178,26 @@ UniValue geterablockheights(const UniValue& params, bool fHelp, const CPubKey& m
     return(ret);
 }
 
+extern int getWorkQueueDepth();
+extern int getWorkQueueMaxDepth();
+
+UniValue rpcinfo(const UniValue& params, bool fHelp, const CPubKey& mypk)
+{
+    UniValue result(UniValue::VOBJ);
+    if (fHelp || params.size() != 0) {
+        throw runtime_error(
+            "rpcinfo\n"
+            "Returns an object containing various RPC state info.\n"
+        );
+    }
+    LOCK(cs_main);
+    int depth = getWorkQueueDepth();
+
+    result.push_back(Pair("work_queue_depth", depth));
+    result.push_back(Pair("work_queue_max_depth", getWorkQueueMaxDepth() ));
+    return result;
+}
+
 UniValue getinfo(const UniValue& params, bool fHelp, const CPubKey& mypk)
 {
     uint256 notarized_hash,notarized_desttxid; int32_t prevMoMheight,notarized_height,longestchain,hushnotarized_height,txid_height;
@@ -1556,6 +1576,7 @@ static const CRPCCommand commands[] =
     { "util",               "z_validateaddress",      &z_validateaddress,      true  }, /* uses wallet if enabled */
     { "util",               "createmultisig",         &createmultisig,         true  },
     { "util",               "verifymessage",          &verifymessage,          true  },
+    { "util",               "rpcinfo",                &rpcinfo,                true  },
 
     /* Not shown in help */
     { "hidden",             "setmocktime",            &setmocktime,            true  },