ObsidianDragon/src/rpc/rpc_busy_gate.h

#pragma once

// Keeping the GUI from flooding a busy node (S6). dragonxd's HTTP server has 8 workers and a 16-deep
// queue. While a long operation holds cs_main (a witness rebuild can take an hour), every request that
// needs cs_main waits, and a client that gives up does NOT free its slot. The GUI's polls used to time
// out and be re-sent until all 24 slots were its own abandoned requests, after which even "stop" was
// refused. The gate stops that: after the first sign the node is busy (a reply timeout, or a queue-full
// refusal), the GUI's clients send nothing that needs cs_main until a single "canary" request (which
// waits as long as it takes) gets through. Pure and header-only; shared by the GUI's RPC clients.

#include "rpc_failure.h"

#include <chrono>
#include <mutex>
#include <string>

namespace dragonx {
namespace rpc {

// Kinds that mean "reached a node that is too busy to answer", as opposed to "no node there".
inline bool rpcKindMeansBusy(RpcFailureKind kind)
{
    return kind == RpcFailureKind::ReplyTimeout || kind == RpcFailureKind::QueueFull ||
           kind == RpcFailureKind::Deferred;
}

// Kinds that mean the connection itself is in doubt (counted toward a reconnect). Busy kinds, a
// warmup reply, an application error and our own abort are not: the node is there.
inline bool rpcKindMeansLost(RpcFailureKind kind)
{
    switch (kind) {
    case RpcFailureKind::None:
    case RpcFailureKind::ReplyTimeout:
    case RpcFailureKind::QueueFull:
    case RpcFailureKind::Deferred:
    case RpcFailureKind::Warmup:
    case RpcFailureKind::AppError:
    case RpcFailureKind::Aborted:
        return false;
    default:
        return true;
    }
}

class RpcBusyGate {
public:
    // Methods dragonxd runs without cs_main: always sent, and their results say nothing about cs_main.
    static bool isLockFree(const std::string& method)
    {
        // getmempoolinfo takes only mempool.cs (rpc/blockchain.cpp), and the dispatcher no global lock.
        return method == "stop" || method == "help" || method == "getrescaninfo" || method == "getmempoolinfo";
    }

    // May this request be sent now? Lock-free methods always; the rest only while the node isn't busy.
    bool admit(const std::string& method) const
    {
        if (isLockFree(method)) return true;
        std::lock_guard<std::mutex> lk(mx_);
        return !busy_;
    }

    // How a sent request went. Any real answer (even an error) means the node is serving cs_main
    // requests again; a reply timeout or a queue-full refusal means it isn't. Other failures (nothing
    // listening, auth, our own abort) say nothing about busyness and leave the state alone.
    void record(const std::string& method, RpcFailureKind kind)
    {
        if (isLockFree(method)) return;
        std::lock_guard<std::mutex> lk(mx_);
        if (kind == RpcFailureKind::None || kind == RpcFailureKind::AppError || kind == RpcFailureKind::Warmup) {
            clearLocked();
        } else if (kind == RpcFailureKind::ReplyTimeout || kind == RpcFailureKind::QueueFull) {
            if (!busy_) {
                busy_ = true;
                since_ = std::chrono::steady_clock::now();
                ++busyPeriods_;
            }
            if (kind == RpcFailureKind::ReplyTimeout) ++abandoned_;   // it still holds a daemon slot
        }
    }

    // The canary got through (or the node is gone): poll normally again.
    void clear()
    {
        std::lock_guard<std::mutex> lk(mx_);
        clearLocked();
    }

    bool busy() const
    {
        std::lock_guard<std::mutex> lk(mx_);
        return busy_;
    }
    // Requests we gave up on that the node may still be holding, since the gate last cleared.
    int abandoned() const
    {
        std::lock_guard<std::mutex> lk(mx_);
        return abandoned_;
    }
    // Increments each time the gate turns busy: lets the owner arm exactly one canary per busy period.
    int busyPeriods() const
    {
        std::lock_guard<std::mutex> lk(mx_);
        return busyPeriods_;
    }
    double busySeconds() const
    {
        std::lock_guard<std::mutex> lk(mx_);
        if (!busy_) return 0.0;
        return std::chrono::duration<double>(std::chrono::steady_clock::now() - since_).count();
    }

private:
    void clearLocked()
    {
        busy_ = false;
        abandoned_ = 0;
    }

    mutable std::mutex mx_;
    bool busy_ = false;
    int abandoned_ = 0;
    int busyPeriods_ = 0;
    std::chrono::steady_clock::time_point since_{};
};

/**
 * How long to wait before the next connection attempt after a failure of this kind. Each repeat of the
 * same kind waits longer, up to its cap; a success or a different kind starts over.
 *   nothing listening / connect timeout: 1, 2, 5s (the node is starting, or gone)
 *   queue full: 2, 4, 8, 16, 30s (the queue is someone's; hammering it only adds to it)
 *   busy (reply timeout / deferred): 30s; the canary brings the connection back as soon as it can
 *   anything else: 4s
 */
class ConnectRetryBackoff {
public:
    float next(RpcFailureKind kind)
    {
        streak_ = (kind == last_) ? streak_ + 1 : 0;
        last_ = kind;
        switch (kind) {
        case RpcFailureKind::NotListening:
        case RpcFailureKind::ConnectTimeout: {
            static const float steps[] = {1.0f, 2.0f, 5.0f};
            return steps[streak_ < 2 ? streak_ : 2];
        }
        case RpcFailureKind::QueueFull: {
            static const float steps[] = {2.0f, 4.0f, 8.0f, 16.0f, 30.0f};
            return steps[streak_ < 4 ? streak_ : 4];
        }
        case RpcFailureKind::ReplyTimeout:
        case RpcFailureKind::Deferred:
            return 30.0f;
        default:
            return 4.0f;
        }
    }
    void reset()
    {
        streak_ = 0;
        last_ = RpcFailureKind::None;
    }

private:
    RpcFailureKind last_ = RpcFailureKind::None;
    int streak_ = 0;
};

}  // namespace rpc
}  // namespace dragonx