#pragma once // Keeping the GUI from flooding a busy node (S6). dragonxd's HTTP server has 8 workers and a 16-deep // queue. While a long operation holds cs_main (a witness rebuild can take an hour), every request that // needs cs_main waits, and a client that gives up does NOT free its slot. The GUI's polls used to time // out and be re-sent until all 24 slots were its own abandoned requests, after which even "stop" was // refused. The gate stops that: after the first sign the node is busy (a reply timeout, or a queue-full // refusal), the GUI's clients send nothing that needs cs_main until a single "canary" request (which // waits as long as it takes) gets through. Pure and header-only; shared by the GUI's RPC clients. #include "rpc_failure.h" #include #include #include namespace dragonx { namespace rpc { // Kinds that mean "reached a node that is too busy to answer", as opposed to "no node there". inline bool rpcKindMeansBusy(RpcFailureKind kind) { return kind == RpcFailureKind::ReplyTimeout || kind == RpcFailureKind::QueueFull || kind == RpcFailureKind::Deferred; } // Kinds that mean the connection itself is in doubt (counted toward a reconnect). Busy kinds, a // warmup reply, an application error and our own abort are not: the node is there. inline bool rpcKindMeansLost(RpcFailureKind kind) { switch (kind) { case RpcFailureKind::None: case RpcFailureKind::ReplyTimeout: case RpcFailureKind::QueueFull: case RpcFailureKind::Deferred: case RpcFailureKind::Warmup: case RpcFailureKind::AppError: case RpcFailureKind::Aborted: return false; default: return true; } } class RpcBusyGate { public: // Methods dragonxd runs without cs_main: always sent, and their results say nothing about cs_main. static bool isLockFree(const std::string& method) { // getmempoolinfo takes only mempool.cs (rpc/blockchain.cpp), and the dispatcher no global lock. return method == "stop" || method == "help" || method == "getrescaninfo" || method == "getmempoolinfo"; } // May this request be sent now? Lock-free methods always; the rest only while the node isn't busy. bool admit(const std::string& method) const { if (isLockFree(method)) return true; std::lock_guard lk(mx_); return !busy_; } // How a sent request went. Any real answer (even an error) means the node is serving cs_main // requests again; a reply timeout or a queue-full refusal means it isn't. Other failures (nothing // listening, auth, our own abort) say nothing about busyness and leave the state alone. void record(const std::string& method, RpcFailureKind kind) { if (isLockFree(method)) return; std::lock_guard lk(mx_); if (kind == RpcFailureKind::None || kind == RpcFailureKind::AppError || kind == RpcFailureKind::Warmup) { clearLocked(); } else if (kind == RpcFailureKind::ReplyTimeout || kind == RpcFailureKind::QueueFull) { if (!busy_) { busy_ = true; since_ = std::chrono::steady_clock::now(); ++busyPeriods_; } if (kind == RpcFailureKind::ReplyTimeout) ++abandoned_; // it still holds a daemon slot } } // The canary got through (or the node is gone): poll normally again. void clear() { std::lock_guard lk(mx_); clearLocked(); } bool busy() const { std::lock_guard lk(mx_); return busy_; } // Requests we gave up on that the node may still be holding, since the gate last cleared. int abandoned() const { std::lock_guard lk(mx_); return abandoned_; } // Increments each time the gate turns busy: lets the owner arm exactly one canary per busy period. int busyPeriods() const { std::lock_guard lk(mx_); return busyPeriods_; } double busySeconds() const { std::lock_guard lk(mx_); if (!busy_) return 0.0; return std::chrono::duration(std::chrono::steady_clock::now() - since_).count(); } private: void clearLocked() { busy_ = false; abandoned_ = 0; } mutable std::mutex mx_; bool busy_ = false; int abandoned_ = 0; int busyPeriods_ = 0; std::chrono::steady_clock::time_point since_{}; }; /** * How long to wait before the next connection attempt after a failure of this kind. Each repeat of the * same kind waits longer, up to its cap; a success or a different kind starts over. * nothing listening / connect timeout: 1, 2, 5s (the node is starting, or gone) * queue full: 2, 4, 8, 16, 30s (the queue is someone's; hammering it only adds to it) * busy (reply timeout / deferred): 30s; the canary brings the connection back as soon as it can * anything else: 4s */ class ConnectRetryBackoff { public: float next(RpcFailureKind kind) { streak_ = (kind == last_) ? streak_ + 1 : 0; last_ = kind; switch (kind) { case RpcFailureKind::NotListening: case RpcFailureKind::ConnectTimeout: { static const float steps[] = {1.0f, 2.0f, 5.0f}; return steps[streak_ < 2 ? streak_ : 2]; } case RpcFailureKind::QueueFull: { static const float steps[] = {2.0f, 4.0f, 8.0f, 16.0f, 30.0f}; return steps[streak_ < 4 ? streak_ : 4]; } case RpcFailureKind::ReplyTimeout: case RpcFailureKind::Deferred: return 30.0f; default: return 4.0f; } } void reset() { streak_ = 0; last_ = RpcFailureKind::None; } private: RpcFailureKind last_ = RpcFailureKind::None; int streak_ = 0; }; } // namespace rpc } // namespace dragonx