fix: cap mempool memory usage (-maxmempool) to bound an OOM DoS

This fork never ported Bitcoin's fee-ordered mempool eviction: CTxMemPool has no
TrimToSize/Expire, and LimitMempoolSize's body + call site are both commented out and
reference an undefined DEFAULT_MAX_MEMPOOL_SIZE. So the mempool had no total-size
ceiling. Together with the just-restored removeExpired() and near-free tx admission, a
peer could flood transactions to exhaust every node's memory (incl. pool/payout nodes).

Add a simple admission cap in AcceptToMemoryPool: once the pool exceeds -maxmempool it
refuses new admissions with DoS(0) (no ban -- a full pool isn't the peer's fault). This
is not fee-ordered eviction (that needs the absent TrimToSize machinery) but it bounds
the footprint; removeExpired() already evicts unmineable expired txs on each block
connect. New DEFAULT_MAX_MEMPOOL_SIZE=300 (MB, Bitcoin's default) is far above DragonX's
normal mempool, so normal operation is unaffected. Reviewed: bytes-vs-bytes comparison,
read under LOCK(pool.cs) on a recursive mutex (no deadlock); only the reorg re-add path
and new sends route through it, and only at 300MB.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/init.cpp

@@ -2002,8 +2002,14 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
     int64_t nBlockTreeDBCache = nTotalCache / 8;
 
     if (GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX) || GetBoolArg("-spentindex", DEFAULT_SPENTINDEX)) {
-        // enable 3/4 of the cache if addressindex and/or spentindex is enabled
+        // Give indexed (address/spent-index) nodes a larger index LevelDB read cache, but CAP it.
+        // With adaptive dbcache, nTotalCache is now multi-GB, so 3/4 of it is several GB -- far more
+        // than the index cache can use, while starving the in-memory UTXO set that actually speeds
+        // IBD (and this chunk is not shrinkable by the memory-pressure controller, which only adjusts
+        // the coins cache). Cap at 1 GiB so the adaptive budget flows to the coins cache. Tunable.
         nBlockTreeDBCache = nTotalCache * 3 / 4;
+        if (nBlockTreeDBCache > ((int64_t)1024 << 20))
+            nBlockTreeDBCache = ((int64_t)1024 << 20);
     } else {
         if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", false)) {
             nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB

src/main.cpp

@@ -2072,6 +2072,12 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
 
         {
             LOCK(pool.cs);
+            // Bound mempool memory: this fork never ported fee-ordered TrimToSize eviction, so
+            // instead of evicting we refuse new admissions once the pool exceeds -maxmempool.
+            // removeExpired() already clears unmineable expired txs on each block connect; this
+            // caps the total footprint against a flood of otherwise-minable/low-fee txs (OOM DoS).
+            if ( pool.DynamicMemoryUsage() > (size_t)GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000 )
+                return state.DoS(0, error("AcceptToMemoryPool: mempool full, rejecting tx %s", hash.ToString()), REJECT_INSUFFICIENTFEE, "mempool-full");
             // Store transaction in memory
             pool.addUnchecked(hash, entry, !IsInitialBlockDownload());
 
@@ -5100,6 +5106,14 @@ bool CheckBlockHeader(int32_t *futureblockp,int32_t height,CBlockIndex *pindex,
 
 int32_t hush_checkPOW(int32_t slowflag,CBlock *pblock,int32_t height);
 
+// RAII: save+restore the thread-local RandomX-skip flag around the verify-once dedup in CheckBlock,
+// so it can never clobber the miner's own fSkipRandomXValidation (TestBlockValidity -> ConnectBlock
+// re-entry) nor leak TRUE on an exception thrown out of hush_checkPOW.
+struct ScopedRandomXSkip {
+    bool prev;
+    ScopedRandomXSkip() : prev(GetSkipRandomXValidation()) { SetSkipRandomXValidation(true); }
+    ~ScopedRandomXSkip() { SetSkipRandomXValidation(prev); }
+};
 bool CheckBlock(int32_t *futureblockp,int32_t height,CBlockIndex *pindex,const CBlock& block, CValidationState& state,
                 libzcash::ProofVerifier& verifier,
                 bool fCheckPOW, bool fCheckMerkleRoot)
@@ -5130,8 +5144,16 @@ bool CheckBlock(int32_t *futureblockp,int32_t height,CBlockIndex *pindex,const C
             fprintf(stderr," failed hash ht.%d\n",height);
             return state.DoS(50, error("CheckBlock: proof of work failed"),REJECT_INVALID, "high-hash");
         }
-        if ( ASSETCHAINS_STAKED == 0 && hush_checkPOW(1,(CBlock *)&block,height) < 0 ) // checks Equihash
-            return state.DoS(100, error("CheckBlock: failed slow_checkPOW"),REJECT_INVALID, "failed-slow_checkPOW");
+        if ( ASSETCHAINS_STAKED == 0 ) {
+            // verify-once: CheckBlockHeader above already verified this block RandomX solution; skip the
+            // redundant recompute inside hush_checkPOW (the un-deduped 2nd verify, ~half the RandomX cost
+            // that dominates IBD). The scoped guard saves/restores the skip flag (never hardcodes false)
+            // so the miner's own skip is preserved and nothing leaks on throw. Equihash + PoW-target in
+            // hush_checkPOW still run.
+            ScopedRandomXSkip _rxskip;
+            if ( hush_checkPOW(1,(CBlock *)&block,height) < 0 )
+                return state.DoS(100, error("CheckBlock: failed slow_checkPOW"),REJECT_INVALID, "failed-slow_checkPOW");
+        }
     }
 	
     // Check the merkle root.
@@ -7779,7 +7801,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
                 if (state.IsInvalid(nDoS) && futureblock == 0)
                 {
                     if (nDoS > 0 && futureblock == 0)
-                        Misbehaving(pfrom->GetId(), nDoS/nDoS);
+                        Misbehaving(pfrom->GetId(), nDoS);
                     return error("invalid header received");
                 }
             }

src/main.h

@@ -66,6 +66,8 @@ class PrecomputedTransactionData;
 
 struct CNodeStateStats;
 #define DEFAULT_MEMPOOL_EXPIRY 1
+/** Default for -maxmempool, maximum megabytes of mempool memory usage */
+#define DEFAULT_MAX_MEMPOOL_SIZE 300
 #define _COINBASE_MATURITY 100
 
 /** Default for -blockmaxsize and -blockminsize, which control the range of sizes the mining code will create **/

src/pow.cpp

@@ -715,6 +715,7 @@ static int64_t nTimeRandomX = 0;   // cumulative RandomX validation time (us), r
 thread_local bool fSkipRandomXValidation = false;
 
 void SetSkipRandomXValidation(bool skip) { fSkipRandomXValidation = skip; }
+bool GetSkipRandomXValidation() { return fSkipRandomXValidation; }
 
 CBlockIndex *hush_chainactive(int32_t height);
 

src/pow.h

@@ -97,6 +97,7 @@ void RandomXValidatorShutdown();
 
 /** Set thread-local flag to skip RandomX validation (used by miner during TestBlockValidity) */
 void SetSkipRandomXValidation(bool skip);
+bool GetSkipRandomXValidation();
 
 /** Return the RandomX key rotation interval in blocks */
 int GetRandomXInterval();

src/txmempool.cpp

@@ -469,10 +469,17 @@ extern char SMART_CHAIN_SYMBOL[];
 
 std::vector<uint256> CTxMemPool::removeExpired(unsigned int nBlockHeight)
 {
-    CBlockIndex *tipindex;
-    // Remove expired txs from the mempool
+    // Remove expired txs from the mempool. (Regression fix: the scan that populates
+    // transactionsToRemove had been dropped, making this a no-op, so expired txs -- which
+    // can never be mined -- were never evicted and accumulated without bound.)
     LOCK(cs);
     list<CTransaction> transactionsToRemove;
+    for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) {
+        const CTransaction& tx = it->GetTx();
+        if (IsExpiredTx(tx, nBlockHeight)) {
+            transactionsToRemove.push_back(tx);
+        }
+    }
 
     std::vector<uint256> ids;
     for (const CTransaction& tx : transactionsToRemove) {