feat(explorer): indexed block search with a bounded cache

The explorer block cache searched with a LIKE query that had to escape
wildcards, grew without any limit, and retried a failed open on every
frame, including a two-second busy wait on the UI thread.

- Search now uses an instr() substring test on a lowercased needle, and
  only checks the height text when the query is all digits, since a
  non-numeric needle can never appear inside a decimal height.
- The table is pruned to the highest 20000 blocks, once per open and
  periodically as blocks are stored, so the file stops growing at its
  high-water mark.
- A failed open is negative-cached (30 s by default) instead of retried
  every frame, and the busy timeout drops to 250 ms.
- When the cache database is unavailable or refuses a write, the block
  spills to a small bounded in-memory map that both the list and search
  read, so a spilled block never appears in one but not the other; the
  spill is cleared when the chain tip regresses.
- A batch of blocks is written under one savepoint instead of one commit
  per block.

Tests: case-insensitive hash and height search, the negative-cache and
spill paths with tip-regression eviction, and batch store plus prune.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/ui/explorer/explorer_block_cache.cpp

@@ -7,8 +7,11 @@
 #include <sqlite3.h>
 
 #include <algorithm>
+#include <cctype>
+#include <cstdlib>
 #include <filesystem>
 #include <fstream>
+#include <iterator>
 #include <utility>
 
 namespace fs = std::filesystem;
@@ -88,37 +91,52 @@ std::string ExplorerBlockCache::defaultLegacyJsonPath()
     return (fs::path(util::Platform::getConfigDir()) / "explorer_blocks_cache.json").string();
 }
 
+bool ExplorerBlockCache::failOpen(const char* why)
+{
+    retry_not_before_ = std::chrono::steady_clock::now() + open_retry_interval_;
+    DEBUG_LOGF("Explorer block cache unavailable (%s); retrying in %lld ms\n", why,
+               static_cast<long long>(open_retry_interval_.count()));
+    return false;
+}
+
 bool ExplorerBlockCache::ensureOpen()
 {
     if (db_) return true;
+    // Negative cache: ~6 cache calls per frame used to each redo create_directories + open + PRAGMAs + CREATE
+    // TABLE (and a 2 s busy wait on the UI thread) when the file was unusable.
+    if (retry_not_before_ != std::chrono::steady_clock::time_point{} &&
+        std::chrono::steady_clock::now() < retry_not_before_) return false;
+    ++open_attempts_;
 
     try {
         fs::path path(database_path_);
         if (!path.parent_path().empty()) fs::create_directories(path.parent_path());
     } catch (const std::exception& e) {
-        DEBUG_LOGF("Failed to create explorer cache directory: %s\n", e.what());
+        return failOpen(e.what());
-        return false;
     }
 
     sqlite3* openedDb = nullptr;
     if (sqlite3_open(database_path_.c_str(), &openedDb) != SQLITE_OK) {
-        DEBUG_LOGF("Failed to open explorer block cache: %s\n",
+        const bool failed = failOpen(openedDb ? sqlite3_errmsg(openedDb) : "unknown error");
-                   openedDb ? sqlite3_errmsg(openedDb) : "unknown error");
         if (openedDb) sqlite3_close(openedDb);
-        return false;
+        return failed;
     }
 
     db_ = openedDb;
-    sqlite3_busy_timeout(db_, 2000);
+    // UI-thread-only handle: a short wait is enough; a miss falls back to the in-memory spill instead of
+    // freezing the frame for 2 s.
+    sqlite3_busy_timeout(db_, 250);
     exec("PRAGMA journal_mode=WAL");
     exec("PRAGMA synchronous=NORMAL");
 
     if (!createSchema()) {
         close();
-        return false;
+        return failOpen("schema");
     }
 
+    retry_not_before_ = {};
     migrateLegacyJsonIfNeeded();
+    pruneTo(kMaxCachedBlocks);   // once per open, never per frame
     return true;
 }
 
@@ -126,55 +144,67 @@ std::map<int, ExplorerBlockSummary> ExplorerBlockCache::loadRange(int minHeight,
 {
     std::map<int, ExplorerBlockSummary> blocks;
     if (minHeight > maxHeight) std::swap(minHeight, maxHeight);
-    if (minHeight < 1 || maxHeight < 1 || !ensureOpen()) return blocks;
+    if (minHeight < 1 || maxHeight < 1) return blocks;
 
-    Statement statement(db_,
+    if (ensureOpen()) {
-        "SELECT height, hash, tx_count, size, time, difficulty "
+        Statement statement(db_,
-        "FROM explorer_blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC");
+            "SELECT height, hash, tx_count, size, time, difficulty "
-    if (!statement.handle) return blocks;
+            "FROM explorer_blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC");
+        if (statement.handle) {
+            sqlite3_bind_int(statement.handle, 1, minHeight);
+            sqlite3_bind_int(statement.handle, 2, maxHeight);
 
-    sqlite3_bind_int(statement.handle, 1, minHeight);
+            while (sqlite3_step(statement.handle) == SQLITE_ROW) {
-    sqlite3_bind_int(statement.handle, 2, maxHeight);
+                ExplorerBlockSummary block;
-
+                block.height = sqlite3_column_int(statement.handle, 0);
-    while (sqlite3_step(statement.handle) == SQLITE_ROW) {
+                const unsigned char* hashText = sqlite3_column_text(statement.handle, 1);
-        ExplorerBlockSummary block;
+                if (hashText) block.hash = reinterpret_cast<const char*>(hashText);
-        block.height = sqlite3_column_int(statement.handle, 0);
+                block.tx_count = sqlite3_column_int(statement.handle, 2);
-        const unsigned char* hashText = sqlite3_column_text(statement.handle, 1);
+                block.size = sqlite3_column_int(statement.handle, 3);
-        if (hashText) block.hash = reinterpret_cast<const char*>(hashText);
+                block.time = static_cast<std::int64_t>(sqlite3_column_int64(statement.handle, 4));
-        block.tx_count = sqlite3_column_int(statement.handle, 2);
+                block.difficulty = sqlite3_column_double(statement.handle, 5);
-        block.size = sqlite3_column_int(statement.handle, 3);
+                if (block.height > 0 && !block.hash.empty()) {
-        block.time = static_cast<std::int64_t>(sqlite3_column_int64(statement.handle, 4));
+                    blocks[block.height] = std::move(block);
-        block.difficulty = sqlite3_column_double(statement.handle, 5);
+                }
-        if (block.height > 0 && !block.hash.empty()) {
+            }
-            blocks[block.height] = std::move(block);
         }
     }
 
+    // Heights the DB could not give (closed, or refused the insert) come from the in-memory spill.
+    for (auto it = spill_.lower_bound(minHeight); it != spill_.end() && it->first <= maxHeight; ++it)
+        blocks.emplace(it->first, it->second);   // emplace never overwrites a DB row
     return blocks;
 }
 
 std::vector<ExplorerBlockSummary> ExplorerBlockCache::searchBlocks(const std::string& query, int limit)
 {
     std::vector<ExplorerBlockSummary> results;
-    if (query.empty() || limit < 1 || !ensureOpen()) return results;
+    if (query.empty() || limit < 1) return results;
 
-    // Escape LIKE wildcards in the user input so '%' / '_' are matched literally.
+    // instr() is a plain substring test (no wildcards to escape). Daemon hashes are lowercase hex, so lowercase
-    std::string escaped;
+    // the needle (LIKE was ASCII case-insensitive). A non-numeric needle can never occur inside a decimal
-    escaped.reserve(query.size());
+    // height, so only an all-digit query also checks the height text.
-    for (char c : query) {
+    std::string needle = query;
-        if (c == '%' || c == '_' || c == '\\') escaped.push_back('\\');
+    for (char& c : needle) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
-        escaped.push_back(c);
+    const bool digitsOnly = std::all_of(needle.begin(), needle.end(),
+                                        [](char ch) { return ch >= '0' && ch <= '9'; });
+
+    if (!ensureOpen()) {   // DB down: search the in-memory spill instead of returning nothing
+        for (auto it = spill_.rbegin(); it != spill_.rend() && (int)results.size() < limit; ++it)
+            if (it->second.hash.find(needle) != std::string::npos ||
+                (digitsOnly && std::to_string(it->first).find(needle) != std::string::npos))
+                results.push_back(it->second);
+        return results;
     }
-    std::string pattern = "%" + escaped + "%";
 
-    Statement statement(db_,
+    Statement statement(db_, digitsOnly
-        "SELECT height, hash, tx_count, size, time, difficulty "
+        ? "SELECT height, hash, tx_count, size, time, difficulty FROM explorer_blocks "
-        "FROM explorer_blocks "
+          "WHERE instr(CAST(height AS TEXT), ?1) > 0 OR instr(hash, ?1) > 0 ORDER BY height DESC LIMIT ?2"
-        "WHERE CAST(height AS TEXT) LIKE ?1 ESCAPE '\\' OR hash LIKE ?1 ESCAPE '\\' "
+        : "SELECT height, hash, tx_count, size, time, difficulty FROM explorer_blocks "
-        "ORDER BY height DESC LIMIT ?2");
+          "WHERE instr(hash, ?1) > 0 ORDER BY height DESC LIMIT ?2");
     if (!statement.handle) return results;
 
-    sqlite3_bind_text(statement.handle, 1, pattern.c_str(), -1, SQLITE_TRANSIENT);
+    sqlite3_bind_text(statement.handle, 1, needle.c_str(), -1, SQLITE_TRANSIENT);
     sqlite3_bind_int(statement.handle, 2, limit);
 
     while (sqlite3_step(statement.handle) == SQLITE_ROW) {
@@ -189,10 +219,22 @@ std::vector<ExplorerBlockSummary> ExplorerBlockCache::searchBlocks(const std::st
         if (block.height > 0 && !block.hash.empty()) results.push_back(std::move(block));
     }
 
+    // A block the DB refused (transient busy) is held in spill_ and shows in the list via loadRange, so
+    // keep search consistent: merge any matching spill entry the DB query did not already return.
+    for (auto it = spill_.rbegin(); it != spill_.rend() && (int)results.size() < limit; ++it) {
+        if (!(it->second.hash.find(needle) != std::string::npos ||
+              (digitsOnly && std::to_string(it->first).find(needle) != std::string::npos)))
+            continue;
+        if (std::any_of(results.begin(), results.end(),
+                        [&](const ExplorerBlockSummary& b) { return b.height == it->first; }))
+            continue;
+        results.push_back(it->second);
+    }
+
     return results;
 }
 
-bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block)
+bool ExplorerBlockCache::storeBlockRow(const ExplorerBlockSummary& block)
 {
     if (block.height < 1 || block.hash.empty() || !ensureOpen()) return false;
 
@@ -211,6 +253,40 @@ bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block)
     return sqlite3_step(statement.handle) == SQLITE_DONE;
 }
 
+bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block)
+{
+    if (block.height < 1 || block.hash.empty()) return false;
+    const bool stored = storeBlockRow(block);
+    if (stored) {
+        spill_.erase(block.height);
+        if (++stores_since_prune_ >= 1000) { stores_since_prune_ = 0; pruneTo(kMaxCachedBlocks); }
+    } else {
+        spillBlock(block);
+    }
+    return stored;
+}
+
+bool ExplorerBlockCache::storeBlocks(const std::vector<ExplorerBlockSummary>& blocks)
+{
+    if (blocks.empty()) return true;
+    // A savepoint, not BEGIN: nests inside a caller's transaction (same reason as updateTip).
+    const bool grouped = ensureOpen() && exec("SAVEPOINT explorer_batch");
+    bool all = true;
+    for (const auto& b : blocks) all = storeBlock(b) && all;
+    if (grouped) exec("RELEASE explorer_batch");
+    return all;
+}
+
+void ExplorerBlockCache::spillBlock(const ExplorerBlockSummary& block)
+{
+    spill_[block.height] = block;
+    while (spill_.size() > kMaxSpillBlocks) {
+        const int farFirst = std::abs(spill_.begin()->first - block.height);
+        const int farLast = std::abs(spill_.rbegin()->first - block.height);
+        spill_.erase(farFirst >= farLast ? spill_.begin() : std::prev(spill_.end()));
+    }
+}
+
 int ExplorerBlockCache::cachedBlockCount()
 {
     if (!ensureOpen()) return 0;
@@ -221,10 +297,25 @@ int ExplorerBlockCache::cachedBlockCount()
 
 void ExplorerBlockCache::clearBlocks()
 {
+    spill_.clear();
     if (!ensureOpen()) return;
     exec("DELETE FROM explorer_blocks");
 }
 
+void ExplorerBlockCache::pruneTo(int maxRows)
+{
+    if (maxRows < 1 || !db_) return;
+    // Keep the maxRows highest blocks. The OFFSET row is the (maxRows+1)-th highest; when the table is smaller the
+    // subquery is NULL and nothing is deleted. Freed pages are reused, so the file stops growing at its high-water
+    // mark (no VACUUM / auto_vacuum: changing auto_vacuum needs a rebuild of existing databases).
+    Statement statement(db_,
+        "DELETE FROM explorer_blocks WHERE height <= "
+        "(SELECT height FROM explorer_blocks ORDER BY height DESC LIMIT 1 OFFSET ?1)");
+    if (!statement.handle) return;
+    sqlite3_bind_int(statement.handle, 1, maxRows);
+    sqlite3_step(statement.handle);
+}
+
 ExplorerBlockCache::SavedTipValidation ExplorerBlockCache::prepareValidation(
     int currentHeight, const std::string& currentBestHash)
 {
@@ -238,7 +329,15 @@ ExplorerBlockCache::SavedTipValidation ExplorerBlockCache::prepareValidation(
         currentHeight == tip_height_ && currentBestHash == tip_hash_) {
         return validation;
     }
-    if (!ensureOpen()) return validation;
+    if (!ensureOpen()) {
+        // No DB to validate against: keep the spill honest ourselves (tip regressed, or a different block at the same height).
+        if (!spill_.empty() && !currentBestHash.empty() &&
+            (currentHeight < spill_tip_height_ ||
+             (currentHeight == spill_tip_height_ && currentBestHash != spill_tip_hash_)))
+            spill_.clear();
+        if (!currentBestHash.empty()) { spill_tip_height_ = currentHeight; spill_tip_hash_ = currentBestHash; }
+        return validation;
+    }
 
     int savedHeight = getMetaInt("tip_height", 0);
     std::string savedHash = getMetaValue("tip_hash");

src/ui/explorer/explorer_block_cache.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <chrono>
+#include <cstddef>
 #include <cstdint>
 #include <map>
 #include <string>
@@ -45,9 +47,21 @@ public:
     // Fuzzy search over cached blocks: matches when the query is a substring of the height (as text)
     // or the block hash (case-insensitive). Returns newest-first, capped at `limit`.
     std::vector<ExplorerBlockSummary> searchBlocks(const std::string& query, int limit);
+    // Always returns the DB result; a block the DB refuses (unopenable file, failed insert) is kept in a small
+    // in-memory spill so the Explorer still shows it instead of refetching it forever.
     bool storeBlock(const ExplorerBlockSummary& block);
+    // storeBlock for a batch inside one SAVEPOINT (one WAL commit instead of one per block). True if every block hit the DB.
+    bool storeBlocks(const std::vector<ExplorerBlockSummary>& blocks);
     int cachedBlockCount();
     void clearBlocks();
+    // Keep only the `maxRows` highest blocks (data-safe: the table is a re-fetchable RPC cache).
+    void pruneTo(int maxRows);
+    static constexpr int kMaxCachedBlocks = 20000;
+
+    // A failed open is not retried until this interval has passed (default 30 s). openAttempts() counts real attempts.
+    void setOpenRetryInterval(std::chrono::milliseconds interval) { open_retry_interval_ = interval; }
+    int openAttempts() const { return open_attempts_; }
+    std::size_t spillCount() const { return spill_.size(); }
 
     SavedTipValidation prepareValidation(int currentHeight, const std::string& currentBestHash);
     void applySavedTipValidation(const SavedTipValidation& validation,
@@ -68,6 +82,9 @@ private:
     bool createSchema();
     void migrateLegacyJsonIfNeeded();
     void close();
+    bool storeBlockRow(const ExplorerBlockSummary& block);   // DB write only
+    void spillBlock(const ExplorerBlockSummary& block);
+    bool failOpen(const char* why);
 
     sqlite3* db_ = nullptr;
     std::string database_path_;
@@ -78,6 +95,19 @@ private:
     bool tip_known_ = false;
     int tip_height_ = 0;
     std::string tip_hash_;
+
+    // Failure latch for ensureOpen(): after a failed open nothing retries until retry_not_before_.
+    std::chrono::steady_clock::time_point retry_not_before_{};
+    std::chrono::milliseconds open_retry_interval_{30000};
+    int open_attempts_ = 0;
+    int stores_since_prune_ = 0;
+
+    // Blocks the DB refused, bounded and UI-thread only. Evicts the entry farthest from the last spilled height
+    // (the Explorer fetches around the visible window). Cleared with clearBlocks() and when the chain tip regresses.
+    static constexpr std::size_t kMaxSpillBlocks = 512;
+    std::map<int, ExplorerBlockSummary> spill_;
+    int spill_tip_height_ = 0;
+    std::string spill_tip_hash_;
 };
 
 } // namespace ui

tests/test_phase4.cpp

@@ -8590,6 +8590,90 @@ void testExplorerBlockCache()
         EXPECT_EQ(reopened.cachedBlockCount(), 0);
     }
 
+    // Search: substring of hash (case-insensitive) and of height text; non-digit needles skip heights.
+    {
+        ExplorerBlockCache searchCache(databasePath.string(), legacyPath.string());
+        for (int h = 100; h < 103; ++h) {
+            ExplorerBlockSummary b;
+            b.height = h;
+            b.hash = "abcdef" + std::to_string(h);
+            b.tx_count = 1;
+            EXPECT_TRUE(searchCache.storeBlock(b));
+        }
+        auto byHash = searchCache.searchBlocks("ABCDEF", 10);
+        EXPECT_EQ(byHash.size(), static_cast<size_t>(3));
+        EXPECT_EQ(byHash.front().height, 102);
+        EXPECT_EQ(searchCache.searchBlocks("101", 10).size(), static_cast<size_t>(1));
+        EXPECT_EQ(searchCache.searchBlocks("abcdef", 2).size(), static_cast<size_t>(2));
+        EXPECT_EQ(searchCache.searchBlocks("%", 10).size(), static_cast<size_t>(0));
+    }
+
+    // DB down (parent is a regular file): negative-cached open, rows spill to memory and stay searchable.
+    {
+        fs::path blocker = dir / "blocker";
+        { std::ofstream f(blocker); f << "x"; }
+        ExplorerBlockCache down((blocker / "sub" / "explorer.sqlite").string(), std::string());
+        down.setOpenRetryInterval(std::chrono::milliseconds(10000));
+        for (int i = 0; i < 50; ++i) EXPECT_FALSE(down.ensureOpen());
+        EXPECT_EQ(down.openAttempts(), 1);
+
+        ExplorerBlockSummary b;
+        b.height = 777;
+        b.hash = "deadbeef777";
+        b.tx_count = 2;
+        EXPECT_FALSE(down.storeBlock(b));
+        EXPECT_EQ(down.spillCount(), static_cast<size_t>(1));
+        EXPECT_EQ(down.loadRange(777, 777).size(), static_cast<size_t>(1));
+        EXPECT_EQ(down.searchBlocks("DEADBE", 10).size(), static_cast<size_t>(1));
+        EXPECT_EQ(down.searchBlocks("777", 10).size(), static_cast<size_t>(1));
+        EXPECT_EQ(down.searchBlocks("nomatch", 10).size(), static_cast<size_t>(0));
+
+        // Tip regression clears the spill; an advancing tip keeps it.
+        down.prepareValidation(800, "tip-800");
+        down.prepareValidation(801, "tip-801");
+        EXPECT_EQ(down.spillCount(), static_cast<size_t>(1));
+        down.prepareValidation(790, "tip-790");
+        EXPECT_EQ(down.spillCount(), static_cast<size_t>(0));
+
+        for (int h = 1; h <= 600; ++h) {
+            ExplorerBlockSummary s;
+            s.height = h;
+            s.hash = "h" + std::to_string(h);
+            down.storeBlock(s);
+        }
+        EXPECT_EQ(down.spillCount(), static_cast<size_t>(512));
+        EXPECT_EQ(down.loadRange(600, 600).size(), static_cast<size_t>(1));
+
+        down.clearBlocks();
+        EXPECT_EQ(down.spillCount(), static_cast<size_t>(0));
+        // A zero retry interval retries every call.
+        ExplorerBlockCache eager((blocker / "sub" / "explorer.sqlite").string(), std::string());
+        eager.setOpenRetryInterval(std::chrono::milliseconds(0));
+        EXPECT_FALSE(eager.ensureOpen());
+        EXPECT_FALSE(eager.ensureOpen());
+        EXPECT_EQ(eager.openAttempts(), 2);
+    }
+
+    // storeBlocks batches into one savepoint; pruneTo keeps the highest rows.
+    {
+        ExplorerBlockCache batch((dir / "batch.sqlite").string(), std::string());
+        std::vector<ExplorerBlockSummary> blocks;
+        for (int h = 1; h <= 50; ++h) {
+            ExplorerBlockSummary s;
+            s.height = h;
+            s.hash = "b" + std::to_string(h);
+            blocks.push_back(s);
+        }
+        EXPECT_TRUE(batch.storeBlocks(blocks));
+        EXPECT_EQ(batch.cachedBlockCount(), 50);
+        EXPECT_TRUE(batch.storeBlocks(blocks));
+        EXPECT_EQ(batch.cachedBlockCount(), 50);
+        batch.pruneTo(20);
+        EXPECT_EQ(batch.cachedBlockCount(), 20);
+        EXPECT_EQ(batch.loadRange(1, 50).size(), static_cast<size_t>(20));
+        EXPECT_EQ(batch.loadRange(1, 50).begin()->first, 31);
+    }
+
     cacheOwner.reset();   // close the database first: Windows can't delete an open file
     removeTree(dir);
 }