DragonX/ObsidianDragon
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

feat(explorer): indexed block search with a bounded cache

Browse Source 
The explorer block cache searched with a LIKE query that had to escape
wildcards, grew without any limit, and retried a failed open on every
frame, including a two-second busy wait on the UI thread.

- Search now uses an instr() substring test on a lowercased needle, and
  only checks the height text when the query is all digits, since a
  non-numeric needle can never appear inside a decimal height.
- The table is pruned to the highest 20000 blocks, once per open and
  periodically as blocks are stored, so the file stops growing at its
  high-water mark.
- A failed open is negative-cached (30 s by default) instead of retried
  every frame, and the busy timeout drops to 250 ms.
- When the cache database is unavailable or refuses a write, the block
  spills to a small bounded in-memory map that both the list and search
  read, so a spilled block never appears in one but not the other; the
  spill is cleared when the chain tip regresses.
- A batch of blocks is written under one savepoint instead of one commit
  per block.

Tests: case-insensitive hash and height search, the negative-cache and
spill paths with tip-regression eviction, and batch store plus prune.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DanS
2026-10-13 09:24:51 -05:00
co-authored by Claude Opus 4.8
parent c3f981b52e
commit ebbeb846e8
3 changed files with 255 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ui/explorer/explorer_block_cache.cpp
+141 -42
View File
@@ -7,8 +7,11 @@
#include <sqlite3.h>
#include <algorithm>
#include <cctype>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iterator>
#include <utility>
namespace fs = std::filesystem;
@@ -88,37 +91,52 @@ std::string ExplorerBlockCache::defaultLegacyJsonPath()
return (fs::path(util::Platform::getConfigDir()) / "explorer_blocks_cache.json").string();
}
bool ExplorerBlockCache::failOpen(const char* why)
{
retry_not_before_ = std::chrono::steady_clock::now() + open_retry_interval_;
DEBUG_LOGF("Explorer block cache unavailable (%s); retrying in %lld ms\n", why,
static_cast<long long>(open_retry_interval_.count()));
return false;
}
bool ExplorerBlockCache::ensureOpen()
{
if (db_) return true;
// Negative cache: ~6 cache calls per frame used to each redo create_directories + open + PRAGMAs + CREATE
// TABLE (and a 2 s busy wait on the UI thread) when the file was unusable.
if (retry_not_before_ != std::chrono::steady_clock::time_point{} &&
std::chrono::steady_clock::now() < retry_not_before_) return false;
++open_attempts_;
try {
fs::path path(database_path_);
if (!path.parent_path().empty()) fs::create_directories(path.parent_path());
} catch (const std::exception& e) {
DEBUG_LOGF("Failed to create explorer cache directory: %s\n", e.what());
return false;
return failOpen(e.what());
}
sqlite3* openedDb = nullptr;
if (sqlite3_open(database_path_.c_str(), &openedDb) != SQLITE_OK) {
DEBUG_LOGF("Failed to open explorer block cache: %s\n",
openedDb ? sqlite3_errmsg(openedDb) : "unknown error");
const bool failed = failOpen(openedDb ? sqlite3_errmsg(openedDb) : "unknown error");
if (openedDb) sqlite3_close(openedDb);
return false;
return failed;
}
db_ = openedDb;
sqlite3_busy_timeout(db_, 2000);
// UI-thread-only handle: a short wait is enough; a miss falls back to the in-memory spill instead of
// freezing the frame for 2 s.
sqlite3_busy_timeout(db_, 250);
exec("PRAGMA journal_mode=WAL");
exec("PRAGMA synchronous=NORMAL");
if (!createSchema()) {
close();
return false;
return failOpen("schema");
}
retry_not_before_ = {};
migrateLegacyJsonIfNeeded();
pruneTo(kMaxCachedBlocks); // once per open, never per frame
return true;
}
@@ -126,55 +144,67 @@ std::map<int, ExplorerBlockSummary> ExplorerBlockCache::loadRange(int minHeight,
{
std::map<int, ExplorerBlockSummary> blocks;
if (minHeight > maxHeight) std::swap(minHeight, maxHeight);
if (minHeight < 1 || maxHeight < 1 || !ensureOpen()) return blocks;
if (minHeight < 1 || maxHeight < 1) return blocks;
Statement statement(db_,
"SELECT height, hash, tx_count, size, time, difficulty "
"FROM explorer_blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC");
if (!statement.handle) return blocks;
if (ensureOpen()) {
Statement statement(db_,
"SELECT height, hash, tx_count, size, time, difficulty "
"FROM explorer_blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC");
if (statement.handle) {
sqlite3_bind_int(statement.handle, 1, minHeight);
sqlite3_bind_int(statement.handle, 2, maxHeight);
sqlite3_bind_int(statement.handle, 1, minHeight);
sqlite3_bind_int(statement.handle, 2, maxHeight);
while (sqlite3_step(statement.handle) == SQLITE_ROW) {
ExplorerBlockSummary block;
block.height = sqlite3_column_int(statement.handle, 0);
const unsigned char* hashText = sqlite3_column_text(statement.handle, 1);
if (hashText) block.hash = reinterpret_cast<const char*>(hashText);
block.tx_count = sqlite3_column_int(statement.handle, 2);
block.size = sqlite3_column_int(statement.handle, 3);
block.time = static_cast<std::int64_t>(sqlite3_column_int64(statement.handle, 4));
block.difficulty = sqlite3_column_double(statement.handle, 5);
if (block.height > 0 && !block.hash.empty()) {
blocks[block.height] = std::move(block);
while (sqlite3_step(statement.handle) == SQLITE_ROW) {
ExplorerBlockSummary block;
block.height = sqlite3_column_int(statement.handle, 0);
const unsigned char* hashText = sqlite3_column_text(statement.handle, 1);
if (hashText) block.hash = reinterpret_cast<const char*>(hashText);
block.tx_count = sqlite3_column_int(statement.handle, 2);
block.size = sqlite3_column_int(statement.handle, 3);
block.time = static_cast<std::int64_t>(sqlite3_column_int64(statement.handle, 4));
block.difficulty = sqlite3_column_double(statement.handle, 5);
if (block.height > 0 && !block.hash.empty()) {
blocks[block.height] = std::move(block);
}
}
}
}
// Heights the DB could not give (closed, or refused the insert) come from the in-memory spill.
for (auto it = spill_.lower_bound(minHeight); it != spill_.end() && it->first <= maxHeight; ++it)
blocks.emplace(it->first, it->second); // emplace never overwrites a DB row
return blocks;
}
std::vector<ExplorerBlockSummary> ExplorerBlockCache::searchBlocks(const std::string& query, int limit)
{
std::vector<ExplorerBlockSummary> results;
if (query.empty() || limit < 1 || !ensureOpen()) return results;
if (query.empty() || limit < 1) return results;
// Escape LIKE wildcards in the user input so '%' / '_' are matched literally.
std::string escaped;
escaped.reserve(query.size());
for (char c : query) {
if (c == '%' || c == '_' || c == '\\') escaped.push_back('\\');
escaped.push_back(c);
// instr() is a plain substring test (no wildcards to escape). Daemon hashes are lowercase hex, so lowercase
// the needle (LIKE was ASCII case-insensitive). A non-numeric needle can never occur inside a decimal
// height, so only an all-digit query also checks the height text.
std::string needle = query;
for (char& c : needle) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
const bool digitsOnly = std::all_of(needle.begin(), needle.end(),
[](char ch) { return ch >= '0' && ch <= '9'; });
if (!ensureOpen()) { // DB down: search the in-memory spill instead of returning nothing
for (auto it = spill_.rbegin(); it != spill_.rend() && (int)results.size() < limit; ++it)
if (it->second.hash.find(needle) != std::string::npos ||
(digitsOnly && std::to_string(it->first).find(needle) != std::string::npos))
results.push_back(it->second);
return results;
}
std::string pattern = "%" + escaped + "%";
Statement statement(db_,
"SELECT height, hash, tx_count, size, time, difficulty "
"FROM explorer_blocks "
"WHERE CAST(height AS TEXT) LIKE ?1 ESCAPE '\\' OR hash LIKE ?1 ESCAPE '\\' "
"ORDER BY height DESC LIMIT ?2");
Statement statement(db_, digitsOnly
? "SELECT height, hash, tx_count, size, time, difficulty FROM explorer_blocks "
"WHERE instr(CAST(height AS TEXT), ?1) > 0 OR instr(hash, ?1) > 0 ORDER BY height DESC LIMIT ?2"
: "SELECT height, hash, tx_count, size, time, difficulty FROM explorer_blocks "
"WHERE instr(hash, ?1) > 0 ORDER BY height DESC LIMIT ?2");
if (!statement.handle) return results;
sqlite3_bind_text(statement.handle, 1, pattern.c_str(), -1, SQLITE_TRANSIENT);
sqlite3_bind_text(statement.handle, 1, needle.c_str(), -1, SQLITE_TRANSIENT);
sqlite3_bind_int(statement.handle, 2, limit);
while (sqlite3_step(statement.handle) == SQLITE_ROW) {
@@ -189,10 +219,22 @@ std::vector<ExplorerBlockSummary> ExplorerBlockCache::searchBlocks(const std::st
if (block.height > 0 && !block.hash.empty()) results.push_back(std::move(block));
}
// A block the DB refused (transient busy) is held in spill_ and shows in the list via loadRange, so
// keep search consistent: merge any matching spill entry the DB query did not already return.
for (auto it = spill_.rbegin(); it != spill_.rend() && (int)results.size() < limit; ++it) {
if (!(it->second.hash.find(needle) != std::string::npos ||
(digitsOnly && std::to_string(it->first).find(needle) != std::string::npos)))
continue;
if (std::any_of(results.begin(), results.end(),
[&](const ExplorerBlockSummary& b) { return b.height == it->first; }))
continue;
results.push_back(it->second);
}
return results;
}
bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block)
bool ExplorerBlockCache::storeBlockRow(const ExplorerBlockSummary& block)
{
if (block.height < 1 || block.hash.empty() || !ensureOpen()) return false;
@@ -211,6 +253,40 @@ bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block)
return sqlite3_step(statement.handle) == SQLITE_DONE;
}
bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block)
{
if (block.height < 1 || block.hash.empty()) return false;
const bool stored = storeBlockRow(block);
if (stored) {
spill_.erase(block.height);
if (++stores_since_prune_ >= 1000) { stores_since_prune_ = 0; pruneTo(kMaxCachedBlocks); }
} else {
spillBlock(block);
}
return stored;
}
bool ExplorerBlockCache::storeBlocks(const std::vector<ExplorerBlockSummary>& blocks)
{
if (blocks.empty()) return true;
// A savepoint, not BEGIN: nests inside a caller's transaction (same reason as updateTip).
const bool grouped = ensureOpen() && exec("SAVEPOINT explorer_batch");
bool all = true;
for (const auto& b : blocks) all = storeBlock(b) && all;
if (grouped) exec("RELEASE explorer_batch");
return all;
}
void ExplorerBlockCache::spillBlock(const ExplorerBlockSummary& block)
{
spill_[block.height] = block;
while (spill_.size() > kMaxSpillBlocks) {
const int farFirst = std::abs(spill_.begin()->first - block.height);
const int farLast = std::abs(spill_.rbegin()->first - block.height);
spill_.erase(farFirst >= farLast ? spill_.begin() : std::prev(spill_.end()));
}
}
int ExplorerBlockCache::cachedBlockCount()
{
if (!ensureOpen()) return 0;
@@ -221,10 +297,25 @@ int ExplorerBlockCache::cachedBlockCount()
void ExplorerBlockCache::clearBlocks()
{
spill_.clear();
if (!ensureOpen()) return;
exec("DELETE FROM explorer_blocks");
}
void ExplorerBlockCache::pruneTo(int maxRows)
{
if (maxRows < 1 || !db_) return;
// Keep the maxRows highest blocks. The OFFSET row is the (maxRows+1)-th highest; when the table is smaller the
// subquery is NULL and nothing is deleted. Freed pages are reused, so the file stops growing at its high-water
// mark (no VACUUM / auto_vacuum: changing auto_vacuum needs a rebuild of existing databases).
Statement statement(db_,
"DELETE FROM explorer_blocks WHERE height <= "
"(SELECT height FROM explorer_blocks ORDER BY height DESC LIMIT 1 OFFSET ?1)");
if (!statement.handle) return;
sqlite3_bind_int(statement.handle, 1, maxRows);
sqlite3_step(statement.handle);
}
ExplorerBlockCache::SavedTipValidation ExplorerBlockCache::prepareValidation(
int currentHeight, const std::string& currentBestHash)
{
@@ -238,7 +329,15 @@ ExplorerBlockCache::SavedTipValidation ExplorerBlockCache::prepareValidation(
currentHeight == tip_height_ && currentBestHash == tip_hash_) {
return validation;
}
if (!ensureOpen()) return validation;
if (!ensureOpen()) {
// No DB to validate against: keep the spill honest ourselves (tip regressed, or a different block at the same height).
if (!spill_.empty() && !currentBestHash.empty() &&
(currentHeight < spill_tip_height_ ||
(currentHeight == spill_tip_height_ && currentBestHash != spill_tip_hash_)))
spill_.clear();
if (!currentBestHash.empty()) { spill_tip_height_ = currentHeight; spill_tip_hash_ = currentBestHash; }
return validation;
}
int savedHeight = getMetaInt("tip_height", 0);
std::string savedHash = getMetaValue("tip_hash");
src/ui/explorer/explorer_block_cache.h
+30
View File
@@ -1,5 +1,7 @@
#pragma once
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <map>
#include <string>
@@ -45,9 +47,21 @@ public:
// Fuzzy search over cached blocks: matches when the query is a substring of the height (as text)
// or the block hash (case-insensitive). Returns newest-first, capped at `limit`.
std::vector<ExplorerBlockSummary> searchBlocks(const std::string& query, int limit);
// Always returns the DB result; a block the DB refuses (unopenable file, failed insert) is kept in a small
// in-memory spill so the Explorer still shows it instead of refetching it forever.
bool storeBlock(const ExplorerBlockSummary& block);
// storeBlock for a batch inside one SAVEPOINT (one WAL commit instead of one per block). True if every block hit the DB.
bool storeBlocks(const std::vector<ExplorerBlockSummary>& blocks);
int cachedBlockCount();
void clearBlocks();
// Keep only the `maxRows` highest blocks (data-safe: the table is a re-fetchable RPC cache).
void pruneTo(int maxRows);
static constexpr int kMaxCachedBlocks = 20000;
// A failed open is not retried until this interval has passed (default 30 s). openAttempts() counts real attempts.
void setOpenRetryInterval(std::chrono::milliseconds interval) { open_retry_interval_ = interval; }
int openAttempts() const { return open_attempts_; }
std::size_t spillCount() const { return spill_.size(); }
SavedTipValidation prepareValidation(int currentHeight, const std::string& currentBestHash);
void applySavedTipValidation(const SavedTipValidation& validation,
@@ -68,6 +82,9 @@ private:
bool createSchema();
void migrateLegacyJsonIfNeeded();
void close();
bool storeBlockRow(const ExplorerBlockSummary& block); // DB write only
void spillBlock(const ExplorerBlockSummary& block);
bool failOpen(const char* why);
sqlite3* db_ = nullptr;
std::string database_path_;
@@ -78,6 +95,19 @@ private:
bool tip_known_ = false;
int tip_height_ = 0;
std::string tip_hash_;
// Failure latch for ensureOpen(): after a failed open nothing retries until retry_not_before_.
std::chrono::steady_clock::time_point retry_not_before_{};
std::chrono::milliseconds open_retry_interval_{30000};
int open_attempts_ = 0;
int stores_since_prune_ = 0;
// Blocks the DB refused, bounded and UI-thread only. Evicts the entry farthest from the last spilled height
// (the Explorer fetches around the visible window). Cleared with clearBlocks() and when the chain tip regresses.
static constexpr std::size_t kMaxSpillBlocks = 512;
std::map<int, ExplorerBlockSummary> spill_;
int spill_tip_height_ = 0;
std::string spill_tip_hash_;
};
} // namespace ui
tests/test_phase4.cpp
+84
View File
@@ -8590,6 +8590,90 @@ void testExplorerBlockCache()
EXPECT_EQ(reopened.cachedBlockCount(), 0);
}
// Search: substring of hash (case-insensitive) and of height text; non-digit needles skip heights.
{
ExplorerBlockCache searchCache(databasePath.string(), legacyPath.string());
for (int h = 100; h < 103; ++h) {
ExplorerBlockSummary b;
b.height = h;
b.hash = "abcdef" + std::to_string(h);
b.tx_count = 1;
EXPECT_TRUE(searchCache.storeBlock(b));
}
auto byHash = searchCache.searchBlocks("ABCDEF", 10);
EXPECT_EQ(byHash.size(), static_cast<size_t>(3));
EXPECT_EQ(byHash.front().height, 102);
EXPECT_EQ(searchCache.searchBlocks("101", 10).size(), static_cast<size_t>(1));
EXPECT_EQ(searchCache.searchBlocks("abcdef", 2).size(), static_cast<size_t>(2));
EXPECT_EQ(searchCache.searchBlocks("%", 10).size(), static_cast<size_t>(0));
}
// DB down (parent is a regular file): negative-cached open, rows spill to memory and stay searchable.
{
fs::path blocker = dir / "blocker";
{ std::ofstream f(blocker); f << "x"; }
ExplorerBlockCache down((blocker / "sub" / "explorer.sqlite").string(), std::string());
down.setOpenRetryInterval(std::chrono::milliseconds(10000));
for (int i = 0; i < 50; ++i) EXPECT_FALSE(down.ensureOpen());
EXPECT_EQ(down.openAttempts(), 1);
ExplorerBlockSummary b;
b.height = 777;
b.hash = "deadbeef777";
b.tx_count = 2;
EXPECT_FALSE(down.storeBlock(b));
EXPECT_EQ(down.spillCount(), static_cast<size_t>(1));
EXPECT_EQ(down.loadRange(777, 777).size(), static_cast<size_t>(1));
EXPECT_EQ(down.searchBlocks("DEADBE", 10).size(), static_cast<size_t>(1));
EXPECT_EQ(down.searchBlocks("777", 10).size(), static_cast<size_t>(1));
EXPECT_EQ(down.searchBlocks("nomatch", 10).size(), static_cast<size_t>(0));
// Tip regression clears the spill; an advancing tip keeps it.
down.prepareValidation(800, "tip-800");
down.prepareValidation(801, "tip-801");
EXPECT_EQ(down.spillCount(), static_cast<size_t>(1));
down.prepareValidation(790, "tip-790");
EXPECT_EQ(down.spillCount(), static_cast<size_t>(0));
for (int h = 1; h <= 600; ++h) {
ExplorerBlockSummary s;
s.height = h;
s.hash = "h" + std::to_string(h);
down.storeBlock(s);
}
EXPECT_EQ(down.spillCount(), static_cast<size_t>(512));
EXPECT_EQ(down.loadRange(600, 600).size(), static_cast<size_t>(1));
down.clearBlocks();
EXPECT_EQ(down.spillCount(), static_cast<size_t>(0));
// A zero retry interval retries every call.
ExplorerBlockCache eager((blocker / "sub" / "explorer.sqlite").string(), std::string());
eager.setOpenRetryInterval(std::chrono::milliseconds(0));
EXPECT_FALSE(eager.ensureOpen());
EXPECT_FALSE(eager.ensureOpen());
EXPECT_EQ(eager.openAttempts(), 2);
}
// storeBlocks batches into one savepoint; pruneTo keeps the highest rows.
{
ExplorerBlockCache batch((dir / "batch.sqlite").string(), std::string());
std::vector<ExplorerBlockSummary> blocks;
for (int h = 1; h <= 50; ++h) {
ExplorerBlockSummary s;
s.height = h;
s.hash = "b" + std::to_string(h);
blocks.push_back(s);
}
EXPECT_TRUE(batch.storeBlocks(blocks));
EXPECT_EQ(batch.cachedBlockCount(), 50);
EXPECT_TRUE(batch.storeBlocks(blocks));
EXPECT_EQ(batch.cachedBlockCount(), 50);
batch.pruneTo(20);
EXPECT_EQ(batch.cachedBlockCount(), 20);
EXPECT_EQ(batch.loadRange(1, 50).size(), static_cast<size_t>(20));
EXPECT_EQ(batch.loadRange(1, 50).begin()->first, 31);
}
cacheOwner.reset(); // close the database first: Windows can't delete an open file
removeTree(dir);
}