DragonX/ObsidianDragon
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

feat(explorer): indexed block search with a bounded cache

Browse Source 
The explorer block cache searched with a LIKE query that had to escape
wildcards, grew without any limit, and retried a failed open on every
frame, including a two-second busy wait on the UI thread.

- Search now uses an instr() substring test on a lowercased needle, and
  only checks the height text when the query is all digits, since a
  non-numeric needle can never appear inside a decimal height.
- The table is pruned to the highest 20000 blocks, once per open and
  periodically as blocks are stored, so the file stops growing at its
  high-water mark.
- A failed open is negative-cached (30 s by default) instead of retried
  every frame, and the busy timeout drops to 250 ms.
- When the cache database is unavailable or refuses a write, the block
  spills to a small bounded in-memory map that both the list and search
  read, so a spilled block never appears in one but not the other; the
  spill is cleared when the chain tip regresses.
- A batch of blocks is written under one savepoint instead of one commit
  per block.

Tests: case-insensitive hash and height search, the negative-cache and
spill paths with tip-regression eviction, and batch store plus prune.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DanS
2026-10-13 09:24:51 -05:00
co-authored by Claude Opus 4.8
parent c3f981b52e
commit ebbeb846e8
3 changed files with 255 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ui/explorer/explorer_block_cache.cpp
+141 -42
View File
@@ -7,8 +7,11 @@
#include <sqlite3.h>
#include <algorithm>
#include <cctype>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iterator>
#include <utility>
namespace fs = std::filesystem;
@@ -88,37 +91,52 @@ std::string ExplorerBlockCache::defaultLegacyJsonPath()
return (fs::path(util::Platform::getConfigDir()) / "explorer_blocks_cache.json").string();
}
bool ExplorerBlockCache::failOpen(const char* why)
{
retry_not_before_ = std::chrono::steady_clock::now() + open_retry_interval_;
DEBUG_LOGF("Explorer block cache unavailable (%s); retrying in %lld ms\n", why,
static_cast<long long>(open_retry_interval_.count()));
return false;
}
bool ExplorerBlockCache::ensureOpen()
{
if (db_) return true;
// Negative cache: ~6 cache calls per frame used to each redo create_directories + open + PRAGMAs + CREATE
// TABLE (and a 2 s busy wait on the UI thread) when the file was unusable.
if (retry_not_before_ != std::chrono::steady_clock::time_point{} &&
std::chrono::steady_clock::now() < retry_not_before_) return false;
++open_attempts_;
try {
fs::path path(database_path_);
if (!path.parent_path().empty()) fs::create_directories(path.parent_path());
} catch (const std::exception& e) {
DEBUG_LOGF("Failed to create explorer cache directory: %s\n", e.what());
return false;
return failOpen(e.what());
}
sqlite3* openedDb = nullptr;
if (sqlite3_open(database_path_.c_str(), &openedDb) != SQLITE_OK) {
DEBUG_LOGF("Failed to open explorer block cache: %s\n",
openedDb ? sqlite3_errmsg(openedDb) : "unknown error");
const bool failed = failOpen(openedDb ? sqlite3_errmsg(openedDb) : "unknown error");
if (openedDb) sqlite3_close(openedDb);
return false;
return failed;
}
db_ = openedDb;
sqlite3_busy_timeout(db_, 2000);
// UI-thread-only handle: a short wait is enough; a miss falls back to the in-memory spill instead of
// freezing the frame for 2 s.
sqlite3_busy_timeout(db_, 250);
exec("PRAGMA journal_mode=WAL");
exec("PRAGMA synchronous=NORMAL");
if (!createSchema()) {
close();
return false;
return failOpen("schema");
}
retry_not_before_ = {};
migrateLegacyJsonIfNeeded();
pruneTo(kMaxCachedBlocks); // once per open, never per frame
return true;
}
@@ -126,55 +144,67 @@ std::map<int, ExplorerBlockSummary> ExplorerBlockCache::loadRange(int minHeight,
{
std::map<int, ExplorerBlockSummary> blocks;
if (minHeight > maxHeight) std::swap(minHeight, maxHeight);
if (minHeight < 1 || maxHeight < 1 || !ensureOpen()) return blocks;
if (minHeight < 1 || maxHeight < 1) return blocks;
Statement statement(db_,
"SELECT height, hash, tx_count, size, time, difficulty "
"FROM explorer_blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC");
if (!statement.handle) return blocks;
if (ensureOpen()) {
Statement statement(db_,
"SELECT height, hash, tx_count, size, time, difficulty "
"FROM explorer_blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC");
if (statement.handle) {
sqlite3_bind_int(statement.handle, 1, minHeight);
sqlite3_bind_int(statement.handle, 2, maxHeight);
sqlite3_bind_int(statement.handle, 1, minHeight);
sqlite3_bind_int(statement.handle, 2, maxHeight);
while (sqlite3_step(statement.handle) == SQLITE_ROW) {
ExplorerBlockSummary block;
block.height = sqlite3_column_int(statement.handle, 0);
const unsigned char* hashText = sqlite3_column_text(statement.handle, 1);
if (hashText) block.hash = reinterpret_cast<const char*>(hashText);
block.tx_count = sqlite3_column_int(statement.handle, 2);
block.size = sqlite3_column_int(statement.handle, 3);
block.time = static_cast<std::int64_t>(sqlite3_column_int64(statement.handle, 4));
block.difficulty = sqlite3_column_double(statement.handle, 5);
if (block.height > 0 && !block.hash.empty()) {
blocks[block.height] = std::move(block);
while (sqlite3_step(statement.handle) == SQLITE_ROW) {
ExplorerBlockSummary block;
block.height = sqlite3_column_int(statement.handle, 0);
const unsigned char* hashText = sqlite3_column_text(statement.handle, 1);
if (hashText) block.hash = reinterpret_cast<const char*>(hashText);
block.tx_count = sqlite3_column_int(statement.handle, 2);
block.size = sqlite3_column_int(statement.handle, 3);
block.time = static_cast<std::int64_t>(sqlite3_column_int64(statement.handle, 4));
block.difficulty = sqlite3_column_double(statement.handle, 5);
if (block.height > 0 && !block.hash.empty()) {
blocks[block.height] = std::move(block);
}
}
}
}
// Heights the DB could not give (closed, or refused the insert) come from the in-memory spill.
for (auto it = spill_.lower_bound(minHeight); it != spill_.end() && it->first <= maxHeight; ++it)
blocks.emplace(it->first, it->second); // emplace never overwrites a DB row
return blocks;
}
std::vector<ExplorerBlockSummary> ExplorerBlockCache::searchBlocks(const std::string& query, int limit)
{
std::vector<ExplorerBlockSummary> results;
if (query.empty() || limit < 1 || !ensureOpen()) return results;
if (query.empty() || limit < 1) return results;
// Escape LIKE wildcards in the user input so '%' / '_' are matched literally.
std::string escaped;
escaped.reserve(query.size());
for (char c : query) {
if (c == '%' || c == '_' || c == '\\') escaped.push_back('\\');
escaped.push_back(c);
// instr() is a plain substring test (no wildcards to escape). Daemon hashes are lowercase hex, so lowercase
// the needle (LIKE was ASCII case-insensitive). A non-numeric needle can never occur inside a decimal
// height, so only an all-digit query also checks the height text.
std::string needle = query;
for (char& c : needle) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
const bool digitsOnly = std::all_of(needle.begin(), needle.end(),
[](char ch) { return ch >= '0' && ch <= '9'; });
if (!ensureOpen()) { // DB down: search the in-memory spill instead of returning nothing
for (auto it = spill_.rbegin(); it != spill_.rend() && (int)results.size() < limit; ++it)
if (it->second.hash.find(needle) != std::string::npos ||
(digitsOnly && std::to_string(it->first).find(needle) != std::string::npos))
results.push_back(it->second);
return results;
}
std::string pattern = "%" + escaped + "%";
Statement statement(db_,
"SELECT height, hash, tx_count, size, time, difficulty "
"FROM explorer_blocks "
"WHERE CAST(height AS TEXT) LIKE ?1 ESCAPE '\\' OR hash LIKE ?1 ESCAPE '\\' "
"ORDER BY height DESC LIMIT ?2");
Statement statement(db_, digitsOnly
? "SELECT height, hash, tx_count, size, time, difficulty FROM explorer_blocks "
"WHERE instr(CAST(height AS TEXT), ?1) > 0 OR instr(hash, ?1) > 0 ORDER BY height DESC LIMIT ?2"
: "SELECT height, hash, tx_count, size, time, difficulty FROM explorer_blocks "
"WHERE instr(hash, ?1) > 0 ORDER BY height DESC LIMIT ?2");
if (!statement.handle) return results;
sqlite3_bind_text(statement.handle, 1, pattern.c_str(), -1, SQLITE_TRANSIENT);
sqlite3_bind_text(statement.handle, 1, needle.c_str(), -1, SQLITE_TRANSIENT);
sqlite3_bind_int(statement.handle, 2, limit);
while (sqlite3_step(statement.handle) == SQLITE_ROW) {
@@ -189,10 +219,22 @@ std::vector<ExplorerBlockSummary> ExplorerBlockCache::searchBlocks(const std::st
if (block.height > 0 && !block.hash.empty()) results.push_back(std::move(block));
}
// A block the DB refused (transient busy) is held in spill_ and shows in the list via loadRange, so
// keep search consistent: merge any matching spill entry the DB query did not already return.
for (auto it = spill_.rbegin(); it != spill_.rend() && (int)results.size() < limit; ++it) {
if (!(it->second.hash.find(needle) != std::string::npos ||
(digitsOnly && std::to_string(it->first).find(needle) != std::string::npos)))
continue;
if (std::any_of(results.begin(), results.end(),
[&](const ExplorerBlockSummary& b) { return b.height == it->first; }))
continue;
results.push_back(it->second);
}
return results;
}
bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block)
bool ExplorerBlockCache::storeBlockRow(const ExplorerBlockSummary& block)
{
if (block.height < 1 || block.hash.empty() || !ensureOpen()) return false;
@@ -211,6 +253,40 @@ bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block)
return sqlite3_step(statement.handle) == SQLITE_DONE;
}
bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block)
{
if (block.height < 1 || block.hash.empty()) return false;
const bool stored = storeBlockRow(block);
if (stored) {
spill_.erase(block.height);
if (++stores_since_prune_ >= 1000) { stores_since_prune_ = 0; pruneTo(kMaxCachedBlocks); }
} else {
spillBlock(block);
}
return stored;
}
bool ExplorerBlockCache::storeBlocks(const std::vector<ExplorerBlockSummary>& blocks)
{
if (blocks.empty()) return true;
// A savepoint, not BEGIN: nests inside a caller's transaction (same reason as updateTip).
const bool grouped = ensureOpen() && exec("SAVEPOINT explorer_batch");
bool all = true;
for (const auto& b : blocks) all = storeBlock(b) && all;
if (grouped) exec("RELEASE explorer_batch");
return all;
}
void ExplorerBlockCache::spillBlock(const ExplorerBlockSummary& block)
{
spill_[block.height] = block;
while (spill_.size() > kMaxSpillBlocks) {
const int farFirst = std::abs(spill_.begin()->first - block.height);
const int farLast = std::abs(spill_.rbegin()->first - block.height);
spill_.erase(farFirst >= farLast ? spill_.begin() : std::prev(spill_.end()));
}
}
int ExplorerBlockCache::cachedBlockCount()
{
if (!ensureOpen()) return 0;
@@ -221,10 +297,25 @@ int ExplorerBlockCache::cachedBlockCount()
void ExplorerBlockCache::clearBlocks()
{
spill_.clear();
if (!ensureOpen()) return;
exec("DELETE FROM explorer_blocks");
}
void ExplorerBlockCache::pruneTo(int maxRows)
{
if (maxRows < 1 || !db_) return;
// Keep the maxRows highest blocks. The OFFSET row is the (maxRows+1)-th highest; when the table is smaller the
// subquery is NULL and nothing is deleted. Freed pages are reused, so the file stops growing at its high-water
// mark (no VACUUM / auto_vacuum: changing auto_vacuum needs a rebuild of existing databases).
Statement statement(db_,
"DELETE FROM explorer_blocks WHERE height <= "
"(SELECT height FROM explorer_blocks ORDER BY height DESC LIMIT 1 OFFSET ?1)");
if (!statement.handle) return;
sqlite3_bind_int(statement.handle, 1, maxRows);
sqlite3_step(statement.handle);
}
ExplorerBlockCache::SavedTipValidation ExplorerBlockCache::prepareValidation(
int currentHeight, const std::string& currentBestHash)
{
@@ -238,7 +329,15 @@ ExplorerBlockCache::SavedTipValidation ExplorerBlockCache::prepareValidation(
currentHeight == tip_height_ && currentBestHash == tip_hash_) {
return validation;
}
if (!ensureOpen()) return validation;
if (!ensureOpen()) {
// No DB to validate against: keep the spill honest ourselves (tip regressed, or a different block at the same height).
if (!spill_.empty() && !currentBestHash.empty() &&
(currentHeight < spill_tip_height_ ||
(currentHeight == spill_tip_height_ && currentBestHash != spill_tip_hash_)))
spill_.clear();
if (!currentBestHash.empty()) { spill_tip_height_ = currentHeight; spill_tip_hash_ = currentBestHash; }
return validation;
}
int savedHeight = getMetaInt("tip_height", 0);
std::string savedHash = getMetaValue("tip_hash");