diff --git a/src/ui/explorer/explorer_block_cache.cpp b/src/ui/explorer/explorer_block_cache.cpp index 41d87c7..678e41d 100644 --- a/src/ui/explorer/explorer_block_cache.cpp +++ b/src/ui/explorer/explorer_block_cache.cpp @@ -7,8 +7,11 @@ #include #include +#include +#include #include #include +#include #include namespace fs = std::filesystem; @@ -88,37 +91,52 @@ std::string ExplorerBlockCache::defaultLegacyJsonPath() return (fs::path(util::Platform::getConfigDir()) / "explorer_blocks_cache.json").string(); } +bool ExplorerBlockCache::failOpen(const char* why) +{ + retry_not_before_ = std::chrono::steady_clock::now() + open_retry_interval_; + DEBUG_LOGF("Explorer block cache unavailable (%s); retrying in %lld ms\n", why, + static_cast(open_retry_interval_.count())); + return false; +} + bool ExplorerBlockCache::ensureOpen() { if (db_) return true; + // Negative cache: ~6 cache calls per frame used to each redo create_directories + open + PRAGMAs + CREATE + // TABLE (and a 2 s busy wait on the UI thread) when the file was unusable. + if (retry_not_before_ != std::chrono::steady_clock::time_point{} && + std::chrono::steady_clock::now() < retry_not_before_) return false; + ++open_attempts_; try { fs::path path(database_path_); if (!path.parent_path().empty()) fs::create_directories(path.parent_path()); } catch (const std::exception& e) { - DEBUG_LOGF("Failed to create explorer cache directory: %s\n", e.what()); - return false; + return failOpen(e.what()); } sqlite3* openedDb = nullptr; if (sqlite3_open(database_path_.c_str(), &openedDb) != SQLITE_OK) { - DEBUG_LOGF("Failed to open explorer block cache: %s\n", - openedDb ? sqlite3_errmsg(openedDb) : "unknown error"); + const bool failed = failOpen(openedDb ? sqlite3_errmsg(openedDb) : "unknown error"); if (openedDb) sqlite3_close(openedDb); - return false; + return failed; } db_ = openedDb; - sqlite3_busy_timeout(db_, 2000); + // UI-thread-only handle: a short wait is enough; a miss falls back to the in-memory spill instead of + // freezing the frame for 2 s. + sqlite3_busy_timeout(db_, 250); exec("PRAGMA journal_mode=WAL"); exec("PRAGMA synchronous=NORMAL"); if (!createSchema()) { close(); - return false; + return failOpen("schema"); } + retry_not_before_ = {}; migrateLegacyJsonIfNeeded(); + pruneTo(kMaxCachedBlocks); // once per open, never per frame return true; } @@ -126,55 +144,67 @@ std::map ExplorerBlockCache::loadRange(int minHeight, { std::map blocks; if (minHeight > maxHeight) std::swap(minHeight, maxHeight); - if (minHeight < 1 || maxHeight < 1 || !ensureOpen()) return blocks; + if (minHeight < 1 || maxHeight < 1) return blocks; - Statement statement(db_, - "SELECT height, hash, tx_count, size, time, difficulty " - "FROM explorer_blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC"); - if (!statement.handle) return blocks; + if (ensureOpen()) { + Statement statement(db_, + "SELECT height, hash, tx_count, size, time, difficulty " + "FROM explorer_blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC"); + if (statement.handle) { + sqlite3_bind_int(statement.handle, 1, minHeight); + sqlite3_bind_int(statement.handle, 2, maxHeight); - sqlite3_bind_int(statement.handle, 1, minHeight); - sqlite3_bind_int(statement.handle, 2, maxHeight); - - while (sqlite3_step(statement.handle) == SQLITE_ROW) { - ExplorerBlockSummary block; - block.height = sqlite3_column_int(statement.handle, 0); - const unsigned char* hashText = sqlite3_column_text(statement.handle, 1); - if (hashText) block.hash = reinterpret_cast(hashText); - block.tx_count = sqlite3_column_int(statement.handle, 2); - block.size = sqlite3_column_int(statement.handle, 3); - block.time = static_cast(sqlite3_column_int64(statement.handle, 4)); - block.difficulty = sqlite3_column_double(statement.handle, 5); - if (block.height > 0 && !block.hash.empty()) { - blocks[block.height] = std::move(block); + while (sqlite3_step(statement.handle) == SQLITE_ROW) { + ExplorerBlockSummary block; + block.height = sqlite3_column_int(statement.handle, 0); + const unsigned char* hashText = sqlite3_column_text(statement.handle, 1); + if (hashText) block.hash = reinterpret_cast(hashText); + block.tx_count = sqlite3_column_int(statement.handle, 2); + block.size = sqlite3_column_int(statement.handle, 3); + block.time = static_cast(sqlite3_column_int64(statement.handle, 4)); + block.difficulty = sqlite3_column_double(statement.handle, 5); + if (block.height > 0 && !block.hash.empty()) { + blocks[block.height] = std::move(block); + } + } } } + // Heights the DB could not give (closed, or refused the insert) come from the in-memory spill. + for (auto it = spill_.lower_bound(minHeight); it != spill_.end() && it->first <= maxHeight; ++it) + blocks.emplace(it->first, it->second); // emplace never overwrites a DB row return blocks; } std::vector ExplorerBlockCache::searchBlocks(const std::string& query, int limit) { std::vector results; - if (query.empty() || limit < 1 || !ensureOpen()) return results; + if (query.empty() || limit < 1) return results; - // Escape LIKE wildcards in the user input so '%' / '_' are matched literally. - std::string escaped; - escaped.reserve(query.size()); - for (char c : query) { - if (c == '%' || c == '_' || c == '\\') escaped.push_back('\\'); - escaped.push_back(c); + // instr() is a plain substring test (no wildcards to escape). Daemon hashes are lowercase hex, so lowercase + // the needle (LIKE was ASCII case-insensitive). A non-numeric needle can never occur inside a decimal + // height, so only an all-digit query also checks the height text. + std::string needle = query; + for (char& c : needle) c = static_cast(std::tolower(static_cast(c))); + const bool digitsOnly = std::all_of(needle.begin(), needle.end(), + [](char ch) { return ch >= '0' && ch <= '9'; }); + + if (!ensureOpen()) { // DB down: search the in-memory spill instead of returning nothing + for (auto it = spill_.rbegin(); it != spill_.rend() && (int)results.size() < limit; ++it) + if (it->second.hash.find(needle) != std::string::npos || + (digitsOnly && std::to_string(it->first).find(needle) != std::string::npos)) + results.push_back(it->second); + return results; } - std::string pattern = "%" + escaped + "%"; - Statement statement(db_, - "SELECT height, hash, tx_count, size, time, difficulty " - "FROM explorer_blocks " - "WHERE CAST(height AS TEXT) LIKE ?1 ESCAPE '\\' OR hash LIKE ?1 ESCAPE '\\' " - "ORDER BY height DESC LIMIT ?2"); + Statement statement(db_, digitsOnly + ? "SELECT height, hash, tx_count, size, time, difficulty FROM explorer_blocks " + "WHERE instr(CAST(height AS TEXT), ?1) > 0 OR instr(hash, ?1) > 0 ORDER BY height DESC LIMIT ?2" + : "SELECT height, hash, tx_count, size, time, difficulty FROM explorer_blocks " + "WHERE instr(hash, ?1) > 0 ORDER BY height DESC LIMIT ?2"); if (!statement.handle) return results; - sqlite3_bind_text(statement.handle, 1, pattern.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(statement.handle, 1, needle.c_str(), -1, SQLITE_TRANSIENT); sqlite3_bind_int(statement.handle, 2, limit); while (sqlite3_step(statement.handle) == SQLITE_ROW) { @@ -189,10 +219,22 @@ std::vector ExplorerBlockCache::searchBlocks(const std::st if (block.height > 0 && !block.hash.empty()) results.push_back(std::move(block)); } + // A block the DB refused (transient busy) is held in spill_ and shows in the list via loadRange, so + // keep search consistent: merge any matching spill entry the DB query did not already return. + for (auto it = spill_.rbegin(); it != spill_.rend() && (int)results.size() < limit; ++it) { + if (!(it->second.hash.find(needle) != std::string::npos || + (digitsOnly && std::to_string(it->first).find(needle) != std::string::npos))) + continue; + if (std::any_of(results.begin(), results.end(), + [&](const ExplorerBlockSummary& b) { return b.height == it->first; })) + continue; + results.push_back(it->second); + } + return results; } -bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block) +bool ExplorerBlockCache::storeBlockRow(const ExplorerBlockSummary& block) { if (block.height < 1 || block.hash.empty() || !ensureOpen()) return false; @@ -211,6 +253,40 @@ bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block) return sqlite3_step(statement.handle) == SQLITE_DONE; } +bool ExplorerBlockCache::storeBlock(const ExplorerBlockSummary& block) +{ + if (block.height < 1 || block.hash.empty()) return false; + const bool stored = storeBlockRow(block); + if (stored) { + spill_.erase(block.height); + if (++stores_since_prune_ >= 1000) { stores_since_prune_ = 0; pruneTo(kMaxCachedBlocks); } + } else { + spillBlock(block); + } + return stored; +} + +bool ExplorerBlockCache::storeBlocks(const std::vector& blocks) +{ + if (blocks.empty()) return true; + // A savepoint, not BEGIN: nests inside a caller's transaction (same reason as updateTip). + const bool grouped = ensureOpen() && exec("SAVEPOINT explorer_batch"); + bool all = true; + for (const auto& b : blocks) all = storeBlock(b) && all; + if (grouped) exec("RELEASE explorer_batch"); + return all; +} + +void ExplorerBlockCache::spillBlock(const ExplorerBlockSummary& block) +{ + spill_[block.height] = block; + while (spill_.size() > kMaxSpillBlocks) { + const int farFirst = std::abs(spill_.begin()->first - block.height); + const int farLast = std::abs(spill_.rbegin()->first - block.height); + spill_.erase(farFirst >= farLast ? spill_.begin() : std::prev(spill_.end())); + } +} + int ExplorerBlockCache::cachedBlockCount() { if (!ensureOpen()) return 0; @@ -221,10 +297,25 @@ int ExplorerBlockCache::cachedBlockCount() void ExplorerBlockCache::clearBlocks() { + spill_.clear(); if (!ensureOpen()) return; exec("DELETE FROM explorer_blocks"); } +void ExplorerBlockCache::pruneTo(int maxRows) +{ + if (maxRows < 1 || !db_) return; + // Keep the maxRows highest blocks. The OFFSET row is the (maxRows+1)-th highest; when the table is smaller the + // subquery is NULL and nothing is deleted. Freed pages are reused, so the file stops growing at its high-water + // mark (no VACUUM / auto_vacuum: changing auto_vacuum needs a rebuild of existing databases). + Statement statement(db_, + "DELETE FROM explorer_blocks WHERE height <= " + "(SELECT height FROM explorer_blocks ORDER BY height DESC LIMIT 1 OFFSET ?1)"); + if (!statement.handle) return; + sqlite3_bind_int(statement.handle, 1, maxRows); + sqlite3_step(statement.handle); +} + ExplorerBlockCache::SavedTipValidation ExplorerBlockCache::prepareValidation( int currentHeight, const std::string& currentBestHash) { @@ -238,7 +329,15 @@ ExplorerBlockCache::SavedTipValidation ExplorerBlockCache::prepareValidation( currentHeight == tip_height_ && currentBestHash == tip_hash_) { return validation; } - if (!ensureOpen()) return validation; + if (!ensureOpen()) { + // No DB to validate against: keep the spill honest ourselves (tip regressed, or a different block at the same height). + if (!spill_.empty() && !currentBestHash.empty() && + (currentHeight < spill_tip_height_ || + (currentHeight == spill_tip_height_ && currentBestHash != spill_tip_hash_))) + spill_.clear(); + if (!currentBestHash.empty()) { spill_tip_height_ = currentHeight; spill_tip_hash_ = currentBestHash; } + return validation; + } int savedHeight = getMetaInt("tip_height", 0); std::string savedHash = getMetaValue("tip_hash"); diff --git a/src/ui/explorer/explorer_block_cache.h b/src/ui/explorer/explorer_block_cache.h index 9e3334b..56752fa 100644 --- a/src/ui/explorer/explorer_block_cache.h +++ b/src/ui/explorer/explorer_block_cache.h @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include #include @@ -45,9 +47,21 @@ public: // Fuzzy search over cached blocks: matches when the query is a substring of the height (as text) // or the block hash (case-insensitive). Returns newest-first, capped at `limit`. std::vector searchBlocks(const std::string& query, int limit); + // Always returns the DB result; a block the DB refuses (unopenable file, failed insert) is kept in a small + // in-memory spill so the Explorer still shows it instead of refetching it forever. bool storeBlock(const ExplorerBlockSummary& block); + // storeBlock for a batch inside one SAVEPOINT (one WAL commit instead of one per block). True if every block hit the DB. + bool storeBlocks(const std::vector& blocks); int cachedBlockCount(); void clearBlocks(); + // Keep only the `maxRows` highest blocks (data-safe: the table is a re-fetchable RPC cache). + void pruneTo(int maxRows); + static constexpr int kMaxCachedBlocks = 20000; + + // A failed open is not retried until this interval has passed (default 30 s). openAttempts() counts real attempts. + void setOpenRetryInterval(std::chrono::milliseconds interval) { open_retry_interval_ = interval; } + int openAttempts() const { return open_attempts_; } + std::size_t spillCount() const { return spill_.size(); } SavedTipValidation prepareValidation(int currentHeight, const std::string& currentBestHash); void applySavedTipValidation(const SavedTipValidation& validation, @@ -68,6 +82,9 @@ private: bool createSchema(); void migrateLegacyJsonIfNeeded(); void close(); + bool storeBlockRow(const ExplorerBlockSummary& block); // DB write only + void spillBlock(const ExplorerBlockSummary& block); + bool failOpen(const char* why); sqlite3* db_ = nullptr; std::string database_path_; @@ -78,6 +95,19 @@ private: bool tip_known_ = false; int tip_height_ = 0; std::string tip_hash_; + + // Failure latch for ensureOpen(): after a failed open nothing retries until retry_not_before_. + std::chrono::steady_clock::time_point retry_not_before_{}; + std::chrono::milliseconds open_retry_interval_{30000}; + int open_attempts_ = 0; + int stores_since_prune_ = 0; + + // Blocks the DB refused, bounded and UI-thread only. Evicts the entry farthest from the last spilled height + // (the Explorer fetches around the visible window). Cleared with clearBlocks() and when the chain tip regresses. + static constexpr std::size_t kMaxSpillBlocks = 512; + std::map spill_; + int spill_tip_height_ = 0; + std::string spill_tip_hash_; }; } // namespace ui diff --git a/tests/test_phase4.cpp b/tests/test_phase4.cpp index d408ced..979c8ee 100644 --- a/tests/test_phase4.cpp +++ b/tests/test_phase4.cpp @@ -8590,6 +8590,90 @@ void testExplorerBlockCache() EXPECT_EQ(reopened.cachedBlockCount(), 0); } + // Search: substring of hash (case-insensitive) and of height text; non-digit needles skip heights. + { + ExplorerBlockCache searchCache(databasePath.string(), legacyPath.string()); + for (int h = 100; h < 103; ++h) { + ExplorerBlockSummary b; + b.height = h; + b.hash = "abcdef" + std::to_string(h); + b.tx_count = 1; + EXPECT_TRUE(searchCache.storeBlock(b)); + } + auto byHash = searchCache.searchBlocks("ABCDEF", 10); + EXPECT_EQ(byHash.size(), static_cast(3)); + EXPECT_EQ(byHash.front().height, 102); + EXPECT_EQ(searchCache.searchBlocks("101", 10).size(), static_cast(1)); + EXPECT_EQ(searchCache.searchBlocks("abcdef", 2).size(), static_cast(2)); + EXPECT_EQ(searchCache.searchBlocks("%", 10).size(), static_cast(0)); + } + + // DB down (parent is a regular file): negative-cached open, rows spill to memory and stay searchable. + { + fs::path blocker = dir / "blocker"; + { std::ofstream f(blocker); f << "x"; } + ExplorerBlockCache down((blocker / "sub" / "explorer.sqlite").string(), std::string()); + down.setOpenRetryInterval(std::chrono::milliseconds(10000)); + for (int i = 0; i < 50; ++i) EXPECT_FALSE(down.ensureOpen()); + EXPECT_EQ(down.openAttempts(), 1); + + ExplorerBlockSummary b; + b.height = 777; + b.hash = "deadbeef777"; + b.tx_count = 2; + EXPECT_FALSE(down.storeBlock(b)); + EXPECT_EQ(down.spillCount(), static_cast(1)); + EXPECT_EQ(down.loadRange(777, 777).size(), static_cast(1)); + EXPECT_EQ(down.searchBlocks("DEADBE", 10).size(), static_cast(1)); + EXPECT_EQ(down.searchBlocks("777", 10).size(), static_cast(1)); + EXPECT_EQ(down.searchBlocks("nomatch", 10).size(), static_cast(0)); + + // Tip regression clears the spill; an advancing tip keeps it. + down.prepareValidation(800, "tip-800"); + down.prepareValidation(801, "tip-801"); + EXPECT_EQ(down.spillCount(), static_cast(1)); + down.prepareValidation(790, "tip-790"); + EXPECT_EQ(down.spillCount(), static_cast(0)); + + for (int h = 1; h <= 600; ++h) { + ExplorerBlockSummary s; + s.height = h; + s.hash = "h" + std::to_string(h); + down.storeBlock(s); + } + EXPECT_EQ(down.spillCount(), static_cast(512)); + EXPECT_EQ(down.loadRange(600, 600).size(), static_cast(1)); + + down.clearBlocks(); + EXPECT_EQ(down.spillCount(), static_cast(0)); + // A zero retry interval retries every call. + ExplorerBlockCache eager((blocker / "sub" / "explorer.sqlite").string(), std::string()); + eager.setOpenRetryInterval(std::chrono::milliseconds(0)); + EXPECT_FALSE(eager.ensureOpen()); + EXPECT_FALSE(eager.ensureOpen()); + EXPECT_EQ(eager.openAttempts(), 2); + } + + // storeBlocks batches into one savepoint; pruneTo keeps the highest rows. + { + ExplorerBlockCache batch((dir / "batch.sqlite").string(), std::string()); + std::vector blocks; + for (int h = 1; h <= 50; ++h) { + ExplorerBlockSummary s; + s.height = h; + s.hash = "b" + std::to_string(h); + blocks.push_back(s); + } + EXPECT_TRUE(batch.storeBlocks(blocks)); + EXPECT_EQ(batch.cachedBlockCount(), 50); + EXPECT_TRUE(batch.storeBlocks(blocks)); + EXPECT_EQ(batch.cachedBlockCount(), 50); + batch.pruneTo(20); + EXPECT_EQ(batch.cachedBlockCount(), 20); + EXPECT_EQ(batch.loadRange(1, 50).size(), static_cast(20)); + EXPECT_EQ(batch.loadRange(1, 50).begin()->first, 31); + } + cacheOwner.reset(); // close the database first: Windows can't delete an open file removeTree(dir); }