feat(util): add CoalescingWriter for background file/snapshot writes
One background thread for blocking write jobs, for the settings.json / wallets.json saves and the tx-history snapshot that later commits move off the UI thread. Jobs are keyed (a file path, "tx-history"): - a submit for a key that is still queued replaces that job and keeps its place in line (latest wins); - keys run first in, first out, one job at a time, so two writes of one file never overlap and an older snapshot never lands after a newer one; - submitAndWait() blocks until that job (or a newer one for the key) has run and returns its result, for writes the caller needs on disk; - flush() waits with a bound; stop() runs everything queued, then joins; - the thread starts on first use and sleeps when idle; with no thread (none could be created, or after stop()) a job runs inline. persistFileAtomically() routes Platform::writeFileAtomically through a writer, or writes synchronously when there is none. Nothing uses it yet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -18,6 +18,7 @@
|
||||
#include "util/fetch_backoff.h"
|
||||
#include "util/logger.h"
|
||||
#include "data/transaction_history_cache.h"
|
||||
#include "util/coalescing_writer.h"
|
||||
#include "data/address_book.h"
|
||||
#include "data/wallet_index.h"
|
||||
#include "util/secure_vault.h"
|
||||
@@ -94,8 +95,11 @@
|
||||
#include <iostream>
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <new>
|
||||
#include <set>
|
||||
#include <stdexcept>
|
||||
@@ -7898,6 +7902,203 @@ void testLiteOfficialServerDetection()
|
||||
EXPECT_FALSE(isOfficialLiteServer(""));
|
||||
}
|
||||
|
||||
// CoalescingWriter (SI-1): the shared background writer behind the settings/wallets.json saves and the tx-history
|
||||
// snapshot. Latest wins per key, distinct keys all run, stop() drains, failures are reported once, a throw is
|
||||
// contained, and a submit after stop() (or before any thread exists) runs inline.
|
||||
void testCoalescingWriter()
|
||||
{
|
||||
using dragonx::util::CoalescingWriter;
|
||||
{ // latest wins: hold the worker in job 1, queue three more under the same key, only the last runs
|
||||
std::mutex m;
|
||||
std::condition_variable cv;
|
||||
bool release = false, started = false;
|
||||
std::vector<int> ran; // written only by the worker; read after flush() (synchronised)
|
||||
// Declared after the locals its jobs capture by reference: if flush() times out, ~CoalescingWriter joins
|
||||
// the still-running job before those locals are destroyed.
|
||||
CoalescingWriter w;
|
||||
w.submit("k", [&] {
|
||||
{ std::lock_guard<std::mutex> l(m); started = true; }
|
||||
cv.notify_all();
|
||||
std::unique_lock<std::mutex> l(m);
|
||||
cv.wait(l, [&] { return release; });
|
||||
ran.push_back(1);
|
||||
return true;
|
||||
});
|
||||
{ std::unique_lock<std::mutex> l(m); cv.wait(l, [&] { return started; }); }
|
||||
w.submit("k", [&] { ran.push_back(2); return true; });
|
||||
w.submit("k", [&] { ran.push_back(3); return true; });
|
||||
w.submit("k", [&] { ran.push_back(4); return true; });
|
||||
{ std::lock_guard<std::mutex> l(m); release = true; }
|
||||
cv.notify_all();
|
||||
EXPECT_TRUE(w.flush(std::chrono::seconds(5)));
|
||||
EXPECT_EQ(ran.size(), static_cast<size_t>(2));
|
||||
EXPECT_EQ(ran.back(), 4);
|
||||
EXPECT_EQ(w.supersededCount(), static_cast<size_t>(2));
|
||||
}
|
||||
{ // distinct keys all run; stop() drains; failure reported once; a throw is contained
|
||||
CoalescingWriter w;
|
||||
std::atomic<int> n{0};
|
||||
w.submit("a", [&] { ++n; return true; });
|
||||
w.submit("b", [&] { ++n; return false; });
|
||||
w.submit("c", [&]() -> bool { ++n; throw std::runtime_error("boom"); });
|
||||
w.stop();
|
||||
EXPECT_EQ(n.load(), 3);
|
||||
EXPECT_FALSE(w.takeFailure("a"));
|
||||
EXPECT_TRUE(w.takeFailure("b"));
|
||||
EXPECT_FALSE(w.takeFailure("b")); // cleared
|
||||
EXPECT_TRUE(w.takeFailure("c"));
|
||||
bool ranInline = false; // after stop() a late submit runs inline, not dropped
|
||||
w.submit("late", [&] { ranInline = true; return true; });
|
||||
EXPECT_TRUE(ranInline);
|
||||
w.stop(); // idempotent
|
||||
}
|
||||
{ // stop() before the first submit: no thread is ever started (nothing left unjoined)
|
||||
CoalescingWriter w;
|
||||
w.stop();
|
||||
bool ran = false;
|
||||
w.submit("x", [&] { ran = true; return true; });
|
||||
EXPECT_TRUE(ran);
|
||||
EXPECT_EQ(w.completedCount(), static_cast<size_t>(1));
|
||||
}
|
||||
}
|
||||
|
||||
// C269: CoalescingWriter ordering, as the settings.json / wallets.json writes use it. Deterministic (jobs are
|
||||
// gated, no sleeps): latest-wins coalescing, FIFO across keys, submitAndWait ordering + result, bounded flush,
|
||||
// drain-on-destroy, and real files through persistFileAtomically.
|
||||
void testCoalescingWriterOrdering()
|
||||
{
|
||||
using dragonx::util::CoalescingWriter;
|
||||
using ms = std::chrono::milliseconds;
|
||||
|
||||
struct Rec {
|
||||
std::mutex m;
|
||||
std::condition_variable cv;
|
||||
std::vector<std::pair<std::string, std::string>> calls;
|
||||
bool hold = false; // jobs block until released
|
||||
int entered = 0; // jobs that have started
|
||||
};
|
||||
auto r = std::make_shared<Rec>();
|
||||
// A job for key p that "writes" content c: it blocks while held, then records (p, c) and succeeds, or with
|
||||
// ok=false fails without recording.
|
||||
auto rec = [r](const std::string& p, const std::string& c, bool ok = true) {
|
||||
return [r, p, c, ok]() {
|
||||
std::unique_lock<std::mutex> lk(r->m);
|
||||
++r->entered;
|
||||
r->cv.notify_all();
|
||||
r->cv.wait(lk, [&] { return !r->hold; });
|
||||
if (!ok) return false;
|
||||
r->calls.emplace_back(p, c);
|
||||
return true;
|
||||
};
|
||||
};
|
||||
auto reset = [r](bool hold) {
|
||||
std::lock_guard<std::mutex> lk(r->m);
|
||||
r->calls.clear(); r->hold = hold; r->entered = 0;
|
||||
};
|
||||
auto release = [r] {
|
||||
{ std::lock_guard<std::mutex> lk(r->m); r->hold = false; }
|
||||
r->cv.notify_all();
|
||||
};
|
||||
auto waitEntered = [r](int n) {
|
||||
std::unique_lock<std::mutex> lk(r->m);
|
||||
return r->cv.wait_for(lk, ms(3000), [&] { return r->entered >= n; });
|
||||
};
|
||||
using Call = std::pair<std::string, std::string>;
|
||||
|
||||
{ // a burst while one write is in flight collapses into ONE follow-up write of the latest content
|
||||
reset(true);
|
||||
CoalescingWriter w;
|
||||
w.submit("a.json", rec("a.json", "v0"));
|
||||
EXPECT_TRUE(waitEntered(1));
|
||||
for (int i = 1; i < 50; ++i) w.submit("a.json", rec("a.json", "v" + std::to_string(i)));
|
||||
EXPECT_EQ(w.pendingCount(), (size_t)2); // v0 in flight + one coalesced slot
|
||||
release();
|
||||
EXPECT_TRUE(w.flush(ms(3000)));
|
||||
std::lock_guard<std::mutex> lk(r->m);
|
||||
EXPECT_EQ(r->calls.size(), (size_t)2);
|
||||
EXPECT_TRUE(r->calls.size() == 2 && r->calls[0] == Call("a.json", "v0")
|
||||
&& r->calls[1] == Call("a.json", "v49"));
|
||||
}
|
||||
{ // FIFO by first enqueue; a replaced job keeps its place; a re-submit of the in-flight key goes last
|
||||
reset(true);
|
||||
CoalescingWriter w;
|
||||
w.submit("p1", rec("p1", "1a"));
|
||||
EXPECT_TRUE(waitEntered(1));
|
||||
w.submit("p2", rec("p2", "2a"));
|
||||
w.submit("p3", rec("p3", "3a"));
|
||||
w.submit("p2", rec("p2", "2b"));
|
||||
w.submit("p1", rec("p1", "1b"));
|
||||
release();
|
||||
EXPECT_TRUE(w.flush(ms(3000)));
|
||||
std::lock_guard<std::mutex> lk(r->m);
|
||||
const std::vector<Call> want = {{"p1", "1a"}, {"p2", "2b"}, {"p3", "3a"}, {"p1", "1b"}};
|
||||
EXPECT_TRUE(r->calls == want);
|
||||
}
|
||||
{ // submitAndWait: done when it returns, never overtaken by the older queued job
|
||||
reset(false);
|
||||
CoalescingWriter w;
|
||||
w.submit("b.json", rec("b.json", "old"));
|
||||
EXPECT_TRUE(w.submitAndWait("b.json", rec("b.json", "new")));
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(r->m);
|
||||
EXPECT_TRUE(!r->calls.empty() && r->calls.back() == Call("b.json", "new"));
|
||||
}
|
||||
EXPECT_EQ(w.pendingCount(), (size_t)0);
|
||||
}
|
||||
{ // submitAndWait reports a failed write; a failing key doesn't block others
|
||||
reset(false);
|
||||
CoalescingWriter w;
|
||||
EXPECT_FALSE(w.submitAndWait("c.json", rec("c.json", "x", /*ok=*/false)));
|
||||
EXPECT_EQ(w.failureCount(), (uint64_t)1);
|
||||
w.submit("d.json", rec("d.json", "y"));
|
||||
EXPECT_TRUE(w.flush(ms(3000)));
|
||||
std::lock_guard<std::mutex> lk(r->m);
|
||||
EXPECT_TRUE(!r->calls.empty() && r->calls.back() == Call("d.json", "y"));
|
||||
}
|
||||
{ // flush is bounded when a write is wedged, and completes once it unwedges
|
||||
reset(true);
|
||||
CoalescingWriter w;
|
||||
w.submit("e.json", rec("e.json", "z"));
|
||||
EXPECT_TRUE(waitEntered(1));
|
||||
EXPECT_FALSE(w.flush(ms(50)));
|
||||
release();
|
||||
EXPECT_TRUE(w.flush(ms(3000)));
|
||||
}
|
||||
{ // the destructor runs what is still queued before joining
|
||||
reset(true);
|
||||
{
|
||||
CoalescingWriter w;
|
||||
w.submit("f.json", rec("f.json", "1"));
|
||||
EXPECT_TRUE(waitEntered(1));
|
||||
w.submit("g.json", rec("g.json", "2"));
|
||||
release();
|
||||
}
|
||||
std::lock_guard<std::mutex> lk(r->m);
|
||||
EXPECT_TRUE(r->calls.size() == 2 && r->calls[1] == Call("g.json", "2"));
|
||||
}
|
||||
|
||||
// Real files through persistFileAtomically: atomic, latest wins, no stray .tmp; with no writer it is sync.
|
||||
{
|
||||
const fs::path dir = makeTempDir();
|
||||
const std::string target = (dir / "n" / "s.json").string();
|
||||
{
|
||||
CoalescingWriter w;
|
||||
EXPECT_TRUE(dragonx::util::persistFileAtomically(&w, false, target, "{\"k\":1}", true));
|
||||
EXPECT_TRUE(dragonx::util::persistFileAtomically(&w, false, target, "{\"k\":2}", true));
|
||||
EXPECT_TRUE(w.flush(ms(5000)));
|
||||
}
|
||||
std::ifstream in(target);
|
||||
std::string body((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
|
||||
EXPECT_EQ(body, std::string("{\"k\":2}"));
|
||||
EXPECT_FALSE(fs::exists(target + ".tmp"));
|
||||
const std::string sync = (dir / "sync.json").string();
|
||||
EXPECT_TRUE(dragonx::util::persistFileAtomically(nullptr, false, sync, "x", true));
|
||||
EXPECT_TRUE(fs::exists(sync));
|
||||
in.close();
|
||||
removeTree(dir);
|
||||
}
|
||||
}
|
||||
|
||||
// H7: atomic + durable file writes (settings.json / addressbook.json persistence).
|
||||
void testAtomicFileWrite()
|
||||
{
|
||||
@@ -12404,6 +12605,8 @@ int main()
|
||||
testEffectivePools();
|
||||
testFormatFeePercent();
|
||||
testPoolWeightedSelection();
|
||||
testCoalescingWriter();
|
||||
testCoalescingWriterOrdering();
|
||||
testAtomicFileWrite();
|
||||
testHushChatCrypto();
|
||||
testHushChatReceivePath();
|
||||
|
||||
Reference in New Issue
Block a user