feat(chat): larger messages — multi-output chunking in one transaction (tier 2)

A message whose ciphertext exceeds one 512-byte memo is now split across up to
kHushChatMaxMessageParts (16) payload outputs in a SINGLE transaction, lifting
the ~239-byte cap to a few KB while staying atomic, instant (0-conf), and
one-fee. The crypto is untouched (one secretstream push/pull); chunking is a
transport concern:
- Header gains an optional part count `n` (emitted only when >1, so a 1-part
  message stays byte-identical on the wire); bounded in the parser.
- Each chunk is a 2-hex part-index prefix + its ciphertext slice, so the
  receiver reorders them after the daemon's output shuffle.
- OutgoingChatMemos carries a payload vector; chatSendOutputs emits header + N
  payloads (both broadcast call sites already iterate the result).
- The grouper gains a multi-part pre-scan: a lone Message header with n>1 →
  collect the N indexed chunks, sort, strip, concat → one pair; anything else
  falls through to the classic single-pair path (older clients mis-pair and
  drop — graceful). A body beyond the cap is refused as TooLong (tier 3 later).
Round-trip (shuffled outputs), 1-part byte-identical, and cap all unit-tested.

Larger-messages tier 2; tier 3 (multi-tx) + the denser raw-memo encoding remain.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-10-24 11:16:22 -05:00
co-authored by Claude Opus 4.8
parent 19e9b62632
commit 6b93a8ab60
5 changed files with 207 additions and 31 deletions
+47 -16
View File
@@ -18,7 +18,8 @@ std::string buildHeaderMemo(const std::string& replyZaddr,
const std::string& streamHeaderHex,
const std::string& publicKeyHex,
std::int64_t sentAt,
const std::string& op = "")
const std::string& op = "",
int parts = 1)
{
nlohmann::json header;
header["h"] = 1; // header number (>= 1)
@@ -29,21 +30,34 @@ std::string buildHeaderMemo(const std::string& replyZaddr,
header["e"] = streamHeaderHex; // 48-hex secretstream header (Memo) / "" (Cont, Ctrl)
header["p"] = publicKeyHex; // my 64-hex crypto_kx public key
if (!op.empty()) header["op"] = op; // Control only; alphabetically sorts to cid,e,h,op,p,t,v,z
if (parts > 1) header["n"] = parts; // multi-part Message only; omitted (==1) keeps 1-part byte-identical
if (sentAt > 0) header["ts"] = sentAt; // optional sender compose time (Unix s) — receiver shows this
return header.dump();
}
// The N 2-hex index prefix for a multi-part ciphertext chunk (index 0..kHushChatMaxMessageParts-1).
std::string partIndexPrefix(std::size_t i)
{
static const char* kHex = "0123456789abcdef";
std::string p;
p += kHex[(i >> 4) & 0xF];
p += kHex[i & 0xF];
return p;
}
bool present(const std::string& value) { return !value.empty(); }
} // namespace
std::array<ChatSendOutput, 2> chatSendOutputs(const OutgoingChatMemos& memos, bool utf8Prefix)
std::vector<ChatSendOutput> chatSendOutputs(const OutgoingChatMemos& memos, bool utf8Prefix)
{
const std::string prefix = utf8Prefix ? "utf8:" : "";
return {{
{ memos.recipientZaddr, prefix + memos.headerMemo }, // header — the lower memo position
{ memos.recipientZaddr, prefix + memos.payloadMemo },
}};
std::vector<ChatSendOutput> outs;
outs.reserve(1 + memos.payloadMemos.size());
outs.push_back({ memos.recipientZaddr, prefix + memos.headerMemo }); // header (position doesn't matter;
for (const auto& p : memos.payloadMemos) // the receiver reassembles by index)
outs.push_back({ memos.recipientZaddr, prefix + p });
return outs;
}
ChatComposeStatus buildOutgoingMessage(const ChatKeyPair& mine,
@@ -69,15 +83,32 @@ ChatComposeStatus buildOutgoingMessage(const ChatKeyPair& mine,
return ChatComposeStatus::EncryptFailed;
}
// Chunk the ciphertext hex across payload memos. A single memo → the classic 1-part message, byte-
// identical on the wire (no "n", no index prefix). Longer → N index-prefixed chunks (tier 2), capped
// at kHushChatMaxMessageParts; beyond that it is refused as TooLong (tier 3 would lift the ceiling).
std::vector<std::string> payloads;
int parts = 1;
if (ciphertextHex.size() <= kHushChatMemoByteLimit) {
payloads.push_back(ciphertextHex);
} else {
const std::size_t chunkHex = (kHushChatMemoByteLimit - kHushChatPartIndexHexLen) & ~std::size_t(1);
const std::size_t n = (ciphertextHex.size() + chunkHex - 1) / chunkHex;
if (static_cast<int>(n) > kHushChatMaxMessageParts) return ChatComposeStatus::TooLong;
parts = static_cast<int>(n);
payloads.reserve(n);
for (std::size_t i = 0; i < n; ++i) {
payloads.push_back(partIndexPrefix(i) + ciphertextHex.substr(i * chunkHex, chunkHex));
}
}
OutgoingChatMemos memos;
memos.recipientZaddr = peerZaddr;
memos.headerMemo = buildHeaderMemo(myReplyZaddr, conversationId, "Memo", streamHeaderHex, myPublicKeyHex,
static_cast<std::int64_t>(std::time(nullptr)));
memos.payloadMemo = ciphertextHex;
if (memos.headerMemo.size() > kHushChatMemoByteLimit ||
memos.payloadMemo.size() > kHushChatMemoByteLimit) {
return ChatComposeStatus::TooLong;
}
static_cast<std::int64_t>(std::time(nullptr)), "", parts);
memos.payloadMemos = std::move(payloads);
if (memos.headerMemo.size() > kHushChatMemoByteLimit) return ChatComposeStatus::TooLong;
for (const auto& p : memos.payloadMemos)
if (p.size() > kHushChatMemoByteLimit) return ChatComposeStatus::TooLong;
out = std::move(memos);
return ChatComposeStatus::Ok;
}
@@ -102,9 +133,9 @@ ChatComposeStatus buildOutgoingContactRequest(const std::string& myPublicKeyHex,
memos.recipientZaddr = peerZaddr;
memos.headerMemo = buildHeaderMemo(myReplyZaddr, conversationId, "Cont", "", myPublicKeyHex,
static_cast<std::int64_t>(std::time(nullptr)));
memos.payloadMemo = requestText;
memos.payloadMemos = { requestText };
if (memos.headerMemo.size() > kHushChatMemoByteLimit ||
memos.payloadMemo.size() > kHushChatMemoByteLimit) {
memos.payloadMemos[0].size() > kHushChatMemoByteLimit) {
return ChatComposeStatus::TooLong;
}
out = std::move(memos);
@@ -131,9 +162,9 @@ ChatComposeStatus buildOutgoingControl(const std::string& myPublicKeyHex,
memos.recipientZaddr = selfZaddr;
memos.headerMemo = buildHeaderMemo(selfZaddr, conversationId, "Ctrl", "", myPublicKeyHex,
static_cast<std::int64_t>(std::time(nullptr)), op);
memos.payloadMemo = label;
memos.payloadMemos = { label };
if (memos.headerMemo.size() > kHushChatMemoByteLimit ||
memos.payloadMemo.size() > kHushChatMemoByteLimit) {
memos.payloadMemos[0].size() > kHushChatMemoByteLimit) {
return ChatComposeStatus::TooLong;
}
out = std::move(memos);
+8 -4
View File
@@ -12,13 +12,17 @@
#include <array>
#include <string>
#include <vector>
namespace dragonx::chat {
struct OutgoingChatMemos {
std::string recipientZaddr; // the peer's z-address (recipient of both memo outputs)
std::string headerMemo; // JSON header — MUST occupy the lower memo position on the wire
std::string payloadMemo; // ciphertext hex (Message) or plaintext (ContactRequest)
std::string recipientZaddr; // the peer's z-address (recipient of every memo output)
std::string headerMemo; // JSON header
// One payload (ciphertext hex for a Message, plaintext for a ContactRequest/Control), OR — for a
// multi-part Message (header "n" = N > 1) — N index-prefixed ciphertext chunks (a 2-hex part index
// then that chunk's ciphertext hex), reassembled by the receiver regardless of on-chain order.
std::vector<std::string> payloadMemos;
};
// One of the two 0-value memo outputs a HushChat send produces (amount is always 0).
@@ -31,7 +35,7 @@ struct ChatSendOutput {
// `utf8Prefix` prepends the daemon's "utf8:" marker required by full-node z_sendmany (which then
// UTF-8-encodes the bytes on-chain, byte-identical to SDXLite's Memo::from_str); lite backends take
// raw UTF-8, so pass false there.
std::array<ChatSendOutput, 2> chatSendOutputs(const OutgoingChatMemos& memos, bool utf8Prefix);
std::vector<ChatSendOutput> chatSendOutputs(const OutgoingChatMemos& memos, bool utf8Prefix);
enum class ChatComposeStatus {
Ok,
+77
View File
@@ -161,6 +161,13 @@ HushChatHeaderParseResult parseHushChatHeaderMemo(const std::string& memo)
!isHexString(header.secretstream_header_hex)) {
return fail("message header must include a 24 byte secretstream header encoded as hex");
}
// Optional part count ("n"); absent/1 = classic single-payload message. Bound it so a garbage or
// future-oversized header can't make the grouper try to collect an unbounded number of payloads.
if (auto it = object.find("n"); it != object.end() && it->is_number_integer()) {
const int n = it->get<int>();
if (n < 1 || n > kHushChatMaxMessageParts) return fail("message part count out of range");
header.parts = n;
}
} else if (!header.secretstream_header_hex.empty()) {
return fail("contact request header must not include a secretstream header");
}
@@ -189,6 +196,76 @@ HushChatMemoGroupingResult groupHushChatMemoOutputs(const std::vector<HushChatMe
});
HushChatMemoGroupingResult result;
// --- Larger-messages tier 2: one message split across N payload outputs in a SINGLE tx. ---
// A lone Message header with part count ("n") > 1 means its ciphertext was chunked across N payload
// outputs, each prefixed with a 2-hex part index so order survives the daemon's output shuffle.
// Reassemble here and emit one pair. Only a clean one-header + exactly-N-chunks tx takes this path;
// anything else falls through to the classic single-pair logic. An older client lacks this branch,
// mis-pairs the first chunk, fails to decrypt, and drops it — graceful degradation.
{
std::optional<HushChatHeader> multiHeader;
std::size_t multiHeaderPos = 0;
int validHeaders = 0;
bool anyOversize = false;
std::vector<const HushChatMemoOutput*> payloads;
for (const auto& entry : ordered) {
const auto& o = entry.output;
if (o.memo.size() > kHushChatMemoByteLimit) { anyOversize = true; break; }
if (!o.memo.empty() && o.memo.front() == '{') {
auto parsed = parseHushChatHeaderMemo(o.memo);
if (parsed.ok) {
++validHeaders;
if (parsed.header.type == HushChatHeaderType::Message && parsed.header.parts > 1) {
multiHeader = std::move(parsed.header);
multiHeaderPos = o.position;
}
}
} else if (!o.memo.empty()) {
payloads.push_back(&o);
}
}
if (!anyOversize && validHeaders == 1 && multiHeader) {
const int want = multiHeader->parts;
auto hexVal = [](char c) -> int {
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
return -1;
};
std::vector<std::string> chunks(static_cast<std::size_t>(want));
std::vector<bool> seen(static_cast<std::size_t>(want), false);
bool ok = (static_cast<int>(payloads.size()) == want);
std::size_t lowPos = 0; bool haveLow = false;
for (const auto* p : payloads) {
if (!ok) break;
const std::string& m = p->memo;
if (m.size() < kHushChatPartIndexHexLen || (m.size() % 2) != 0 || !isHexString(m)) { ok = false; break; }
const int idx = hexVal(m[0]) * 16 + hexVal(m[1]);
if (idx < 0 || idx >= want || seen[static_cast<std::size_t>(idx)]) { ok = false; break; }
seen[static_cast<std::size_t>(idx)] = true;
chunks[static_cast<std::size_t>(idx)] = m.substr(kHushChatPartIndexHexLen);
if (!haveLow || p->position < lowPos) { lowPos = p->position; haveLow = true; }
}
if (ok) {
std::string full;
for (const auto& c : chunks) full += c;
if (!full.empty() && (full.size() % 2) == 0) {
HushChatMemoPair pair;
pair.header = std::move(*multiHeader);
pair.header_position = multiHeaderPos;
pair.payload_position = haveLow ? lowPos : 0;
pair.payload_memo = std::move(full); // the reassembled full ciphertext hex
result.pairs.push_back(std::move(pair));
return result;
}
}
result.issues.push_back(HushChatMemoGroupingIssueInfo{
HushChatMemoGroupingIssue::MissingPayload, multiHeaderPos,
"multi-part message did not reassemble"});
return result;
}
}
std::optional<HushChatMemoOutput> pending_header_output;
std::optional<HushChatHeader> pending_header;
// A payload memo seen BEFORE its header. The full-node daemon shuffles the two 0-value memo
+11
View File
@@ -25,6 +25,10 @@ struct HushChatHeader {
std::string op; // Control only ("del" / "res" / "sensitive" / "unsensitive" / future ops); empty otherwise
std::string secretstream_header_hex;
std::string public_key_hex;
// Message only: how many payload outputs this message's ciphertext is split across (header "n").
// 1 = the classic single-payload message (field omitted on the wire). >1 means the payloads are
// index-prefixed ciphertext chunks reassembled in order (larger-messages tier 2, one transaction).
int parts = 1;
// Optional sender-stamped compose time (header "ts", Unix seconds). 0 = absent (older sender) → the
// receiver falls back to the tx/receive time. Lets both sides show the SAME (send) time.
std::int64_t sent_at = 0;
@@ -98,6 +102,13 @@ struct HushChatTransactionExtractionResult {
constexpr int kHushChatSupportedVersion = 0;
constexpr std::size_t kHushChatMemoByteLimit = 512;
// Larger-messages tier 2: the most payload outputs a single message may be split across (one tx).
// Keeps the chat tx well under the shielded max_tx_size after Sietch decoy padding + proving cost.
// A message that would need more parts than this is refused at compose time (tier 3 would lift it).
constexpr int kHushChatMaxMessageParts = 16;
// Each multi-part payload chunk is prefixed with a 2-hex-char (1-byte) part index so the receiver can
// reorder chunks after the daemon shuffles a tx's outputs. 2 hex covers indices 0..255 (>> the cap).
constexpr std::size_t kHushChatPartIndexHexLen = 2;
constexpr std::size_t kHushChatPublicKeyHexLength = 64;
constexpr std::size_t kHushChatSecretstreamHeaderHexLength = 48;
+64 -11
View File
@@ -15098,7 +15098,7 @@ void testHushChatControlMemo()
EXPECT_EQ(cm.recipientZaddr, std::string("zs-self"));
HushChatTransactionInput rt; rt.txid = "txid-ctrl-rt";
rt.outputs.push_back({0, cm.headerMemo});
rt.outputs.push_back({1, cm.payloadMemo});
rt.outputs.push_back({1, cm.payloadMemos[0]});
auto rx = extractHushChatTransactionMetadata(rt, true);
EXPECT_EQ((int)rx.metadata.size(), 1);
EXPECT_TRUE(rx.metadata[0].type == HushChatHeaderType::Control);
@@ -15132,7 +15132,7 @@ void testHushChatControlReducer()
auto feed = [&](const OutgoingChatMemos& m, const std::string& txid, std::int64_t ts) {
HushChatTransactionInput tx; tx.txid = txid;
tx.outputs.push_back({0, m.headerMemo});
tx.outputs.push_back({1, m.payloadMemo});
tx.outputs.push_back({1, m.payloadMemos[0]});
auto ex = extractHushChatTransactionMetadata(tx, true);
return svc.ingest(ex.metadata, {{txid, ts}}, ts);
};
@@ -15409,7 +15409,7 @@ void testHushChatSensitiveService()
EXPECT_TRUE(buildOutgoingMessage(peer, rp.public_key_hex, "zs-peer", rm.public_key_hex, "zs-me",
cid, body, m) == ChatComposeStatus::Ok);
HushChatTransactionInput tx; tx.txid = txid;
tx.outputs.push_back({0, m.headerMemo}); tx.outputs.push_back({1, m.payloadMemo});
tx.outputs.push_back({0, m.headerMemo}); tx.outputs.push_back({1, m.payloadMemos[0]});
auto ex = extractHushChatTransactionMetadata(tx, true);
const std::int64_t t = static_cast<std::int64_t>(std::time(nullptr));
return svc.ingest(ex.metadata, {{txid, t}}, t);
@@ -15694,6 +15694,58 @@ void testHushChatDeleteConversation()
// Phase 4: outgoing memo construction round-trips through the receive parser + decrypt, and
// ChatService compose/recordOutgoing echoes into the store.
// Larger messages (tier 2): a long message is chunked across N index-prefixed payload outputs in ONE
// tx and reassembled on receipt regardless of on-chain output order; the cap refuses an over-long body.
void testHushChatLargeMessage()
{
using namespace dragonx::chat;
ChatKeyPair alice, bob;
ChatIdentityResult ra = deriveChatIdentityFromSecret("lm-alice", alice, true);
ChatIdentityResult rb = deriveChatIdentityFromSecret("lm-bob", bob, true);
EXPECT_TRUE(ra.status == ChatIdentityStatus::Ready && rb.status == ChatIdentityStatus::Ready);
const std::string cid = "conv-lm";
std::string big; // well beyond one memo (~239 bytes)
for (int i = 0; i < 1500; ++i) big += static_cast<char>('A' + (i % 26));
OutgoingChatMemos memos;
EXPECT_TRUE(buildOutgoingMessage(alice, ra.public_key_hex, "zs-alice", rb.public_key_hex, "zs-bob",
cid, big, memos) == ChatComposeStatus::Ok);
EXPECT_TRUE((int)memos.payloadMemos.size() > 1); // actually chunked
EXPECT_TRUE(memos.headerMemo.find("\"n\":") != std::string::npos); // carries the part count
// Build the tx (header + all payloads) and REVERSE the output order to mimic the daemon's
// ShuffleOutputs — payloads before the header, indices descending. Reassembly must still work.
std::vector<std::string> wire;
wire.push_back(memos.headerMemo);
for (const auto& p : memos.payloadMemos) wire.push_back(p);
std::reverse(wire.begin(), wire.end());
HushChatTransactionInput tx; tx.txid = "lm1";
for (std::size_t i = 0; i < wire.size(); ++i) tx.outputs.push_back({i, wire[i]});
auto ext = extractHushChatTransactionMetadata(tx, true);
EXPECT_EQ((int)ext.metadata.size(), 1);
EXPECT_TRUE(ext.metadata[0].type == HushChatHeaderType::Message);
std::string plain;
EXPECT_TRUE(decryptIncoming(bob, ext.metadata[0].sender_public_key_hex,
ext.metadata[0].secretstream_header_hex,
ext.metadata[0].payload_memo, plain) == ChatCryptoStatus::Ok);
EXPECT_EQ(plain, big);
// A short message stays single-payload and byte-identical (no "n" field, no index prefix).
OutgoingChatMemos small;
EXPECT_TRUE(buildOutgoingMessage(alice, ra.public_key_hex, "zs-alice", rb.public_key_hex, "zs-bob",
cid, "hi", small) == ChatComposeStatus::Ok);
EXPECT_EQ((int)small.payloadMemos.size(), 1);
EXPECT_TRUE(small.headerMemo.find("\"n\":") == std::string::npos);
// Cap: a body needing more than kHushChatMaxMessageParts chunks is refused.
std::string huge(6000, 'x');
OutgoingChatMemos tooBig;
EXPECT_TRUE(buildOutgoingMessage(alice, ra.public_key_hex, "zs-alice", rb.public_key_hex, "zs-bob",
cid, huge, tooBig) == ChatComposeStatus::TooLong);
}
void testHushChatOutgoing()
{
using namespace dragonx::chat;
@@ -15717,7 +15769,7 @@ void testHushChatOutgoing()
HushChatTransactionInput tx;
tx.txid = "txout1";
tx.outputs.push_back({0, memos.headerMemo}); // header at the lower position
tx.outputs.push_back({1, memos.payloadMemo});
tx.outputs.push_back({1, memos.payloadMemos[0]});
auto extracted = extractHushChatTransactionMetadata(tx, true);
EXPECT_EQ((int)extracted.metadata.size(), 1);
@@ -15742,7 +15794,7 @@ void testHushChatOutgoing()
HushChatTransactionInput tx2;
tx2.txid = "txout2";
tx2.outputs.push_back({0, creq.headerMemo});
tx2.outputs.push_back({1, creq.payloadMemo});
tx2.outputs.push_back({1, creq.payloadMemos[0]});
auto ex2 = extractHushChatTransactionMetadata(tx2, true);
EXPECT_EQ((int)ex2.metadata.size(), 1);
EXPECT_TRUE(ex2.metadata[0].type == HushChatHeaderType::ContactRequest);
@@ -15817,18 +15869,18 @@ void testHushChatTransport()
"zs-bob", "cid-tx", "over the wire", memos) == ChatComposeStatus::Ok);
// Full-node: header at index 0, both to the peer, each "utf8:"-prefixed.
std::array<ChatSendOutput, 2> fn = chatSendOutputs(memos, /*utf8Prefix=*/true);
auto fn = chatSendOutputs(memos, /*utf8Prefix=*/true);
EXPECT_EQ(fn[0].address, std::string("zs-bob"));
EXPECT_EQ(fn[1].address, std::string("zs-bob"));
EXPECT_TRUE(fn[0].memo.rfind("utf8:", 0) == 0);
EXPECT_TRUE(fn[1].memo.rfind("utf8:", 0) == 0);
EXPECT_EQ(fn[0].memo, std::string("utf8:") + memos.headerMemo);
EXPECT_EQ(fn[1].memo, std::string("utf8:") + memos.payloadMemo);
EXPECT_EQ(fn[1].memo, std::string("utf8:") + memos.payloadMemos[0]);
// Lite: raw, no prefix.
std::array<ChatSendOutput, 2> lt = chatSendOutputs(memos, /*utf8Prefix=*/false);
auto lt = chatSendOutputs(memos, /*utf8Prefix=*/false);
EXPECT_EQ(lt[0].memo, memos.headerMemo);
EXPECT_EQ(lt[1].memo, memos.payloadMemo);
EXPECT_EQ(lt[1].memo, memos.payloadMemos[0]);
// On-chain form round-trips: the daemon stores the UTF-8 text (the string after "utf8:"), which
// is exactly what the harvest parser reads back (header first / lower position).
@@ -15863,7 +15915,7 @@ void testHushChatShuffledReceive()
"zs-bob", "cid-shuf", "shuffled hi", memos) == ChatComposeStatus::Ok);
HushChatTransactionInput tx;
tx.txid = "txshuf";
tx.outputs.push_back({0, memos.payloadMemo}); // payload at the LOWER position
tx.outputs.push_back({0, memos.payloadMemos[0]}); // payload at the LOWER position
tx.outputs.push_back({1, memos.headerMemo}); // header at the HIGHER position
auto extracted = extractHushChatTransactionMetadata(tx, true);
EXPECT_EQ((int)extracted.metadata.size(), 1);
@@ -15877,7 +15929,7 @@ void testHushChatShuffledReceive()
HushChatTransactionInput tx2;
tx2.txid = "txshuf2";
tx2.outputs.push_back({0, std::string()}); // change output: empty memo (ignored)
tx2.outputs.push_back({1, memos2.payloadMemo}); // payload
tx2.outputs.push_back({1, memos2.payloadMemos[0]}); // payload
tx2.outputs.push_back({2, memos2.headerMemo}); // header last
auto ex2 = extractHushChatTransactionMetadata(tx2, true);
EXPECT_EQ((int)ex2.metadata.size(), 1);
@@ -20303,6 +20355,7 @@ int main()
testChatStoreTxidIndex();
testChatStoreLatestAndMove();
testHushChatOutgoing();
testHushChatLargeMessage();
testHushChatTransport();
testHushChatShuffledReceive();
testWalletFileProbe();