fix(net): cap market and bootstrap HTTP response bodies
Several HTTP fetches read a remote body into memory without a tight bound. The price and exchange-ticker calls shared one large 16 MiB metadata cap, and the bootstrap checksum download had no size limit at all, so a hostile or MITM'd server streaming an unbounded (and Content-Length-free) response could grow memory far past anything these endpoints legitimately return. - The HTTP GET helper now takes a per-call byte cap and redirect limit. The write callback refuses bytes past the cap, which aborts the transfer, and the same value is set as curl's max file size. - The price call is capped at 256 KiB and the ticker list at 2 MiB, each sized to its real payload rather than the shared default. - The bootstrap checksum fetch caps its body at 64 KiB in both the write callback and curl's max file size. Tests cover the sink accept/abort boundary. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+6
-2
@@ -3201,7 +3201,9 @@ void App::refreshPrice()
|
|||||||
NetworkRefreshService::PriceHttpResult result;
|
NetworkRefreshService::PriceHttpResult result;
|
||||||
try {
|
try {
|
||||||
const char* const url = "https://api.coingecko.com/api/v3/simple/price?ids=dragonx-2&vs_currencies=usd,btc&include_24hr_change=true&include_24hr_vol=true&include_market_cap=true";
|
const char* const url = "https://api.coingecko.com/api/v3/simple/price?ids=dragonx-2&vs_currencies=usd,btc&include_24hr_change=true&include_24hr_vol=true&include_market_cap=true";
|
||||||
util::HttpGetResult r = util::httpGet(url, "[price]", marketHttp(&http_cancel_, 10));
|
util::HttpGetOptions priceOpt = marketHttp(&http_cancel_, 10);
|
||||||
|
priceOpt.maxBytes = 256 * 1024;
|
||||||
|
util::HttpGetResult r = util::httpGet(url, "[price]", priceOpt);
|
||||||
NetworkRefreshService::PriceHttpResponse response{
|
NetworkRefreshService::PriceHttpResponse response{
|
||||||
r.transportOk, r.httpStatus, std::move(r.body), r.error};
|
r.transportOk, r.httpStatus, std::move(r.body), r.error};
|
||||||
result = NetworkRefreshService::parsePriceHttpResponse(response, std::time(nullptr));
|
result = NetworkRefreshService::parsePriceHttpResponse(response, std::time(nullptr));
|
||||||
@@ -3433,9 +3435,11 @@ void App::refreshExchanges()
|
|||||||
|
|
||||||
http_worker_->post([this]() -> rpc::RPCWorker::MainCb {
|
http_worker_->post([this]() -> rpc::RPCWorker::MainCb {
|
||||||
// httpGetString verifies TLS, caps the response size and returns "" on any failure.
|
// httpGetString verifies TLS, caps the response size and returns "" on any failure.
|
||||||
|
util::HttpGetOptions tickersOpt = marketHttp(&http_cancel_, 12);
|
||||||
|
tickersOpt.maxBytes = 2 * 1024 * 1024;
|
||||||
const std::string body = util::httpGetString(
|
const std::string body = util::httpGetString(
|
||||||
"https://api.coingecko.com/api/v3/coins/dragonx-2/tickers", "[tickers]",
|
"https://api.coingecko.com/api/v3/coins/dragonx-2/tickers", "[tickers]",
|
||||||
marketHttp(&http_cancel_, 12));
|
tickersOpt);
|
||||||
std::vector<data::ExchangeInfo> exchanges;
|
std::vector<data::ExchangeInfo> exchanges;
|
||||||
if (!body.empty()) {
|
if (!body.empty()) {
|
||||||
try { exchanges = data::parseCoinGeckoTickers(body); } catch (...) {}
|
try { exchanges = data::parseCoinGeckoTickers(body); } catch (...) {}
|
||||||
|
|||||||
@@ -868,6 +868,7 @@ bool Bootstrap::cleanChainData(const std::string& dataDir) {
|
|||||||
static size_t writeStringCallback(void* contents, size_t size, size_t nmemb, void* userp) {
|
static size_t writeStringCallback(void* contents, size_t size, size_t nmemb, void* userp) {
|
||||||
size_t total = size * nmemb;
|
size_t total = size * nmemb;
|
||||||
auto* str = static_cast<std::string*>(userp);
|
auto* str = static_cast<std::string*>(userp);
|
||||||
|
if (str->size() + total > (64u * 1024)) return 0; // refuse an oversized checksum file -> curl aborts
|
||||||
str->append(static_cast<const char*>(contents), total);
|
str->append(static_cast<const char*>(contents), total);
|
||||||
return total;
|
return total;
|
||||||
}
|
}
|
||||||
@@ -881,6 +882,7 @@ std::string Bootstrap::downloadSmallFile(const std::string& url) {
|
|||||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeStringCallback);
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeStringCallback);
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
|
||||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, static_cast<curl_off_t>(64 * 1024));
|
||||||
curl_easy_setopt(curl, CURLOPT_USERAGENT, "ObsidianDragon/1.0");
|
curl_easy_setopt(curl, CURLOPT_USERAGENT, "ObsidianDragon/1.0");
|
||||||
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L);
|
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L);
|
||||||
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15L);
|
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15L);
|
||||||
|
|||||||
@@ -22,16 +22,20 @@ namespace {
|
|||||||
|
|
||||||
// Cap for in-memory text/JSON responses (release metadata, price/candle data). Far above any
|
// Cap for in-memory text/JSON responses (release metadata, price/candle data). Far above any
|
||||||
// legitimate body, but bounds memory if a hostile/MITM'd server streams an unbounded response.
|
// legitimate body, but bounds memory if a hostile/MITM'd server streams an unbounded response.
|
||||||
constexpr std::size_t kMaxMetadataBytes = 16u * 1024 * 1024;
|
|
||||||
|
struct StringSink {
|
||||||
|
std::string* body;
|
||||||
|
long long cap;
|
||||||
|
};
|
||||||
|
|
||||||
size_t writeStringCb(void* contents, size_t size, size_t nmemb, void* userp)
|
size_t writeStringCb(void* contents, size_t size, size_t nmemb, void* userp)
|
||||||
{
|
{
|
||||||
auto* s = static_cast<std::string*>(userp);
|
auto* sink = static_cast<StringSink*>(userp);
|
||||||
const size_t n = size * nmemb;
|
const size_t n = size * nmemb;
|
||||||
// Hard cap: a chunked response omits Content-Length, so CURLOPT_MAXFILESIZE cannot catch it —
|
// Hard cap: a chunked response omits Content-Length, so CURLOPT_MAXFILESIZE cannot catch it —
|
||||||
// aborting here (short count) makes curl fail the transfer instead of exhausting memory.
|
// aborting here (short count) makes curl fail the transfer instead of exhausting memory.
|
||||||
if (s->size() + n > kMaxMetadataBytes) return 0;
|
if (httpSinkAccept(sink->body->size(), n, sink->cap) != n) return 0;
|
||||||
s->append(static_cast<char*>(contents), n);
|
sink->body->append(static_cast<char*>(contents), n);
|
||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,13 +72,15 @@ HttpGetResult httpGet(const std::string& url, const char* logTag, const HttpGetO
|
|||||||
if (!curl) { out.error = "failed to initialize curl"; return out; }
|
if (!curl) { out.error = "failed to initialize curl"; return out; }
|
||||||
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeStringCb);
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeStringCb);
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out.body);
|
StringSink sink{&out.body, opt.maxBytes};
|
||||||
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &sink);
|
||||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, opt.maxRedirects);
|
||||||
curl_easy_setopt(curl, CURLOPT_USERAGENT, "ObsidianDragon/1.0");
|
curl_easy_setopt(curl, CURLOPT_USERAGENT, "ObsidianDragon/1.0");
|
||||||
curl_easy_setopt(curl, CURLOPT_TIMEOUT, opt.timeoutSec);
|
curl_easy_setopt(curl, CURLOPT_TIMEOUT, opt.timeoutSec);
|
||||||
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, opt.connectTimeoutSec);
|
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, opt.connectTimeoutSec);
|
||||||
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); // no SIGALRM for timeouts: these run on worker threads
|
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); // no SIGALRM for timeouts: these run on worker threads
|
||||||
curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, static_cast<curl_off_t>(kMaxMetadataBytes)); // reject oversized metadata
|
curl_easy_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, static_cast<curl_off_t>(opt.maxBytes)); // reject oversized metadata
|
||||||
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
|
||||||
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L);
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L);
|
||||||
if (opt.cancel) {
|
if (opt.cancel) {
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
|
#include <cstddef>
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
@@ -25,8 +26,16 @@ struct HttpGetOptions {
|
|||||||
// When non-null and set (before or during the transfer) the GET is abandoned: checked once up front,
|
// When non-null and set (before or during the transfer) the GET is abandoned: checked once up front,
|
||||||
// then by libcurl's progress callback (about once a second even while resolving / connecting).
|
// then by libcurl's progress callback (about once a second even while resolving / connecting).
|
||||||
const std::atomic<bool>* cancel = nullptr;
|
const std::atomic<bool>* cancel = nullptr;
|
||||||
|
long long maxBytes = 16LL * 1024 * 1024; // per-call body cap (also sets MAXFILESIZE_LARGE)
|
||||||
|
long maxRedirects = 5;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Append `incoming` bytes only if within `cap`; returns bytes accepted (0 => abort the transfer).
|
||||||
|
// cap <= 0 means unlimited.
|
||||||
|
inline std::size_t httpSinkAccept(std::size_t have, std::size_t incoming, long long cap) {
|
||||||
|
return (cap > 0 && static_cast<long long>(have + incoming) > cap) ? 0 : incoming;
|
||||||
|
}
|
||||||
|
|
||||||
struct HttpGetResult {
|
struct HttpGetResult {
|
||||||
bool transportOk = false; // curl finished without error (any HTTP status)
|
bool transportOk = false; // curl finished without error (any HTTP status)
|
||||||
long httpStatus = 0; // 0 when no response arrived
|
long httpStatus = 0; // 0 when no response arrived
|
||||||
|
|||||||
@@ -855,6 +855,20 @@ void testSecureWipeAndRemove()
|
|||||||
fs::remove_all(dir, ec);
|
fs::remove_all(dir, ec);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void testHttpSinkAccept()
|
||||||
|
{
|
||||||
|
using dragonx::util::httpSinkAccept;
|
||||||
|
EXPECT_EQ(httpSinkAccept(0, 100, 1000), (size_t)100);
|
||||||
|
EXPECT_EQ(httpSinkAccept(900, 100, 1000), (size_t)100); // exactly at the cap is allowed
|
||||||
|
EXPECT_EQ(httpSinkAccept(901, 100, 1000), (size_t)0); // crossing the cap aborts
|
||||||
|
EXPECT_EQ(httpSinkAccept(0, 1001, 1000), (size_t)0);
|
||||||
|
EXPECT_EQ(httpSinkAccept(5000, 100, 0), (size_t)100); // cap <= 0 => unlimited
|
||||||
|
EXPECT_EQ(httpSinkAccept(5000, 100, -1), (size_t)100);
|
||||||
|
dragonx::util::HttpGetOptions o;
|
||||||
|
EXPECT_EQ(o.maxBytes, 16LL * 1024 * 1024);
|
||||||
|
EXPECT_EQ(o.maxRedirects, 5L);
|
||||||
|
}
|
||||||
|
|
||||||
void testClassifyUnlockFailure()
|
void testClassifyUnlockFailure()
|
||||||
{
|
{
|
||||||
using dragonx::rpc::RpcError;
|
using dragonx::rpc::RpcError;
|
||||||
@@ -18283,6 +18297,7 @@ int main()
|
|||||||
testConnectionConfig();
|
testConnectionConfig();
|
||||||
testClassifyRpcFailure();
|
testClassifyRpcFailure();
|
||||||
testClassifyUnlockFailure();
|
testClassifyUnlockFailure();
|
||||||
|
testHttpSinkAccept();
|
||||||
testSecureWipeAndRemove();
|
testSecureWipeAndRemove();
|
||||||
testFormatHttpError();
|
testFormatHttpError();
|
||||||
testRpcFailureKindOf();
|
testRpcFailureKindOf();
|
||||||
|
|||||||
Reference in New Issue
Block a user