qa: add fd-guard, the close() guard and regtest harness that found the double close
shim/ is an LD_PRELOAD guard around close/fclose/dup2/dup3: - before each close it resolves /proc/self/fd/N; - it reports, with a backtrace, a non-owner close of wallet.dat, database/log.* or blocks/*.dat, and every close that fails with EBADF (the second half of a double close); - FDGUARD_ABORT=1 aborts on the first hit; - run_tests.sh self-tests it (51 checks). repro/ runs a shimmed dragonxd on isolated regtest. Each run gets its own unprivileged network namespace (loopback only). The loads are: - rpc: a hammer with impatient/RST/half-open clients, work-queue bursts and long-polls; - p2p: TLS peers churned by disconnectnode, ban and kill, plus a raw TCP/TLS misbehaver; - tor: a fake control port; - mine: mining with blocknotify and autoshield. tools/tally.py groups the reports by closer. tools/scan_bdb.py finds BDB pages written into block files. With the wolfSSL CloseSocket macro in effect: - p2pmin (300 s) gives one EBADF double close per disconnect; - the full load (1200 s) gives 930 EBADF closes, 186 non-owner closes (37 of wallet/log/blk files) and a 632 KB db.log with BDB0060 PANIC; - RPC-only and Tor-only loads give none. After the fix, every count is 0 and db.log stays empty. The README has the commands and the numbers. Paths come from the environment (GOLDEN, BIN, RUNS, SHIM, ASMAP and the Sapling params). Run output goes to $RUNS, outside the tree. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""tally.py <fdguard.log> -- attribute every fdguard report to (type, closer function, target class)
|
||||
and, for EBADF/NON-OWNER reports, to the code that made the previous close of that fd number.
|
||||
|
||||
The point: show that the stray closes ORIGINATE in CNode code (net.cpp CloseSocketDisconnect /
|
||||
~CNode closing a number they already closed) and that libevent / BDB / CreateNodeFromAcceptedSocket /
|
||||
GetTransaction only appear as victims-turned-amplifiers downstream of that.
|
||||
"""
|
||||
import collections, re, sys
|
||||
|
||||
ORIGIN = ("_ZN5CNode21CloseSocketDisconnectEv", "_ZN5CNodeD1Ev", "_ZN5CNodeD2Ev", "_ZN11CNetCleanupD2Ev")
|
||||
NAMES = {"_ZN5CNode21CloseSocketDisconnectEv": "CNode::CloseSocketDisconnect (net.cpp:572)",
|
||||
"_ZN5CNodeD1Ev": "CNode::~CNode (net.cpp:2819)",
|
||||
"_ZN11CNetCleanupD2Ev": "CNetCleanup::~CNetCleanup (net.cpp:2465, shutdown)",
|
||||
"evhttp_connection_free": "libevent evhttp_connection_free (http.c:1244)",
|
||||
"__os_closehandle": "BDB __os_closehandle",
|
||||
"_Z28CreateNodeFromAcceptedSocketjbRK8CAddressS1_": "CreateNodeFromAcceptedSocket (net.cpp:1336)",
|
||||
"_Z14GetTransactionRK7uint256R12CTransactionRS_b": "GetTransaction fclose (main.cpp)",
|
||||
"epoll_nochangelist_add": "libevent epoll ADD", "epoll_nochangelist_del": "libevent epoll DEL/MOD"}
|
||||
|
||||
|
||||
def fn(frame_line):
|
||||
m = re.search(r"\+0x[0-9a-f]+ (\S+?)\+0x", frame_line)
|
||||
return m.group(1) if m else "?"
|
||||
|
||||
|
||||
def main(path):
|
||||
reps, cur = [], None
|
||||
for line in open(path, errors="replace"):
|
||||
if line.startswith("=== FDGUARD #"):
|
||||
cur = {"hdr": line.strip(), "lines": []}
|
||||
reps.append(cur)
|
||||
elif cur is not None:
|
||||
if line.startswith("=== END"):
|
||||
cur = None
|
||||
else:
|
||||
cur["lines"].append(line.rstrip("\n"))
|
||||
tally = collections.Counter()
|
||||
prev_tally = collections.Counter()
|
||||
db_blk = []
|
||||
for r in reps:
|
||||
typ = r["hdr"].split()[3]
|
||||
L = r["lines"]
|
||||
cls = next((re.search(r"class=(\S+)", l).group(1) for l in L if " target=" in l and "class=" in l), "?")
|
||||
try:
|
||||
i = next(k for k, l in enumerate(L) if l.strip() == "closer stack:")
|
||||
closer = fn(L[i + 1])
|
||||
except StopIteration:
|
||||
closer = "?"
|
||||
prev = "-"
|
||||
for k, l in enumerate(L):
|
||||
if "previous successful close of this fd number" in l:
|
||||
# first frame line after the target= line
|
||||
for l2 in L[k + 1:k + 4]:
|
||||
if l2.strip().startswith("#0"):
|
||||
prev = fn(l2)
|
||||
break
|
||||
break
|
||||
tally[(typ, NAMES.get(closer, closer), cls)] += 1
|
||||
if typ in ("EBADF-CLOSE", "NON-OWNER-CLOSE"):
|
||||
prev_tally[NAMES.get(prev, prev)] += 1
|
||||
if typ == "NON-OWNER-CLOSE" and cls in ("db", "blk"):
|
||||
tgt = next((l.split("target=")[1].split()[0] for l in L if " target=" in l), "?")
|
||||
db_blk.append((r["hdr"].split()[2], r["hdr"].split()[4], NAMES.get(closer, closer), tgt.rsplit("/", 2)[-2:]))
|
||||
print(f"{len(reps)} reports in {path}")
|
||||
print("\n count type class closer")
|
||||
for (typ, closer, cls), n in tally.most_common():
|
||||
print(f"{n:6d} {typ:17s} {cls:6s} {closer}")
|
||||
print("\nEBADF + NON-OWNER reports by the code that made the PREVIOUS close of the same number:")
|
||||
for k, n in prev_tally.most_common():
|
||||
print(f"{n:6d} {k}")
|
||||
o = sum(n for (typ, c, cls), n in tally.items() if typ == "NON-OWNER-CLOSE" and cls in ("db", "blk") and c.startswith(("CNode", "CNetCleanup")))
|
||||
a = sum(n for (typ, c, cls), n in tally.items() if typ == "NON-OWNER-CLOSE" and cls in ("db", "blk")) - o
|
||||
print(f"\nNON-OWNER closes of wallet/BDB-log/block files: {o + a} (by CNode code directly: {o}; by an amplifier: {a})")
|
||||
for d in db_blk[:40]:
|
||||
print(" ", d[0], d[1], "closer=" + d[2], "target=" + "/".join(d[3]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1])
|
||||
Reference in New Issue
Block a user