basic implementation of transaction replacement. requires cleanup and testing

src/Makefile.am

@@ -137,6 +137,7 @@ BITCOIN_CORE_H = \
   init.h \
   key.h \
   keystore.h \
+  komodo_cryptoconditions.h \
   leveldbwrapper.h \
   limitedmap.h \
   main.h \
@@ -155,6 +156,7 @@ BITCOIN_CORE_H = \
   protocol.h \
   pubkey.h \
   random.h \
+  replacementpool.h \
   reverselock.h \
   rpcclient.h \
   rpcprotocol.h \
@@ -223,6 +225,7 @@ libbitcoin_server_a_SOURCES = \
   httprpc.cpp \
   httpserver.cpp \
   init.cpp \
+  komodo_cryptoconditions.cpp \
   leveldbwrapper.cpp \
   main.cpp \
   merkleblock.cpp \
@@ -232,6 +235,7 @@ libbitcoin_server_a_SOURCES = \
   noui.cpp \
   policy/fees.cpp \
   pow.cpp \
+  replacementpool.cpp \
   rest.cpp \
   rpcblockchain.cpp \
   rpcmining.cpp \
@@ -325,6 +329,7 @@ libbitcoin_common_a_SOURCES = \
   hash.cpp \
   key.cpp \
   keystore.cpp \
+  komodo_cryptoconditions.cpp \
   netbase.cpp \
   primitives/block.cpp \
   primitives/transaction.cpp \
@@ -336,7 +341,6 @@ libbitcoin_common_a_SOURCES = \
   script/script_error.cpp \
   script/sign.cpp \
   script/standard.cpp \
-  komodo_cryptoconditions.cpp \
   $(BITCOIN_CORE_H) \
   $(LIBZCASH_H)
 
@@ -506,7 +510,6 @@ libzcashconsensus_la_SOURCES = \
   script/zcashconsensus.cpp \
   script/interpreter.cpp \
   script/script.cpp \
-  komodo_cryptoconditions.cpp \
   uint256.cpp \
   utilstrencodings.cpp
 

src/komodo_cryptoconditions.cpp

@@ -1,24 +1,93 @@
 
+#include "replacementpool.h"
+#include "komodo_cryptoconditions.h"
 #include "cryptoconditions/include/cryptoconditions.h"
 #include "script/interpreter.h"
+#include "coins.h"
 
 
-int TransactionSignatureChecker::CheckAuxCondition(const CC *cond) const
+#define REPLACEMENT_WINDOW_BLOCKS 2
+
+
+bool GetOpReturnData(const CScript &sig, std::vector<unsigned char> &data)
 {
-    // Check that condition is equal to fulfillment
-    if (0 == strcmp((const char*)cond->method, "equals")) {
-        return (cond->conditionAuxLength == cond->fulfillmentAuxLength) &&
-               (0 == memcmp(cond->conditionAux, cond->fulfillmentAux, cond->conditionAuxLength));
-    }
+    auto pc = sig.begin();
+    opcodetype opcode;
+    if (sig.GetOp(pc, opcode))
+        if (opcode == OP_RETURN)
+            if (sig.GetOp(pc, opcode, data))
+                return opcode > OP_0 && opcode <= OP_PUSHDATA4;
+    return false;
+}
 
-    // Check that pubKeyScript specified in fulfillment is OP_RETURN
-    if (0 == strcmp((const char*)cond->method, "inputIsReturn")) {
-        if (cond->fulfillmentAuxLength != 1) return 0;
-        int n = (int) cond->fulfillmentAux[0];
-        if (n >= txTo->vout.size()) return 0;
-        uint8_t *ptr = (uint8_t *)txTo->vout[n].scriptPubKey.data();
-        return ptr[0] == OP_RETURN;
+
+bool EvalConditionBool(const CC *cond, const CTransaction *txTo)
+{
+    if (strcmp(cond->method, "testEval") == 0) {
+        return cond->paramsBinLength == 8 &&
+            memcmp(cond->paramsBin, "testEval", 8) == 0;
     }
-    printf("no defined behaviour for method:%s\n", cond->method);
+    if (strcmp(cond->method, "testReplace") == 0) {
+        return true;
+    }
+    fprintf(stderr, "no defined behaviour for method: %s\n", cond->method);
     return 0;
 }
+
+
+bool GetConditionPriority(const CC *cond, CTxReplacementPoolItem *rep)
+{
+    if (strcmp((const char*)cond->method, "testReplace") == 0) {
+        std::vector<unsigned char> data;
+        if (GetOpReturnData(rep->tx.vout[0].scriptPubKey, data)) {
+            rep->priority = (uint64_t) data[0];
+            rep->replacementWindow = REPLACEMENT_WINDOW_BLOCKS;
+            return true;
+        }
+    }
+    return false;
+}
+
+
+bool TransactionSignatureChecker::CheckEvalCondition(const CC *cond) const
+{
+    return EvalConditionBool(cond, txTo);
+}
+
+
+extern "C"
+{
+    int visitConditionPriority(CC *cond, struct CCVisitor visitor);
+}
+
+
+int visitConditionPriority(CC *cond, struct CCVisitor visitor)
+{
+    if (cc_typeId(cond) == CC_Eval) {
+        if (GetConditionPriority(cond, (CTxReplacementPoolItem*)visitor.context)) {
+            return 0; // stop
+        }
+    }
+    return 1; // continue
+}
+
+
+bool SetReplacementParams(CTxReplacementPoolItem &rep)
+{
+    // first, see if we have a cryptocondition
+    const CScript &sig = rep.tx.vin[0].scriptSig;
+    if (!sig.IsPushOnly()) return false;
+    CScript::const_iterator pc = sig.begin();
+    std::vector<unsigned char> data;
+    opcodetype opcode;
+    if (!sig.GetOp(pc, opcode, data)) return false;
+    CC *cond = cc_readFulfillmentBinary((unsigned char*)data.data(), data.size());
+    if (!cond) return false;
+
+    // now, see if it has a replacement node
+    CC *replacementNode = 0;
+    CCVisitor visitor = {&visitConditionPriority, (const unsigned char*)"", 0, &rep};
+    bool out = cc_visit(cond, visitor);
+    cc_free(cond);
+    return !out;
+}

src/komodo_cryptoconditions.h

@@ -1,12 +1,19 @@
 #ifndef KOMODO_CRYPTOCONDITIONS_H
 #define KOMODO_CRYPTOCONDITIONS_H
 
+#include "coins.h"
+#include "replacementpool.h"
 #include "cryptoconditions/include/cryptoconditions.h"
 
 extern int32_t ASSETCHAINS_CC;
 
+extern CTxReplacementPool replacementPool;
+
 static bool IsCryptoConditionsEnabled() {
     return 0 != ASSETCHAINS_CC;
 }
 
+bool EvalConditionBool(const CC *cond, const CTransaction *tx);
+//uint64_t EvalConditionPriority(const CC *cond, const CTransaction *tx);
+bool SetReplacementParams(CTxReplacementPoolItem &rep);
 #endif /* KOMODO_CRYPTOCONDITIONS_H */

src/main.cpp

@@ -21,6 +21,8 @@
 #include "pow.h"
 #include "txdb.h"
 #include "txmempool.h"
+#include "replacementpool.h"
+#include "komodo_cryptoconditions.h"
 #include "ui_interface.h"
 #include "undo.h"
 #include "util.h"
@@ -53,6 +55,7 @@ extern uint8_t NOTARY_PUBKEY33[33];
 
 BlockMap mapBlockIndex;
 CChain chainActive;
+
 CBlockIndex *pindexBestHeader = NULL;
 int64_t nTimeBestReceived = 0;
 CWaitableCriticalSection csBestBlock;
@@ -1107,7 +1110,7 @@ CAmount GetMinRelayFee(const CTransaction& tx, unsigned int nBytes, bool fAllowF
 }
 
 
-bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree,bool* pfMissingInputs, bool fRejectAbsurdFee)
+bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree,bool* pfMissingInputs, bool fRejectAbsurdFee, bool fAcceptReplacement)
 {
     AssertLockHeld(cs_main);
     if (pfMissingInputs)
@@ -1178,6 +1181,8 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
     }
     }
 
+    CTxReplacementPoolResult rpr = RP_NotReplace;
+
     {
         CCoinsView dummy;
         CCoinsViewCache view(&dummy);
@@ -1331,13 +1336,35 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
             return error("AcceptToMemoryPool: BUG! PLEASE REPORT THIS! ConnectInputs failed against MANDATORY but not STANDARD flags %s", hash.ToString());
         }
 
-        // Store transaction in memory
         if ( komodo_is_notarytx(tx) == 0 )
             KOMODO_ON_DEMAND++;
-        pool.addUnchecked(hash, entry, !IsInitialBlockDownload());
+
+        if (fAcceptReplacement) {
+            CTxReplacementPoolItem item(tx, GetHeight());
+            if (SetReplacementParams(item)) {
+                rpr = replacementPool.replace(item);
+            }
+        }
+
+        if (rpr == RP_Invalid) {
+            // already have a better one
+            // TODO: Shouldn't neccesary log this as invalid, not sure if punishing peers is the best idea
+            fprintf(stderr,"accept failure.20\n");
+            return error("AcceptToMemoryPool: Replacement is worse %s", hash.ToString());
+        }
+
+        // If there is no replacement action happening...
+        if (rpr == RP_NotReplace) {
+            // Store transaction in memory
+            pool.addUnchecked(hash, entry, !IsInitialBlockDownload());
+        }
     }
 
-    SyncWithWallets(tx, NULL);
+    // in order for replaceable transactions to sync with wallet, replacementpool should advise
+    // wallet of transaction eviction
+    if (rpr == RP_NotReplace) {
+        SyncWithWallets(tx, NULL);
+    }
 
     return true;
 }
@@ -1354,6 +1381,12 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock
         return true;
     }
 
+    // replacementPool lookup is O(n) since there's no index by txid
+    if (fAllowSlow && replacementPool.lookup(hash, txOut))
+    {
+        return true;
+    }
+
     if (fTxIndex) {
         CDiskTxPos postx;
         if (pblocktree->ReadTxIndex(hash, postx)) {
@@ -1403,6 +1436,17 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock
     return false;
 }
 
+
+void ProcessReplacementPool(int newHeight)
+{
+    std::vector<CTransaction> pending;
+    replacementPool.removePending(newHeight, pending);
+    CValidationState stateDummy;
+    BOOST_FOREACH(CTransaction tx, pending) {
+        AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, false, false);
+    }
+}
+
 /*char *komodo_getspendscript(uint256 hash,int32_t n)
 {
     CTransaction tx; uint256 hashBlock;
@@ -2702,6 +2746,13 @@ bool static DisconnectTip(CValidationState &state) {
     // Update cached incremental witnesses
     //fprintf(stderr,"chaintip false\n");
     GetMainSignals().ChainTip(pindexDelete, &block, newTree, false);
+
+    /* if chain tip disconnects, some transactions may return to the replacementPool
+     * via AcceptToMemoryPool.
+     *
+     * No double send conflicts may result as the winning transaction will be picked.
+     */
+
     return true;
 }
 
@@ -2761,6 +2812,8 @@ bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock *
     mempool.check(pcoinsTip);
     // Update chainActive & related variables.
     UpdateTip(pindexNew);
+    // Process pending replacements
+    ProcessReplacementPool(pindexNew->nHeight-1);
     // Tell wallet about transactions that went from mempool
     // to conflicted:
     BOOST_FOREACH(const CTransaction &tx, txConflicted) {
@@ -2770,6 +2823,7 @@ bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock *
     BOOST_FOREACH(const CTransaction &tx, pblock->vtx) {
         SyncWithWallets(tx, pblock);
     }
+
     // Update cached incremental witnesses
     //fprintf(stderr,"chaintip true\n");
     GetMainSignals().ChainTip(pindexNew, pblock, oldTree, true);

src/main.h

@@ -250,7 +250,7 @@ void PruneAndFlush();
 
 /** (try to) add transaction to memory pool **/
 bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree,
-                        bool* pfMissingInputs, bool fRejectAbsurdFee=false);
+                        bool* pfMissingInputs, bool fRejectAbsurdFee=false, bool fAcceptReplacement=true);
 
 
 struct CNodeStateStats {

src/replacementpool.cpp

@@ -0,0 +1,96 @@
+
+#include <map>
+#include <string>
+#include <iterator>
+
+#include "main.h"
+#include "coins.h"
+#include "replacementpool.h"
+
+
+CTxReplacementPool replacementPool;
+
+
+/*
+ * Add a transaction to the pool, with a priority.
+ * Return true if valid, false if not valid. */
+bool ValidateReplacementPoolItem(CTxReplacementPoolItem item)
+{
+    // Perform some validations.
+    if (item.tx.vin.size() > 1) {
+        // Replaceable transactions with multiple inputs are disabled for now. It's not yet clear
+        // what edge cases may arise. It is speculated that it will "just work", since if
+        // replaceable transactions spend multiple outputs using the replacement protocol,
+        // they will never conflict in the replaceMap data structure. But for now, to be prudent, disable.
+        return false;
+    }
+
+    // A transaction with 0 priority is not valid.
+    if (item.priority == 0) {
+        return false;
+    }
+
+    return true;
+}
+
+
+/*
+ * Validate the item
+ * 
+ * Compare the item with any current replacement candidate
+ *
+ * Ensure that the item is not passed the replacement window
+ *
+ * Insert the item into the map
+ */
+CTxReplacementPoolResult CTxReplacementPool::replace(CTxReplacementPoolItem &item)
+{
+    if (!ValidateReplacementPoolItem(item)) {
+        return RP_Invalid;
+    }
+
+    auto it = replaceMap.find(item.tx.vin[0].prevout);
+
+    if (it != replaceMap.end()) {
+        if (it->second.priority >= item.priority) {
+            // Already have a transaction with equal or greater priority; this is not valid
+            return RP_Invalid;
+        }
+        // copy the previous starting block over
+        item.startBlock = it->second.startBlock;
+    }
+
+    // This transaction has higher priority
+    replaceMap[item.tx.vin[0].prevout] = item;
+
+    return RP_Valid;
+}
+
+
+void CTxReplacementPool::removePending(int height, std::vector<CTransaction> &txs)
+{
+    for (auto it = replaceMap.begin(); it != replaceMap.end(); /**/) {
+        CTxReplacementPoolItem &rep = it->second;
+        if (rep.GetTargetBlock() <= height) {
+            txs.push_back(rep.tx);
+            replaceMap.erase(it++);
+        } else {
+            ++it;
+        }
+    }
+}
+
+
+/*
+ * O(n) lookup of tx by hash
+ */
+bool CTxReplacementPool::lookup(uint256 txHash, CTransaction &tx)
+{
+    for (auto it = replaceMap.begin(); it != replaceMap.end(); it++) {
+        if (it->second.tx.GetHash() == txHash) {
+            tx = it->second.tx;
+            return true;
+        }
+    }
+    return false;
+}

src/replacementpool.h

@@ -0,0 +1,68 @@
+// Copyright (c) 2018 The Komodo Developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef KOMODO_REPLACEMENTCACHE_H
+#define KOMODO_REPLACEMENTCACHE_H
+
+#include "coins.h"
+#include "primitives/transaction.h"
+
+
+enum CTxReplacementPoolResult { RP_NotReplace, RP_Valid, RP_Invalid };
+
+class CTxReplacementPoolItem
+{
+public:
+    CTransaction tx;
+    int startBlock;
+    uint64_t priority;
+    uint32_t replacementWindow;
+
+    CTxReplacementPoolItem() {}
+
+    CTxReplacementPoolItem(const CTransaction &_tx, int _startBlock) {
+        tx = _tx;
+        startBlock = _startBlock;
+        priority = 0;
+        replacementWindow = 0;
+    }
+
+    int GetTargetBlock()
+    {
+        return startBlock + replacementWindow;
+    }
+};
+
+/**
+ * CTxReplacementPool stores valid-according-to-the-current-best-chain (??? do we need to do this?)
+ * transactions that are valid but held for period of time during which
+ * they may be replaced.
+ *
+ * Transactions are added when they are found to be valid but not added
+ * to the mempool until a timeout.
+ *
+ * Replacement pool is like another mempool before the main mempool.
+ */
+class CTxReplacementPool
+{
+private:
+    // A potential replacement is first stored here, not in the replaceMap.
+    // This is in case some other checks fail, during AcceptToMemoryPool.
+    // Later on, if all checks pass, processReplacement() is called.
+    
+    /* Index of spends that may be replaced */
+    std::map<COutPoint, CTxReplacementPoolItem> replaceMap;
+public:
+    CTxReplacementPoolResult replace(CTxReplacementPoolItem &item);
+
+    // Remove and return all transactions up to a given block height.
+    void removePending(int height, std::vector<CTransaction> &txs);
+
+    bool lookup(uint256 txHash, CTransaction &tx);
+};
+
+
+extern CTxReplacementPool replacementPool;
+
+#endif // KOMODO_REPLACEMENTCACHE_H

src/script/interpreter.cpp

@@ -1154,18 +1154,18 @@ bool TransactionSignatureChecker::CheckSig(const vector<unsigned char>& vchSigIn
 
 extern "C"
 {
-    static int komodoCCAux(CC *cond, void *transactionSignatureChecker);
+    static int komodoCCEval(CC *cond, void *transactionSignatureChecker);
 }
 
-static int komodoCCAux(CC *cond, void *checker) {
-    return ((TransactionSignatureChecker*)checker)->CheckAuxCondition(cond);
+static int komodoCCEval(CC *cond, void *checker) {
+    return ((TransactionSignatureChecker*)checker)->CheckEvalCondition(cond);
 }
 
 bool TransactionSignatureChecker::CheckCryptoCondition(const CC *cond, const std::vector<unsigned char>& condBin, const CScript& scriptCode) const
 {
     uint256 message = SignatureHash(scriptCode, *txTo, nIn, SIGHASH_ALL);
     return cc_verify(cond, (const unsigned char*)&message, 32, 0,
-            condBin.data(), condBin.size(), komodoCCAux, (void*)this);
+            condBin.data(), condBin.size(), komodoCCEval, (void*)this);
 }
 
 bool TransactionSignatureChecker::CheckLockTime(const CScriptNum& nLockTime) const

src/script/interpreter.h

@@ -125,7 +125,7 @@ public:
     bool CheckSig(const std::vector<unsigned char>& scriptSig, const std::vector<unsigned char>& vchPubKey, const CScript& scriptCode) const;
     bool CheckLockTime(const CScriptNum& nLockTime) const;
     bool CheckCryptoCondition(const CC *cond, const std::vector<unsigned char>& condBin, const CScript& scriptCode) const;
-    int CheckAuxCondition(const CC *cond) const;
+    bool CheckEvalCondition(const CC *cond) const;
 };
 
 class MutableTransactionSignatureChecker : public TransactionSignatureChecker