diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp
index 3bc3b99cd..233851d64 100644
--- a/src/wallet/wallet.cpp
+++ b/src/wallet/wallet.cpp
@@ -2717,10 +2717,12 @@ int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate)
                 ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), pindex, false) - dProgressStart) / (dProgressTip - dProgressStart) * 100))));
 
             CBlock block;
+            bool involvesMe = false;
             ReadBlockFromDisk(block, pindex,1);
             BOOST_FOREACH(CTransaction& tx, block.vtx)
             {
                 if (AddToWalletIfInvolvingMe(tx, &block, fUpdate)) {
+                    involvesMe = true;
                     ret++;
                 }
             }
@@ -2734,8 +2736,11 @@ int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate)
                 }
             }
 
-            // Build initial witness caches
-            BuildWitnessCache(pindex, true);
+            // Build initial witness caches for blocks involving one of our addresses
+            if (involvesMe) {
+                LogPrintf("%s: block has one of our transactions, building witness cache", __func__);
+                BuildWitnessCache(pindex, true);
+            }
 
             //Delete Transactions
             if (fTxDeleteEnabled) {