diff --git a/src/main.cpp b/src/main.cpp
index 88686df6d..94fa6aeca 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -4657,6 +4657,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
         bool fMissingInputs = false;
         CValidationState state;
 
+        pfrom->setAskFor.erase(inv.hash);
         mapAlreadyAskedFor.erase(inv);
 
         // Check for recently rejected (and do other quick existence checks)
@@ -5451,8 +5452,10 @@ bool SendMessages(CNode* pto, bool fSendTrickle)
                     pto->PushMessage("getdata", vGetData);
                     vGetData.clear();
                 }
+            } else {
+                //If we're not going to ask, don't expect a response.
+                pto->setAskFor.erase(inv.hash);
             }
-            pto->setAskFor.erase(inv.hash);
             pto->mapAskFor.erase(pto->mapAskFor.begin());
         }
         if (!vGetData.empty())
diff --git a/src/net.cpp b/src/net.cpp
index 710e61678..636e3d5e7 100644
--- a/src/net.cpp
+++ b/src/net.cpp
@@ -2178,9 +2178,9 @@ CNode::~CNode()
 
 void CNode::AskFor(const CInv& inv)
 {
-    if (mapAskFor.size() > MAPASKFOR_MAX_SZ)
+    if (mapAskFor.size() > MAPASKFOR_MAX_SZ || setAskFor.size() > SETASKFOR_MAX_SZ)
         return;
-    // a peer may not occupy multiple positions in an inv's request queue
+    // a peer may not have multiple non-responded queue positions for a single inv item
     if (!setAskFor.insert(inv.hash).second)
         return;
 
diff --git a/src/net.h b/src/net.h
index bc6ab291a..44f14fd43 100644
--- a/src/net.h
+++ b/src/net.h
@@ -59,6 +59,8 @@ static const bool DEFAULT_UPNP = false;
 #endif
 /** The maximum number of entries in mapAskFor */
 static const size_t MAPASKFOR_MAX_SZ = MAX_INV_SZ;
+/** The maximum number of entries in setAskFor (larger due to getdata latency)*/
+static const size_t SETASKFOR_MAX_SZ = 2 * MAX_INV_SZ;
 
 unsigned int ReceiveFloodSize();
 unsigned int SendBufferSize();