changeset 23719:8a79be96660f

Xtreme Thinblocks and Xpress Validation using Orphan Cache Many tx's now are found in the orphan cache for which there is no parent tx. When a thinblock is retrieved we have to add the tx hashes to the bloom filter and also check the orphan cache on return to see if there are any valid tx's to add to the thinblock. Furthermore we have to make sure we do not invoke Xpress Validation for these orphan tx's as they have never been fully verified and we must do so before accepting the block.
author Peter Tschipper <peter.tschipper@gmailcom>
date Sat, 05 Mar 2016 11:08:23 -0800
parents ca7ade6993ea
children 8e0bb7bb88b6
files src/main.cpp src/main.h src/unlimited.cpp src/unlimited.h
diffstat 4 files changed, 83 insertions(+), 23 deletions(-) [+]
line wrap: on
line diff
--- a/src/main.cpp	Fri Mar 04 20:30:37 2016 -0800
+++ b/src/main.cpp	Sat Mar 05 11:08:23 2016 -0800
@@ -609,12 +609,18 @@
     // have been mined or received.
     // 10,000 orphans, each of which is at most 5,000 bytes big is
     // at most 500 megabytes of orphans:
-    unsigned int sz = tx.GetSerializeSize(SER_NETWORK, CTransaction::CURRENT_VERSION);
-    if (sz > 5000)
-    {
-        LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString());
-        return false;
-    }
+
+    // BU - Xtreme Thinblocks - begin section
+    // BU - we do not limit the size of orphans.  There is no danger to having memory overrun since the
+    //      orphan cache is limited to only 5000 entries by default. Only 500MB of memory could be consumed
+    //      if there were some kind of orphan memory exhaustion attack.
+    //unsigned int sz = tx.GetSerializeSize(SER_NETWORK, CTransaction::CURRENT_VERSION);
+    //if (sz > 5000)
+    //{
+    //    LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString());
+    //    return false;
+    //}
+    // BU - Xtreme Thinblocks - end section
 
     mapOrphanTransactions[hash].tx = tx;
     mapOrphanTransactions[hash].fromPeer = peer;
@@ -2126,7 +2132,8 @@
     std::vector<std::pair<uint256, CDiskTxPos> > vPos;
     vPos.reserve(block.vtx.size());
     blockundo.vtxundo.reserve(block.vtx.size() - 1);
-    int nChecked = 0;      
+    int nChecked = 0;
+    int nOrphansChecked = 0;     
     for (unsigned int i = 0; i < block.vtx.size(); i++)
     {
         const CTransaction &tx = block.vtx[i];
@@ -2160,15 +2167,20 @@
             bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */
             // Only check inputs when the tx hash in not in the setPreVerifiedTxHash as would only
             // happen if this were a regular block or when a tx is found w?ithin the returning XThinblock.
-            uint256 hash = tx.GetHash(); 
-            if (!setPreVerifiedTxHash.count(hash)) {          
+            uint256 hash = tx.GetHash();
+            bool inOrphanCache = setUnVerifiedOrphanTxHash.count(hash);
+            if ((inOrphanCache) || (!setPreVerifiedTxHash.count(hash) && !inOrphanCache)) {          
                 nChecked++;
+                if (inOrphanCache)
+                    nOrphansChecked++;
                 if (!CheckInputs(tx, state, view, fScriptChecks, flags, fCacheResults, nScriptCheckThreads ? &vChecks : NULL))
                     return error("ConnectBlock(): CheckInputs on %s failed with %s",
                         tx.GetHash().ToString(), FormatStateMessage(state));
             }
-            else
+            else {
                 setPreVerifiedTxHash.erase(hash);
+                setUnVerifiedOrphanTxHash.erase(hash);
+            }
             control.Add(vChecks);
         }
 
@@ -2181,7 +2193,7 @@
         vPos.push_back(std::make_pair(tx.GetHash(), pos));
         pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION);
     }
-    LogPrint("thin", "Number of CheckInputs() performed is %d\n", nChecked);
+    LogPrint("thin", "Number of CheckInputs() performed: %d  Orphan count: %d\n", nChecked, nOrphansChecked);
 
     int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2;
     LogPrint("bench", "      - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * 0.000001);
@@ -4669,7 +4681,10 @@
                                 if (pfrom->mapThinBlocksInFlight.size() < 1 && pfrom->nVersion >= THINBLOCKS_VERSION) { // We can only send one thinblock per peer at a time
                                     pfrom->mapThinBlocksInFlight[inv2.hash] = GetTime();
                                     inv2.type = MSG_XTHINBLOCK;
-                                    BuildSeededBloomFilter(filterMemPool);
+                                    std::vector<uint256> vOrphanHashes;
+                                    for (map<uint256, COrphanTx>::iterator mi = mapOrphanTransactions.begin(); mi != mapOrphanTransactions.end(); ++mi)
+                                        vOrphanHashes.push_back((*mi).first);
+                                    BuildSeededBloomFilter(filterMemPool, vOrphanHashes);
                                     ss << inv2;
                                     ss << filterMemPool;
                                     pfrom->PushMessage(NetMsgType::GET_XTHIN, ss);
@@ -4682,7 +4697,10 @@
                                 if (pfrom->mapThinBlocksInFlight.size() < 1 && pfrom->nVersion >= THINBLOCKS_VERSION) { // We can only send one thinblock per peer at a time
                                     pfrom->mapThinBlocksInFlight[inv2.hash] = GetTime();
                                     inv2.type = MSG_XTHINBLOCK;
-                                    BuildSeededBloomFilter(filterMemPool);
+                                    std::vector<uint256> vOrphanHashes;
+                                    for (map<uint256, COrphanTx>::iterator mi = mapOrphanTransactions.begin(); mi != mapOrphanTransactions.end(); ++mi)
+                                        vOrphanHashes.push_back((*mi).first);
+                                    BuildSeededBloomFilter(filterMemPool, vOrphanHashes);
                                     ss << inv2;
                                     ss << filterMemPool;
                                     pfrom->PushMessage(NetMsgType::GET_XTHIN, ss);
@@ -5134,6 +5152,12 @@
                 collision = true;
             mapPartialTxHash[cheapHash] = (*mi).first;
         }
+        for (map<uint256, COrphanTx>::iterator mi = mapOrphanTransactions.begin(); mi != mapOrphanTransactions.end(); ++mi) {
+            uint64_t cheapHash = (*mi).first.GetCheapHash();
+            if(mapPartialTxHash.count(cheapHash)) //Check for collisions
+                collision = true;
+            mapPartialTxHash[cheapHash] = (*mi).first;
+        }
 
         // There is a remote possiblity of a Tx hash collision therefore if it occurs we re-request a normal
         // thinblock which has the full Tx hash data rather than just the truncated hash.
@@ -5161,10 +5185,16 @@
             {
                 bool inMemPool = mempool.lookup(hash, tx);
                 bool inMissingTx = thinBlock.mapMissingTx.count(hash) > 0;
-                if (inMemPool && inMissingTx)
+                bool inOrphanCache = mapOrphanTransactions.count(hash) > 0;
+
+                if ((inMemPool && inMissingTx) || (inOrphanCache && inMissingTx))
                     unnecessaryCount++;
 
-                if (inMemPool && fXVal)
+                if (inOrphanCache) {
+                    tx = mapOrphanTransactions[hash].tx;
+                    setUnVerifiedOrphanTxHash.insert(hash);
+                }
+                else if (inMemPool && fXVal)
                     setPreVerifiedTxHash.insert(hash);
                 else if (inMissingTx)
                     tx = thinBlock.mapMissingTx[hash];
@@ -5189,6 +5219,8 @@
                      ((float) blockSize) / ((float) nSizeThinBlock)
                      );
             HandleBlockMessage(pfrom, strCommand, pfrom->thinBlock, inv);  // clears the thin block
+            BOOST_FOREACH(uint64_t &cheapHash, thinBlock.vTxHashes)
+                EraseOrphanTx(mapPartialTxHash[cheapHash]);
         }
         else if (pfrom->thinBlockWaitingForTxns > 0) {
             // This marks the end of the transactions we've received. If we get this and we have NOT been able to
@@ -5246,10 +5278,16 @@
             {
                 bool inMemPool = mempool.lookup(hash, tx);
                 bool inMissingTx = thinBlock.mapMissingTx.count(hash) > 0;
-                if (inMemPool && inMissingTx)
+                bool inOrphanCache = mapOrphanTransactions.count(hash) > 0;
+
+                if ((inMemPool && inMissingTx) || (inOrphanCache && inMissingTx))
                     unnecessaryCount++;
 
-                if (inMemPool && fXVal)
+                if (inOrphanCache) {
+                    tx = mapOrphanTransactions[hash].tx;
+                    setUnVerifiedOrphanTxHash.insert(hash);
+                }
+                else if (inMemPool && fXVal)
                     setPreVerifiedTxHash.insert(hash);
                 else if (inMissingTx)
                     tx = thinBlock.mapMissingTx[hash];
@@ -5274,6 +5312,8 @@
                      ((float) blockSize) / ((float) nSizeThinBlock)
                      );
             HandleBlockMessage(pfrom, strCommand, pfrom->thinBlock, inv);
+            BOOST_FOREACH(uint256 &hash, thinBlock.vTxHashes)
+                EraseOrphanTx(hash);
         }
         else if (pfrom->thinBlockWaitingForTxns > 0) {
             // This marks the end of the transactions we've received. If we get this and we have NOT been able to
@@ -5320,7 +5360,10 @@
                      pfrom->nSizeThinBlock,
                      ((float) blockSize) / ((float) pfrom->nSizeThinBlock)
                      );
+            std::vector<CTransaction> vTx = pfrom->thinBlock.vtx;
             HandleBlockMessage(pfrom, strCommand, pfrom->thinBlock, inv);
+            for (unsigned int i = 0; i < vTx.size(); i++)
+                EraseOrphanTx(vTx[i].GetHash());
         }
         else {
             LogPrint("thin", "Failed to retrieve all transactions for block - DOS Banned\n");
@@ -5391,6 +5434,8 @@
 
         // BUIP010 Extreme Thinblocks: Handle Block Message
         HandleBlockMessage(pfrom, strCommand, block, inv);
+        for (unsigned int i = 0; i < block.vtx.size(); i++)
+            EraseOrphanTx(block.vtx[i].GetHash());
     }
 
 
@@ -6085,7 +6130,10 @@
                         // Must download a block from a ThinBlock peer
                         if (pto->mapThinBlocksInFlight.size() < 1 && pto->nVersion >= THINBLOCKS_VERSION) { // We can only send one thinblock per peer at a time
                             pto->mapThinBlocksInFlight[pindex->GetBlockHash()] = GetTime();
-                            BuildSeededBloomFilter(filterMemPool);
+                            std::vector<uint256> vOrphanHashes;
+                            for (map<uint256, COrphanTx>::iterator mi = mapOrphanTransactions.begin(); mi != mapOrphanTransactions.end(); ++mi)
+                                vOrphanHashes.push_back((*mi).first);
+                            BuildSeededBloomFilter(filterMemPool, vOrphanHashes);
                             ss << CInv(MSG_XTHINBLOCK, pindex->GetBlockHash());
                             ss << filterMemPool;
                             pto->PushMessage(NetMsgType::GET_XTHIN, ss);
@@ -6098,7 +6146,10 @@
                         // Try to download a thinblock if possible otherwise just download a regular block
                         if (pto->mapThinBlocksInFlight.size() < 1 && pto->nVersion >= THINBLOCKS_VERSION) { // We can only send one thinblock per peer at a time
                             pto->mapThinBlocksInFlight[pindex->GetBlockHash()] = GetTime();
-                            BuildSeededBloomFilter(filterMemPool);
+                            std::vector<uint256> vOrphanHashes;
+                            for (map<uint256, COrphanTx>::iterator mi = mapOrphanTransactions.begin(); mi != mapOrphanTransactions.end(); ++mi)
+                                vOrphanHashes.push_back((*mi).first);
+                            BuildSeededBloomFilter(filterMemPool, vOrphanHashes);
                             ss << CInv(MSG_XTHINBLOCK, pindex->GetBlockHash());
                             ss << filterMemPool;
                             pto->PushMessage(NetMsgType::GET_XTHIN, ss);
--- a/src/main.h	Fri Mar 04 20:30:37 2016 -0800
+++ b/src/main.h	Sat Mar 05 11:08:23 2016 -0800
@@ -155,6 +155,11 @@
  * processing a new block.  (Protected by cs_main)
  */
 static std::set<uint256> setPreVerifiedTxHash;
+/**
+ * Orphans that are added to the thinblock must be verifed since they have never been
+ *  accepted into the memory pool.
+ */
+static std::set<uint256> setUnVerifiedOrphanTxHash;
 // BU - Xpress Validation: end section
 
 // BU - Xtreme Thinblocks Auto Mempool Limiter - begin section
--- a/src/unlimited.cpp	Fri Mar 04 20:30:37 2016 -0800
+++ b/src/unlimited.cpp	Sat Mar 05 11:08:23 2016 -0800
@@ -521,7 +521,7 @@
     return true;
 }
 
-void BuildSeededBloomFilter(CBloomFilter& filterMemPool)
+void BuildSeededBloomFilter(CBloomFilter& filterMemPool, std::vector<uint256> vOrphanHashes)
 {
     LogPrint("thin", "Starting creation of bloom filter\n");
     seed_insecure_rand();
@@ -529,7 +529,7 @@
     if (nBloomPoolSize > MAX_BLOOM_FILTER_SIZE / 1.8)
         nBloomPoolSize = MAX_BLOOM_FILTER_SIZE / 1.8;
     double nBloomDecay = 1.5 - (nBloomPoolSize * 1.8 / MAX_BLOOM_FILTER_SIZE);  // We should never go below 0.5 as we will start seeing re-requests for tx's
-    int nElements = std::max((int)((int)mempool.mapTx.size() * nBloomDecay), 1); // Must make sure nElements is greater than zero or will assert
+    int nElements = std::max((int)(((int)mempool.mapTx.size() + (int)vOrphanHashes.size()) * nBloomDecay), 1); // Must make sure nElements is greater than zero or will assert
     double nFPRate = .001 + (((double)nElements * 1.8 / MAX_BLOOM_FILTER_SIZE) * .004); // The false positive rate in percent decays as the mempool grows
     filterMemPool = CBloomFilter(nElements, nFPRate, insecure_rand(), BLOOM_UPDATE_ALL);
     LogPrint("thin", "Bloom multiplier: %f FPrate: %f Num elements in bloom filter: %d num mempool entries: %d\n", nBloomDecay, nFPRate, nElements, (int)mempool.mapTx.size());
@@ -540,6 +540,8 @@
     mempool.queryHashes(vMemPoolHashes);
     for (uint64_t i = 0; i < vMemPoolHashes.size(); i++)
          filterMemPool.insert(vMemPoolHashes[i]);
+    for (uint64_t i = 0; i < vOrphanHashes.size(); i++)
+         filterMemPool.insert(vOrphanHashes[i]);
     LogPrint("thin", "Created bloom filter: %d bytes\n",::GetSerializeSize(filterMemPool, SER_NETWORK, PROTOCOL_VERSION));
 }
 
@@ -605,8 +607,10 @@
 
         // When we no longer have any thinblocks in flight then clear the set
         // just to make sure we don't somehow get growth over time.
-        if (nTotalThinBlocksInFlight == 0)
+        if (nTotalThinBlocksInFlight == 0) {
             setPreVerifiedTxHash.clear();
+            setUnVerifiedOrphanTxHash.clear();
+        }
     }
 
     // Clear the thinblock timer used for preferential download
--- a/src/unlimited.h	Fri Mar 04 20:30:37 2016 -0800
+++ b/src/unlimited.h	Sat Mar 05 11:08:23 2016 -0800
@@ -71,7 +71,7 @@
 extern void ClearThinblockTimer(uint256 hash);
 extern bool IsThinBlocksEnabled();
 extern bool IsChainNearlySyncd();
-extern void BuildSeededBloomFilter(CBloomFilter& memPoolFilter);
+extern void BuildSeededBloomFilter(CBloomFilter& memPoolFilter, std::vector<uint256> vOrphanHashes);
 extern void LoadFilter(CNode *pfrom, CBloomFilter *filter);
 extern void HandleBlockMessage(CNode *pfrom, const std::string &strCommand, CBlock &block, const CInv &inv);
 extern void ConnectToThinBlockNodes();