changeset 23694:a78a20598afd

Xpress Validation Building on Xtreme Thinblocks, Xpress Validation allows for a much faster and more scalable approach to validating a block. Becuase most transactions have already been validated when entering the memory pool we only have to validate new trasactions arriving in the XThinblock before re-assembling the block.
author Peter Tschipper <peter.tschipper@gmailcom>
date Fri, 15 Jan 2016 21:37:25 -0800
parents 483574c6c912
children c5dff189e132
files src/main.cpp src/main.h src/unlimited.cpp
diffstat 3 files changed, 53 insertions(+), 18 deletions(-) [+]
line wrap: on
line diff
--- a/src/main.cpp	Thu Feb 18 10:05:23 2016 -0500
+++ b/src/main.cpp	Fri Jan 15 21:37:25 2016 -0800
@@ -2072,6 +2072,7 @@
     std::vector<std::pair<uint256, CDiskTxPos> > vPos;
     vPos.reserve(block.vtx.size());
     blockundo.vtxundo.reserve(block.vtx.size() - 1);
+    int nChecked = 0;      
     for (unsigned int i = 0; i < block.vtx.size(); i++)
     {
         const CTransaction &tx = block.vtx[i];
@@ -2103,9 +2104,17 @@
 
             std::vector<CScriptCheck> vChecks;
             bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */
-            if (!CheckInputs(tx, state, view, fScriptChecks, flags, fCacheResults, nScriptCheckThreads ? &vChecks : NULL))
-                return error("ConnectBlock(): CheckInputs on %s failed with %s",
-                    tx.GetHash().ToString(), FormatStateMessage(state));
+            // Only check inputs when the tx hash in not in the setPreVerifiedTxHash as would only
+            // happen if this were a regular block or when a tx is found w?ithin the returning XThinblock.
+            uint256 hash = tx.GetHash(); 
+            if (!setPreVerifiedTxHash.count(hash)) {          
+                nChecked++;
+                if (!CheckInputs(tx, state, view, fScriptChecks, flags, fCacheResults, nScriptCheckThreads ? &vChecks : NULL))
+                    return error("ConnectBlock(): CheckInputs on %s failed with %s",
+                        tx.GetHash().ToString(), FormatStateMessage(state));
+            }
+            else
+                setPreVerifiedTxHash.erase(hash);
             control.Add(vChecks);
         }
 
@@ -2118,6 +2127,8 @@
         vPos.push_back(std::make_pair(tx.GetHash(), pos));
         pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION);
     }
+    LogPrint("thin", "Number of CheckInputs() performed is %d\n", nChecked);
+
     int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2;
     LogPrint("bench", "      - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * 0.000001);
 
@@ -5077,19 +5088,19 @@
             const uint256 hash = mapPartialTxHash[cheapHash];
             CTransaction tx;
             if (!hash.IsNull())
-              {
+            {
                 bool inMemPool = mempool.lookup(hash, tx);
                 bool inMissingTx = thinBlock.mapMissingTx.count(hash) > 0;
-                if (inMemPool && inMissingTx) {
-                  unnecessaryCount++;
-                }
-                if (!inMemPool) {
-                  if (inMissingTx)
+                if (inMemPool && inMissingTx)
+                    unnecessaryCount++;
+
+                if (inMemPool)
+                    setPreVerifiedTxHash.insert(hash);
+                else if (inMissingTx)
                     tx = thinBlock.mapMissingTx[hash];
-                }
-              }
+            }
             if (tx.IsNull())
-                missingCount++;             
+                missingCount++;
             // This will push an empty/invalid transaction if we don't have it yet
             pfrom->thinBlock.vtx.push_back(tx);
         }
@@ -5152,13 +5163,22 @@
         pfrom->thinBlockHashes = thinBlock.vTxHashes;
 
         int missingCount = 0;
+        int unnecessaryCount = 0;
         LOCK(cs_main);
         // Look for each transaction in our various pools and buffers.
         BOOST_FOREACH(const uint256 &hash, thinBlock.vTxHashes) 
         {
             CTransaction tx;
-            if (!mempool.lookup(hash, tx)) {
-                if (thinBlock.mapMissingTx.count(hash))
+            if (!hash.IsNull())
+            {
+                bool inMemPool = mempool.lookup(hash, tx);
+                bool inMissingTx = thinBlock.mapMissingTx.count(hash) > 0;
+                if (inMemPool && inMissingTx)
+                    unnecessaryCount++;
+
+                if (inMemPool)
+                    setPreVerifiedTxHash.insert(hash);
+                else if (inMissingTx)
                     tx = thinBlock.mapMissingTx[hash];
             }
             if (tx.IsNull())
@@ -5167,6 +5187,7 @@
             pfrom->thinBlock.vtx.push_back(tx);
         }
         pfrom->thinBlockWaitingForTxns = missingCount;
+        LogPrint("thin", "thinblock waiting for: %d, unnecessary: %d, txs: %d full: %d\n", pfrom->thinBlockWaitingForTxns, unnecessaryCount, pfrom->thinBlock.vtx.size(), thinBlock.mapMissingTx.size());
 
         if (pfrom->thinBlockWaitingForTxns == 0) {
             // We have all the transactions now that are in this block: try to reassemble and process.
@@ -5187,6 +5208,7 @@
             vector<CInv> vGetData;
             vGetData.push_back(CInv(MSG_BLOCK, thinBlock.header.GetHash())); 
             pfrom->PushMessage("getdata", vGetData);
+            setPreVerifiedTxHash.clear(); // Xpress Validation - clear the set since we do not do XVal on regular blocks
             LogPrint("thin", "Missing %d Thinblock transactions, re-requesting a regular block\n",  
                        pfrom->thinBlockWaitingForTxns);
         }
--- a/src/main.h	Thu Feb 18 10:05:23 2016 -0500
+++ b/src/main.h	Fri Jan 15 21:37:25 2016 -0800
@@ -145,6 +145,15 @@
 extern bool fAlerts;
 extern bool fEnableReplacement;  // BU TODO is this RBF flag?
 
+// Xpress Validation: begin section
+/**
+ * Transactions that have already been accepted into the memory pool do not need to be
+ * re-verified and can avoid having to do a second and expensive CheckInputs() when 
+ * processing a new block.  (Protected by cs_main)
+ */
+static std::set<uint256> setPreVerifiedTxHash;
+// Xpress Validation: end section
+
 /** Best header we've seen so far (used for getheaders queries' starting points). */
 extern CBlockIndex *pindexBestHeader;
 
--- a/src/unlimited.cpp	Thu Feb 18 10:05:23 2016 -0500
+++ b/src/unlimited.cpp	Fri Jan 15 21:37:25 2016 -0800
@@ -547,15 +547,22 @@
     // the block didn't arrive from some other peer.  This code ALSO cleans up the thin block that
     // was passed to us (&block), so do not use it after this.
     {
+        int nTotalThinBlocksInFlight = 0;
         LOCK(cs_vNodes);
         BOOST_FOREACH(CNode* pnode, vNodes) {
             if (pnode->mapThinBlocksInFlight.count(inv.hash)) {
                 pnode->mapThinBlocksInFlight.erase(inv.hash); 
                 pnode->thinBlockWaitingForTxns = -1;
                 pnode->thinBlock.SetNull();
-                if (pnode != pfrom) LogPrintf("Removing thinblock in flight %s from %s (%d)\n",inv.hash.ToString(), pnode->addrName.c_str(), pnode->id);
             }
+            if (pnode->mapThinBlocksInFlight.size() > 0)
+                nTotalThinBlocksInFlight++;
         }
+
+        // When we no longer have any thinblocks in flight then clear the set
+        // just to make sure we don't somehow get growth over time.
+        if (nTotalThinBlocksInFlight == 0)
+            setPreVerifiedTxHash.clear();
     }
 
     // Clear the thinblock timer used for preferential download
@@ -668,7 +675,4 @@
             LogPrint("thin", "Sent regular block instead - thinblock size: %d vs block size: %d => tx hashes: %d transactions: %d  peerid=%d\n", nSizeThinBlock, nSizeBlock, thinBlock.vTxHashes.size(), thinBlock.mapMissingTx.size(), pfrom->id);
         }
     }
-
 }
-
-