Merge pull request #6777

dcd8e27 Refer to obfuscate_key via pointer in peripheral CLevelDB classes (James O'Beirne)
1488506 Add tests for gettxoutsetinfo, CLevelDBBatch, CLevelDBIterator (James O'Beirne)
0fdf8c8 Handle obfuscation in CLevelDBIterator (James O'Beirne)
3499ce1 Encapsulate CLevelDB iterators cleanly (Pieter Wuille)

qa/pull-tester/rpc-tests.py

@@ -67,6 +67,7 @@ testScripts = [
     'reindex.py',
     'decodescript.py',
     'p2p-fullblocktest.py',
+    'blockchain.py',
 ]
 testScriptsExt = [
     'bipdersig-p2p.py',

qa/rpc-tests/blockchain.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python2
+# Copyright (c) 2014 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Test RPC calls related to blockchain state.
+#
+
+import decimal
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    initialize_chain,
+    assert_equal,
+    start_nodes,
+    connect_nodes_bi,
+)
+
+class BlockchainTest(BitcoinTestFramework):
+    """
+    Test blockchain-related RPC calls:
+
+        - gettxoutsetinfo
+
+    """
+
+    def setup_chain(self):
+        print("Initializing test directory " + self.options.tmpdir)
+        initialize_chain(self.options.tmpdir)
+
+    def setup_network(self, split=False):
+        self.nodes = start_nodes(2, self.options.tmpdir)
+        connect_nodes_bi(self.nodes, 0, 1)
+        self.is_network_split = False
+        self.sync_all()
+
+    def run_test(self):
+        node = self.nodes[0]
+        res = node.gettxoutsetinfo()
+
+        assert_equal(res[u'total_amount'], decimal.Decimal('8725.00000000'))
+        assert_equal(res[u'transactions'], 200)
+        assert_equal(res[u'height'], 200)
+        assert_equal(res[u'txouts'], 200)
+        assert_equal(res[u'bytes_serialized'], 13000),
+        assert_equal(len(res[u'bestblock']), 64)
+        assert_equal(len(res[u'hash_serialized']), 64)
+
+
+if __name__ == '__main__':
+    BlockchainTest().main()

src/leveldbwrapper.cpp

@@ -131,7 +131,7 @@ std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const
 
 bool CLevelDBWrapper::IsEmpty()
 {
-    boost::scoped_ptr<leveldb::Iterator> it(NewIterator());
+    boost::scoped_ptr<CLevelDBIterator> it(NewIterator());
     it->SeekToFirst();
     return !(it->Valid());
 }
@@ -145,3 +145,10 @@ std::string CLevelDBWrapper::GetObfuscateKeyHex() const
 { 
     return HexStr(obfuscate_key); 
 }
+
+CLevelDBIterator::~CLevelDBIterator() { delete piter; }
+bool CLevelDBIterator::Valid() { return piter->Valid(); }
+void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); }
+void CLevelDBIterator::SeekToLast() { piter->SeekToLast(); }
+void CLevelDBIterator::Next() { piter->Next(); }
+void CLevelDBIterator::Prev() { piter->Prev(); }

src/leveldbwrapper.h

@@ -32,13 +32,13 @@ class CLevelDBBatch
 
 private:
     leveldb::WriteBatch batch;
-    const std::vector<unsigned char> obfuscate_key;
+    const std::vector<unsigned char> *obfuscate_key;
 
 public:
     /**
      * @param[in] obfuscate_key    If passed, XOR data with this key.
      */
-    CLevelDBBatch(const std::vector<unsigned char>& obfuscate_key) : obfuscate_key(obfuscate_key) { };
+    CLevelDBBatch(const std::vector<unsigned char> *obfuscate_key) : obfuscate_key(obfuscate_key) { };
 
     template <typename K, typename V>
     void Write(const K& key, const V& value)
@@ -51,7 +51,7 @@ public:
         CDataStream ssValue(SER_DISK, CLIENT_VERSION);
         ssValue.reserve(ssValue.GetSerializeSize(value));
         ssValue << value;
-        ssValue.Xor(obfuscate_key);
+        ssValue.Xor(*obfuscate_key);
         leveldb::Slice slValue(&ssValue[0], ssValue.size());
 
         batch.Put(slKey, slValue);
@@ -69,6 +69,71 @@ public:
     }
 };
  
+class CLevelDBIterator
+{
+private:
+    leveldb::Iterator *piter;
+    const std::vector<unsigned char> *obfuscate_key;
+
+public:
+
+    /**
+     * @param[in] piterIn          The original leveldb iterator.
+     * @param[in] obfuscate_key    If passed, XOR data with this key.
+     */
+    CLevelDBIterator(leveldb::Iterator *piterIn, const std::vector<unsigned char>* obfuscate_key) :
+        piter(piterIn), obfuscate_key(obfuscate_key) { };
+    ~CLevelDBIterator();
+
+    bool Valid();
+
+    void SeekToFirst();
+    void SeekToLast();
+
+    template<typename K> void Seek(const K& key) {
+        CDataStream ssKey(SER_DISK, CLIENT_VERSION);
+        ssKey.reserve(ssKey.GetSerializeSize(key));
+        ssKey << key;
+        leveldb::Slice slKey(&ssKey[0], ssKey.size());
+        piter->Seek(slKey);
+    }
+
+    void Next();
+    void Prev();
+
+    template<typename K> bool GetKey(K& key) {
+        leveldb::Slice slKey = piter->key();
+        try {
+            CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION);
+            ssKey >> key;
+        } catch(std::exception &e) {
+            return false;
+        }
+        return true;
+    }
+
+    unsigned int GetKeySize() {
+        return piter->key().size();
+    }
+
+    template<typename V> bool GetValue(V& value) {
+        leveldb::Slice slValue = piter->value();
+        try {
+            CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION);
+            ssValue.Xor(*obfuscate_key);
+            ssValue >> value;
+        } catch(std::exception &e) {
+            return false;
+        }
+        return true;
+    }
+
+    unsigned int GetValueSize() {
+        return piter->value().size();
+    }
+
+};
+ 
 class CLevelDBWrapper
 {
 private:
@@ -145,7 +210,7 @@ public:
     template <typename K, typename V>
     bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error)
     {
-        CLevelDBBatch batch(obfuscate_key);
+        CLevelDBBatch batch(&obfuscate_key);
         batch.Write(key, value);
         return WriteBatch(batch, fSync);
     }
@@ -172,7 +237,7 @@ public:
     template <typename K>
     bool Erase(const K& key, bool fSync = false) throw(leveldb_error)
     {
-        CLevelDBBatch batch(obfuscate_key);
+        CLevelDBBatch batch(&obfuscate_key);
         batch.Erase(key);
         return WriteBatch(batch, fSync);
     }
@@ -187,14 +252,13 @@ public:
 
     bool Sync() throw(leveldb_error)
     {
-        CLevelDBBatch batch(obfuscate_key);
+        CLevelDBBatch batch(&obfuscate_key);
         return WriteBatch(batch, true);
     }
 
-    // not exactly clean encapsulation, but it's easiest for now
+    CLevelDBIterator *NewIterator() 
-    leveldb::Iterator* NewIterator()
     {
-        return pdb->NewIterator(iteroptions);
+        return new CLevelDBIterator(pdb->NewIterator(iteroptions), &obfuscate_key);
     }
 
     /**

src/test/leveldbwrapper_tests.cpp

@@ -47,6 +47,85 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper)
     }
 }
 
+// Test batch operations
+BOOST_AUTO_TEST_CASE(leveldbwrapper_batch)
+{
+    // Perform tests both obfuscated and non-obfuscated.
+    for (int i = 0; i < 2; i++) {
+        bool obfuscate = (bool)i;
+        path ph = temp_directory_path() / unique_path();
+        CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate);
+
+        char key = 'i';
+        uint256 in = GetRandHash();
+        char key2 = 'j';
+        uint256 in2 = GetRandHash();
+        char key3 = 'k';
+        uint256 in3 = GetRandHash();
+
+        uint256 res;
+        CLevelDBBatch batch(&dbw.GetObfuscateKey());
+
+        batch.Write(key, in);
+        batch.Write(key2, in2);
+        batch.Write(key3, in3);
+
+        // Remove key3 before it's even been written
+        batch.Erase(key3);
+
+        dbw.WriteBatch(batch);
+
+        BOOST_CHECK(dbw.Read(key, res));
+        BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
+        BOOST_CHECK(dbw.Read(key2, res));
+        BOOST_CHECK_EQUAL(res.ToString(), in2.ToString());
+
+        // key3 never should've been written
+        BOOST_CHECK(dbw.Read(key3, res) == false);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator)
+{
+    // Perform tests both obfuscated and non-obfuscated.
+    for (int i = 0; i < 2; i++) {
+        bool obfuscate = (bool)i;
+        path ph = temp_directory_path() / unique_path();
+        CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate);
+
+        // The two keys are intentionally chosen for ordering
+        char key = 'j';
+        uint256 in = GetRandHash();
+        BOOST_CHECK(dbw.Write(key, in));
+        char key2 = 'k';
+        uint256 in2 = GetRandHash();
+        BOOST_CHECK(dbw.Write(key2, in2));
+
+        boost::scoped_ptr<CLevelDBIterator> it(const_cast<CLevelDBWrapper*>(&dbw)->NewIterator());
+
+        // Be sure to seek past the obfuscation key (if it exists)
+        it->Seek(key);
+
+        char key_res;
+        uint256 val_res;
+
+        it->GetKey(key_res);
+        it->GetValue(val_res);
+        BOOST_CHECK_EQUAL(key_res, key);
+        BOOST_CHECK_EQUAL(val_res.ToString(), in.ToString());
+
+        it->Next();
+
+        it->GetKey(key_res);
+        it->GetValue(val_res);
+        BOOST_CHECK_EQUAL(key_res, key2);
+        BOOST_CHECK_EQUAL(val_res.ToString(), in2.ToString());
+
+        it->Next();
+        BOOST_CHECK_EQUAL(it->Valid(), false);
+    }
+}
+
 // Test that we do not obfuscation if there is existing data.
 BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
 {

src/txdb.cpp

@@ -49,7 +49,7 @@ uint256 CCoinsViewDB::GetBestBlock() const {
 }
 
 bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
-    CLevelDBBatch batch(db.GetObfuscateKey());
+    CLevelDBBatch batch(&db.GetObfuscateKey());
     size_t count = 0;
     size_t changed = 0;
     for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
@@ -98,8 +98,8 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
     /* It seems that there are no "const iterators" for LevelDB.  Since we
        only need read operations on it, use a const-cast to get around
        that restriction.  */
-    boost::scoped_ptr<leveldb::Iterator> pcursor(const_cast<CLevelDBWrapper*>(&db)->NewIterator());
+    boost::scoped_ptr<CLevelDBIterator> pcursor(const_cast<CLevelDBWrapper*>(&db)->NewIterator());
-    pcursor->SeekToFirst();
+    pcursor->Seek('c');
 
     CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION);
     stats.hashBlock = GetBestBlock();
@@ -107,22 +107,10 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
     CAmount nTotalAmount = 0;
     while (pcursor->Valid()) {
         boost::this_thread::interruption_point();
-        try {
+        std::pair<char, uint256> key;
-            leveldb::Slice slKey = pcursor->key();
-            CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION);
-            char chType;
-            ssKey >> chType;
-            if (chType == DB_COINS) {
-                leveldb::Slice slValue = pcursor->value();
-                CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
         CCoins coins;
-                ssValue >> coins;
+        if (pcursor->GetKey(key) && key.first == 'c') {
-                uint256 txhash;
+            if (pcursor->GetValue(coins)) {
-                ssKey >> txhash;
-                ss << txhash;
-                ss << VARINT(coins.nVersion);
-                ss << (coins.fCoinBase ? 'c' : 'n');
-                ss << VARINT(coins.nHeight);
                 stats.nTransactions++;
                 for (unsigned int i=0; i<coins.vout.size(); i++) {
                     const CTxOut &out = coins.vout[i];
@@ -133,13 +121,15 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
                         nTotalAmount += out.nValue;
                     }
                 }
-                stats.nSerializedSize += 32 + slValue.size();
+                stats.nSerializedSize += 32 + pcursor->GetKeySize();
                 ss << VARINT(0);
+            } else {
+                return error("CCoinsViewDB::GetStats() : unable to read value");
             }
-            pcursor->Next();
+        } else {
-        } catch (const std::exception& e) {
+            break;
-            return error("%s: Deserialize or I/O error - %s", __func__, e.what());
         }
+        pcursor->Next();
     }
     {
         LOCK(cs_main);
@@ -151,7 +141,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
 }
 
 bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
-    CLevelDBBatch batch(GetObfuscateKey());
+    CLevelDBBatch batch(&GetObfuscateKey());
     for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
         batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second);
     }
@@ -167,7 +157,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) {
 }
 
 bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) {
-    CLevelDBBatch batch(GetObfuscateKey());
+    CLevelDBBatch batch(&GetObfuscateKey());
     for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++)
         batch.Write(make_pair(DB_TXINDEX, it->first), it->second);
     return WriteBatch(batch);
@@ -187,26 +177,17 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
 
 bool CBlockTreeDB::LoadBlockIndexGuts()
 {
-    boost::scoped_ptr<leveldb::Iterator> pcursor(NewIterator());
+    boost::scoped_ptr<CLevelDBIterator> pcursor(NewIterator());
 
-    CDataStream ssKeySet(SER_DISK, CLIENT_VERSION);
+    pcursor->Seek(make_pair('b', uint256()));
-    ssKeySet << make_pair(DB_BLOCK_INDEX, uint256());
-    pcursor->Seek(ssKeySet.str());
 
     // Load mapBlockIndex
     while (pcursor->Valid()) {
         boost::this_thread::interruption_point();
-        try {
+        std::pair<char, uint256> key;
-            leveldb::Slice slKey = pcursor->key();
+        if (pcursor->GetKey(key) && key.first == 'b') {
-            CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION);
-            char chType;
-            ssKey >> chType;
-            if (chType == DB_BLOCK_INDEX) {
-                leveldb::Slice slValue = pcursor->value();
-                CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION);
             CDiskBlockIndex diskindex;
-                ssValue >> diskindex;
+            if (pcursor->GetValue(diskindex)) {
-
                 // Construct block index object
                 CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash());
                 pindexNew->pprev          = InsertBlockIndex(diskindex.hashPrev);
@@ -227,10 +208,10 @@ bool CBlockTreeDB::LoadBlockIndexGuts()
 
                 pcursor->Next();
             } else {
-                break; // if shutdown requested or finished loading block index
+                return error("LoadBlockIndex() : failed to read value");
             }
-        } catch (const std::exception& e) {
+        } else {
-            return error("%s: Deserialize or I/O error - %s", __func__, e.what());
+            break;
         }
     }