Merge #8020: Use SipHash-2-4 for various non-cryptographic hashes

a68ec21 Use SipHash-2-4 for address relay selection (Pieter Wuille)
8cc9cfe Switch CTxMempool::mapTx to use a hash index for txids (Pieter Wuille)
382c871 Use SipHash-2-4 for CCoinsCache index (Pieter Wuille)
0b1295b Add SipHash-2-4 primitives to hash (Pieter Wuille)

src/coins.cpp

@@ -56,7 +56,11 @@ void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; }
 bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); }
 CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); }
 
-CCoinsKeyHasher::CCoinsKeyHasher() : salt(GetRandHash()) {}
+SaltedTxidHasher::SaltedTxidHasher()
+{
+    GetRandBytes((unsigned char*)&k0, sizeof(k0));
+    GetRandBytes((unsigned char*)&k1, sizeof(k1));
+}
 
 CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { }
 

src/coins.h

@@ -8,6 +8,7 @@
 
 #include "compressor.h"
 #include "core_memusage.h"
+#include "hash.h"
 #include "memusage.h"
 #include "serialize.h"
 #include "uint256.h"
@@ -264,21 +265,22 @@ public:
     }
 };
 
-class CCoinsKeyHasher
+class SaltedTxidHasher
 {
 private:
-    uint256 salt;
+    /** Salt */
+    uint64_t k0, k1;
 
 public:
-    CCoinsKeyHasher();
+    SaltedTxidHasher();
 
     /**
      * This *must* return size_t. With Boost 1.46 on 32-bit systems the
      * unordered_map will behave unpredictably if the custom hasher returns a
      * uint64_t, resulting in failures when syncing the chain (#4634).
      */
-    size_t operator()(const uint256& key) const {
+    size_t operator()(const uint256& txid) const {
-        return key.GetHash(salt);
+        return SipHashUint256(k0, k1, txid);
     }
 };
 
@@ -295,7 +297,7 @@ struct CCoinsCacheEntry
     CCoinsCacheEntry() : coins(), flags(0) {}
 };
 
-typedef boost::unordered_map<uint256, CCoinsCacheEntry, CCoinsKeyHasher> CCoinsMap;
+typedef boost::unordered_map<uint256, CCoinsCacheEntry, SaltedTxidHasher> CCoinsMap;
 
 /** Cursor for iterating over CoinsView state */
 class CCoinsViewCursor

src/hash.cpp

@@ -81,3 +81,97 @@ void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char he
     num[3] = (nChild >>  0) & 0xFF;
     CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output);
 }
+
+#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
+
+#define SIPROUND do { \
+    v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; \
+    v0 = ROTL(v0, 32); \
+    v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \
+    v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \
+    v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; \
+    v2 = ROTL(v2, 32); \
+} while (0)
+
+CSipHasher::CSipHasher(uint64_t k0, uint64_t k1)
+{
+    v[0] = 0x736f6d6570736575ULL ^ k0;
+    v[1] = 0x646f72616e646f6dULL ^ k1;
+    v[2] = 0x6c7967656e657261ULL ^ k0;
+    v[3] = 0x7465646279746573ULL ^ k1;
+    count = 0;
+}
+
+CSipHasher& CSipHasher::Write(uint64_t data)
+{
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
+
+    v3 ^= data;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= data;
+
+    v[0] = v0;
+    v[1] = v1;
+    v[2] = v2;
+    v[3] = v3;
+
+    count++;
+    return *this;
+}
+
+uint64_t CSipHasher::Finalize() const
+{
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
+
+    v3 ^= ((uint64_t)count) << 59;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= ((uint64_t)count) << 59;
+    v2 ^= 0xFF;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    return v0 ^ v1 ^ v2 ^ v3;
+}
+
+uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val)
+{
+    /* Specialized implementation for efficiency */
+    uint64_t d = val.GetUint64(0);
+
+    uint64_t v0 = 0x736f6d6570736575ULL ^ k0;
+    uint64_t v1 = 0x646f72616e646f6dULL ^ k1;
+    uint64_t v2 = 0x6c7967656e657261ULL ^ k0;
+    uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d;
+
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(1);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(2);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(3);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    v3 ^= ((uint64_t)4) << 59;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= ((uint64_t)4) << 59;
+    v2 ^= 0xFF;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    return v0 ^ v1 ^ v2 ^ v3;
+}

src/hash.h

@@ -171,4 +171,19 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char
 
 void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]);
 
+/** SipHash-2-4, using a uint64_t-based (rather than byte-based) interface */
+class CSipHasher
+{
+private:
+    uint64_t v[4];
+    int count;
+
+public:
+    CSipHasher(uint64_t k0, uint64_t k1);
+    CSipHasher& Write(uint64_t data);
+    uint64_t Finalize() const;
+};
+
+uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val);
+
 #endif // BITCOIN_HASH_H

src/main.cpp

@@ -4698,25 +4698,23 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
                     LOCK(cs_vNodes);
                     // Use deterministic randomness to send to the same nodes for 24 hours
                     // at a time so the addrKnowns of the chosen nodes prevent repeats
-                    static uint256 hashSalt;
+                    static uint64_t salt0 = 0, salt1 = 0;
-                    if (hashSalt.IsNull())
+                    while (salt0 == 0 && salt1 == 0) {
-                        hashSalt = GetRandHash();
+                        GetRandBytes((unsigned char*)&salt0, sizeof(salt0));
+                        GetRandBytes((unsigned char*)&salt1, sizeof(salt1));
+                    }
                     uint64_t hashAddr = addr.GetHash();
-                    uint256 hashRand = ArithToUint256(UintToArith256(hashSalt) ^ (hashAddr<<32) ^ ((GetTime()+hashAddr)/(24*60*60)));
+                    multimap<uint64_t, CNode*> mapMix;
-                    hashRand = Hash(BEGIN(hashRand), END(hashRand));
+                    const CSipHasher hasher = CSipHasher(salt0, salt1).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60));
-                    multimap<uint256, CNode*> mapMix;
                     BOOST_FOREACH(CNode* pnode, vNodes)
                     {
                         if (pnode->nVersion < CADDR_TIME_VERSION)
                             continue;
-                        unsigned int nPointer;
+                        uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize();
-                        memcpy(&nPointer, &pnode, sizeof(nPointer));
-                        uint256 hashKey = ArithToUint256(UintToArith256(hashRand) ^ nPointer);
-                        hashKey = Hash(BEGIN(hashKey), END(hashKey));
                         mapMix.insert(make_pair(hashKey, pnode));
                     }
                     int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s)
-                    for (multimap<uint256, CNode*>::iterator mi = mapMix.begin(); mi != mapMix.end() && nRelayNodes-- > 0; ++mi)
+                    for (multimap<uint64_t, CNode*>::iterator mi = mapMix.begin(); mi != mapMix.end() && nRelayNodes-- > 0; ++mi)
                         ((*mi).second)->PushAddress(addr);
                 }
             }

src/test/hash_tests.cpp

@@ -47,4 +47,24 @@ BOOST_AUTO_TEST_CASE(murmurhash3)
 #undef T
 }
 
+BOOST_AUTO_TEST_CASE(siphash)
+{
+    CSipHasher hasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0x726fdb47dd0e0e31ull);
+    hasher.Write(0x0706050403020100ULL);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0x93f5f5799a932462ull);
+    hasher.Write(0x0F0E0D0C0B0A0908ULL);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0x3f2acc7f57c29bdbull);
+    hasher.Write(0x1716151413121110ULL);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0xb8ad50c6f649af94ull);
+    hasher.Write(0x1F1E1D1C1B1A1918ULL);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0x7127512f72f27cceull);
+    hasher.Write(0x2726252423222120ULL);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0x0e3ea96b5304a7d0ull);
+    hasher.Write(0x2F2E2D2C2B2A2928ULL);
+    BOOST_CHECK_EQUAL(hasher.Finalize(),  0xe612a3cb9ecba951ull);
+
+    BOOST_CHECK_EQUAL(SipHashUint256(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL, uint256S("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100")), 0x7127512f72f27cceull);
+}
+
 BOOST_AUTO_TEST_SUITE_END()

src/txmempool.h

@@ -17,6 +17,7 @@
 #undef foreach
 #include "boost/multi_index_container.hpp"
 #include "boost/multi_index/ordered_index.hpp"
+#include "boost/multi_index/hashed_index.hpp"
 
 class CAutoFile;
 class CBlockIndex;
@@ -422,7 +423,7 @@ public:
         CTxMemPoolEntry,
         boost::multi_index::indexed_by<
             // sorted by txid
-            boost::multi_index::ordered_unique<mempoolentry_txid>,
+            boost::multi_index::hashed_unique<mempoolentry_txid, SaltedTxidHasher>,
             // sorted by fee rate
             boost::multi_index::ordered_non_unique<
                 boost::multi_index::tag<descendant_score>,

src/uint256.cpp

@@ -80,67 +80,3 @@ template std::string base_blob<256>::GetHex() const;
 template std::string base_blob<256>::ToString() const;
 template void base_blob<256>::SetHex(const char*);
 template void base_blob<256>::SetHex(const std::string&);
-
-static void inline HashMix(uint32_t& a, uint32_t& b, uint32_t& c)
-{
-    // Taken from lookup3, by Bob Jenkins.
-    a -= c;
-    a ^= ((c << 4) | (c >> 28));
-    c += b;
-    b -= a;
-    b ^= ((a << 6) | (a >> 26));
-    a += c;
-    c -= b;
-    c ^= ((b << 8) | (b >> 24));
-    b += a;
-    a -= c;
-    a ^= ((c << 16) | (c >> 16));
-    c += b;
-    b -= a;
-    b ^= ((a << 19) | (a >> 13));
-    a += c;
-    c -= b;
-    c ^= ((b << 4) | (b >> 28));
-    b += a;
-}
-
-static void inline HashFinal(uint32_t& a, uint32_t& b, uint32_t& c)
-{
-    // Taken from lookup3, by Bob Jenkins.
-    c ^= b;
-    c -= ((b << 14) | (b >> 18));
-    a ^= c;
-    a -= ((c << 11) | (c >> 21));
-    b ^= a;
-    b -= ((a << 25) | (a >> 7));
-    c ^= b;
-    c -= ((b << 16) | (b >> 16));
-    a ^= c;
-    a -= ((c << 4) | (c >> 28));
-    b ^= a;
-    b -= ((a << 14) | (a >> 18));
-    c ^= b;
-    c -= ((b << 24) | (b >> 8));
-}
-
-uint64_t uint256::GetHash(const uint256& salt) const
-{
-    uint32_t a, b, c;
-    const uint32_t *pn = (const uint32_t*)data;
-    const uint32_t *salt_pn = (const uint32_t*)salt.data;
-    a = b = c = 0xdeadbeef + WIDTH;
-
-    a += pn[0] ^ salt_pn[0];
-    b += pn[1] ^ salt_pn[1];
-    c += pn[2] ^ salt_pn[2];
-    HashMix(a, b, c);
-    a += pn[3] ^ salt_pn[3];
-    b += pn[4] ^ salt_pn[4];
-    c += pn[5] ^ salt_pn[5];
-    HashMix(a, b, c);
-    a += pn[6] ^ salt_pn[6];
-    b += pn[7] ^ salt_pn[7];
-    HashFinal(a, b, c);
-
-    return ((((uint64_t)b) << 32) | c);
-}

src/uint256.h

@@ -83,6 +83,19 @@ public:
         return sizeof(data);
     }
 
+    uint64_t GetUint64(int pos) const
+    {
+        const uint8_t* ptr = data + pos * 8;
+        return ((uint64_t)ptr[0]) | \
+               ((uint64_t)ptr[1]) << 8 | \
+               ((uint64_t)ptr[2]) << 16 | \
+               ((uint64_t)ptr[3]) << 24 | \
+               ((uint64_t)ptr[4]) << 32 | \
+               ((uint64_t)ptr[5]) << 40 | \
+               ((uint64_t)ptr[6]) << 48 | \
+               ((uint64_t)ptr[7]) << 56;
+    }
+
     template<typename Stream>
     void Serialize(Stream& s, int nType, int nVersion) const
     {
@@ -127,11 +140,6 @@ public:
     {
         return ReadLE64(data);
     }
-
-    /** A more secure, salted hash function.
-     * @note This hash is not stable between little and big endian.
-     */
-    uint64_t GetHash(const uint256& salt) const;
 };
 
 /* uint256 from const char *.