From d40cd00cdb01ae3080f49bad2ef151fe474e6051 Mon Sep 17 00:00:00 2001 From: orignal Date: Wed, 22 Feb 2023 15:58:20 -0500 Subject: [PATCH] use DHT table for floodfills --- libi2pd/KadDHT.cpp | 10 ++--- libi2pd/KadDHT.h | 12 ++--- libi2pd/NetDb.cpp | 108 +++++++++++++++------------------------------ libi2pd/NetDb.hpp | 6 +-- 4 files changed, 49 insertions(+), 87 deletions(-) diff --git a/libi2pd/KadDHT.cpp b/libi2pd/KadDHT.cpp index 48486675..b171cdbc 100644 --- a/libi2pd/KadDHT.cpp +++ b/libi2pd/KadDHT.cpp @@ -186,7 +186,7 @@ namespace data return false; } - std::shared_ptr DHTTable::FindClosest (const IdentHash& h, const Filter& filter) + std::shared_ptr DHTTable::FindClosest (const IdentHash& h, const Filter& filter) const { if (filter) m_Filter = filter; auto r = FindClosest (h, m_Root, 0); @@ -194,7 +194,7 @@ namespace data return r; } - std::shared_ptr DHTTable::FindClosest (const IdentHash& h, DHTNode * root, int level) + std::shared_ptr DHTTable::FindClosest (const IdentHash& h, DHTNode * root, int level) const { bool split = false; do @@ -241,7 +241,7 @@ namespace data return nullptr; } - std::vector > DHTTable::FindClosest (const IdentHash& h, size_t num, const Filter& filter) + std::vector > DHTTable::FindClosest (const IdentHash& h, size_t num, const Filter& filter) const { std::vector > vec; if (num > 0) @@ -253,7 +253,7 @@ namespace data return vec; } - void DHTTable::FindClosest (const IdentHash& h, size_t num, DHTNode * root, int level, std::vector >& hashes) + void DHTTable::FindClosest (const IdentHash& h, size_t num, DHTNode * root, int level, std::vector >& hashes) const { if (hashes.size () >= num) return; bool split = false; @@ -292,7 +292,7 @@ namespace data } } - void DHTTable::Cleanup (Filter filter) + void DHTTable::Cleanup (const Filter& filter) { if (filter) { diff --git a/libi2pd/KadDHT.h b/libi2pd/KadDHT.h index c280a1de..3bc31780 100644 --- a/libi2pd/KadDHT.h +++ b/libi2pd/KadDHT.h @@ -44,20 +44,20 @@ namespace data void Insert (const std::shared_ptr& r); bool Remove (const IdentHash& h); - std::shared_ptr FindClosest (const IdentHash& h, const Filter& filter = nullptr); - std::vector > FindClosest (const IdentHash& h, size_t num, const Filter& filter = nullptr); + std::shared_ptr FindClosest (const IdentHash& h, const Filter& filter = nullptr) const; + std::vector > FindClosest (const IdentHash& h, size_t num, const Filter& filter = nullptr) const; void Print (std::stringstream& s); size_t GetSize () const { return m_Size; }; void Clear (); - void Cleanup (Filter filter); + void Cleanup (const Filter& filter); private: void Insert (const std::shared_ptr& r, DHTNode * root, int level); // recursive bool Remove (const IdentHash& h, DHTNode * root, int level); - std::shared_ptr FindClosest (const IdentHash& h, DHTNode * root, int level); - void FindClosest (const IdentHash& h, size_t num, DHTNode * root, int level, std::vector >& hashes); + std::shared_ptr FindClosest (const IdentHash& h, DHTNode * root, int level) const; + void FindClosest (const IdentHash& h, size_t num, DHTNode * root, int level, std::vector >& hashes) const; void Cleanup (DHTNode * root); void Print (std::stringstream& s, DHTNode * root, int level); @@ -66,7 +66,7 @@ namespace data DHTNode * m_Root; size_t m_Size; // transient - Filter m_Filter; + mutable Filter m_Filter; }; } } diff --git a/libi2pd/NetDb.cpp b/libi2pd/NetDb.cpp index 11cf8066..09d57bac 100644 --- a/libi2pd/NetDb.cpp +++ b/libi2pd/NetDb.cpp @@ -55,7 +55,7 @@ namespace data Load (); uint16_t threshold; i2p::config::GetOption("reseed.threshold", threshold); - if (m_RouterInfos.size () < threshold || m_Floodfills.size () < NETDB_MIN_FLOODFILLS) // reseed if # of router less than threshold or too few floodfiils + if (m_RouterInfos.size () < threshold || m_Floodfills.GetSize () < NETDB_MIN_FLOODFILLS) // reseed if # of router less than threshold or too few floodfiils { Reseed (); } @@ -66,13 +66,13 @@ namespace data if (it != m_RouterInfos.end ()) { // remove own router - m_Floodfills.remove (it->second); + m_Floodfills.Remove (it->second->GetIdentHash ()); m_RouterInfos.erase (it); } // insert own router m_RouterInfos.emplace (i2p::context.GetIdentHash (), i2p::context.GetSharedRouterInfo ()); if (i2p::context.IsFloodfill ()) - m_Floodfills.push_back (i2p::context.GetSharedRouterInfo ()); + m_Floodfills.Insert (i2p::context.GetSharedRouterInfo ()); i2p::config::GetOption("persist.profiles", m_PersistProfiles); @@ -88,7 +88,7 @@ namespace data SaveProfiles (); DeleteObsoleteProfiles (); m_RouterInfos.clear (); - m_Floodfills.clear (); + m_Floodfills.Clear (); if (m_Thread) { m_IsRunning = false; @@ -289,7 +289,7 @@ namespace data if (wasFloodfill) { std::unique_lock l(m_FloodfillsMutex); - m_Floodfills.remove (r); + m_Floodfills.Remove (r->GetIdentHash ()); } m_Requests.RequestComplete (ident, nullptr); return nullptr; @@ -301,9 +301,9 @@ namespace data LogPrint (eLogDebug, "NetDb: RouterInfo floodfill status updated: ", ident.ToBase64()); std::unique_lock l(m_FloodfillsMutex); if (wasFloodfill) - m_Floodfills.remove (r); + m_Floodfills.Remove (r->GetIdentHash ()); else if (r->IsEligibleFloodfill ()) - m_Floodfills.push_back (r); + m_Floodfills.Insert (r); } } else @@ -329,7 +329,7 @@ namespace data if (r->IsFloodfill () && r->IsEligibleFloodfill ()) { std::unique_lock l(m_FloodfillsMutex); - m_Floodfills.push_back (r); + m_Floodfills.Insert (r); } } else @@ -530,7 +530,7 @@ namespace data if (m_RouterInfos.emplace (r->GetIdentHash (), r).second) { if (r->IsFloodfill () && r->IsEligibleFloodfill ()) - m_Floodfills.push_back (r); + m_Floodfills.Insert (r); } } else @@ -614,7 +614,7 @@ namespace data { // make sure we cleanup netDb from previous attempts m_RouterInfos.clear (); - m_Floodfills.clear (); + m_Floodfills.Clear (); uint64_t ts = i2p::util::GetMillisecondsSinceEpoch(); std::vector files; @@ -622,14 +622,14 @@ namespace data for (const auto& path : files) LoadRouterInfo (path, ts); - LogPrint (eLogInfo, "NetDb: ", m_RouterInfos.size(), " routers loaded (", m_Floodfills.size (), " floodfils)"); + LogPrint (eLogInfo, "NetDb: ", m_RouterInfos.size(), " routers loaded (", m_Floodfills.GetSize (), " floodfils)"); } void NetDb::SaveUpdated () { int updatedCount = 0, deletedCount = 0, deletedFloodfillsCount = 0; auto total = m_RouterInfos.size (); - auto totalFloodfills = m_Floodfills.size (); + auto totalFloodfills = m_Floodfills.GetSize (); uint64_t expirationTimeout = NETDB_MAX_EXPIRATION_TIMEOUT*1000LL; uint64_t ts = i2p::util::GetMillisecondsSinceEpoch(); auto uptime = i2p::context.GetUptime (); @@ -721,11 +721,10 @@ namespace data // clean up expired floodfills or not floodfills anymore { std::unique_lock l(m_FloodfillsMutex); - for (auto it = m_Floodfills.begin (); it != m_Floodfills.end ();) - if ((*it)->IsUnreachable () || !(*it)->IsFloodfill ()) - it = m_Floodfills.erase (it); - else - it++; + m_Floodfills.Cleanup ([](const std::shared_ptr& r)->bool + { + return r && r->IsFloodfill () && !r->IsUnreachable (); + }); } } } @@ -1362,75 +1361,38 @@ namespace data std::shared_ptr NetDb::GetClosestFloodfill (const IdentHash& destination, const std::set& excluded) const { - std::shared_ptr r; - XORMetric minMetric; IdentHash destKey = CreateRoutingKey (destination); - minMetric.SetMax (); std::unique_lock l(m_FloodfillsMutex); - for (const auto& it: m_Floodfills) - { - if (!it->IsUnreachable () && !it->GetProfile ()->IsUnreachable ()) + return m_Floodfills.FindClosest (destKey, [&excluded](const std::shared_ptr& r)->bool { - XORMetric m = destKey ^ it->GetIdentHash (); - if (m < minMetric && !excluded.count (it->GetIdentHash ())) - { - minMetric = m; - r = it; - } - } - } - return r; + return r && !r->IsUnreachable () && !r->GetProfile ()->IsUnreachable () && + !excluded.count (r->GetIdentHash ()); + }); } std::vector NetDb::GetClosestFloodfills (const IdentHash& destination, size_t num, std::set& excluded, bool closeThanUsOnly) const { - struct Sorted - { - std::shared_ptr r; - XORMetric metric; - bool operator< (const Sorted& other) const { return metric < other.metric; }; - }; - - std::set sorted; + std::vector res; IdentHash destKey = CreateRoutingKey (destination); - XORMetric ourMetric; - if (closeThanUsOnly) ourMetric = destKey ^ i2p::context.GetIdentHash (); + std::vector > v; { std::unique_lock l(m_FloodfillsMutex); - for (const auto& it: m_Floodfills) - { - if (!it->IsUnreachable () && !it->GetProfile ()->IsUnreachable ()) + v = m_Floodfills.FindClosest (destKey, num, [&excluded](const std::shared_ptr& r)->bool { - XORMetric m = destKey ^ it->GetIdentHash (); - if (closeThanUsOnly && ourMetric < m) continue; - if (sorted.size () < num) - sorted.insert ({it, m}); - else if (m < sorted.rbegin ()->metric) - { - sorted.insert ({it, m}); - sorted.erase (std::prev (sorted.end ())); - } - } - } - } - - std::vector res; - size_t i = 0; - for (const auto& it: sorted) + return r && !r->IsUnreachable () && !r->GetProfile ()->IsUnreachable () && + !excluded.count (r->GetIdentHash ()); + }); + } + if (v.empty ()) return res; + + XORMetric ourMetric; + if (closeThanUsOnly) ourMetric = destKey ^ i2p::context.GetIdentHash (); + for (auto& it: v) { - if (i < num) - { - const auto& ident = it.r->GetIdentHash (); - if (!excluded.count (ident)) - { - res.push_back (ident); - i++; - } - } - else - break; - } + if (closeThanUsOnly && ourMetric < (destKey ^ it->GetIdentHash ())) break; + res.push_back (it->GetIdentHash ()); + } return res; } diff --git a/libi2pd/NetDb.hpp b/libi2pd/NetDb.hpp index 192d2644..74774f0d 100644 --- a/libi2pd/NetDb.hpp +++ b/libi2pd/NetDb.hpp @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -31,6 +30,7 @@ #include "Family.h" #include "version.h" #include "util.h" +#include "KadDHT.h" namespace i2p { @@ -110,7 +110,7 @@ namespace data // for web interface int GetNumRouters () const { return m_RouterInfos.size (); }; - int GetNumFloodfills () const { return m_Floodfills.size (); }; + int GetNumFloodfills () const { return m_Floodfills.GetSize (); }; int GetNumLeaseSets () const { return m_LeaseSets.size (); }; /** visit all lease sets we currently store */ @@ -164,7 +164,7 @@ namespace data mutable std::mutex m_RouterInfosMutex; std::unordered_map > m_RouterInfos; mutable std::mutex m_FloodfillsMutex; - std::list > m_Floodfills; + DHTTable m_Floodfills; bool m_IsRunning; std::thread * m_Thread;