From 48a3c767e58435ee448ced2bf9664f4c6a09822d Mon Sep 17 00:00:00 2001 From: orignal Date: Sat, 11 Feb 2023 16:22:02 -0500 Subject: [PATCH] in-meory storage for router profiles --- libi2pd/NetDb.cpp | 16 +++++----- libi2pd/Profiling.cpp | 68 ++++++++++++++++++++++++++++++++++--------- libi2pd/Profiling.h | 14 ++++++--- libi2pd/RouterInfo.h | 2 +- 4 files changed, 74 insertions(+), 26 deletions(-) diff --git a/libi2pd/NetDb.cpp b/libi2pd/NetDb.cpp index d28b0bb3..0d68cee3 100644 --- a/libi2pd/NetDb.cpp +++ b/libi2pd/NetDb.cpp @@ -85,8 +85,7 @@ namespace data if (m_IsRunning) { if (m_PersistProfiles) - for (auto& it: m_RouterInfos) - it.second->SaveProfile (); + SaveProfiles (); DeleteObsoleteProfiles (); m_RouterInfos.clear (); m_Floodfills.clear (); @@ -179,6 +178,7 @@ namespace data if (ts - lastProfilesCleanup >= (uint64_t)(i2p::data::PEER_PROFILE_AUTOCLEAN_TIMEOUT + profilesCleanupVariance) || ts + i2p::data::PEER_PROFILE_AUTOCLEAN_TIMEOUT < lastProfilesCleanup) { + if (m_PersistProfiles) PersistProfiles (); DeleteObsoleteProfiles (); lastProfilesCleanup = ts; profilesCleanupVariance = (rand () % (2 * i2p::data::PEER_PROFILE_AUTOCLEAN_VARIANCE) - i2p::data::PEER_PROFILE_AUTOCLEAN_VARIANCE); @@ -684,12 +684,12 @@ namespace data for (auto it = m_RouterInfos.begin (); it != m_RouterInfos.end ();) { if (it->second->IsUnreachable ()) - { - if (m_PersistProfiles) it->second->SaveProfile (); it = m_RouterInfos.erase (it); - continue; - } - ++it; + else + { + it->second->DropProfile (); + it++; + } } } // clean up expired floodfills or not floodfills anymore @@ -699,7 +699,7 @@ namespace data if ((*it)->IsUnreachable () || !(*it)->IsFloodfill ()) it = m_Floodfills.erase (it); else - ++it; + it++; } } } diff --git a/libi2pd/Profiling.cpp b/libi2pd/Profiling.cpp index a127ce81..2dafe8ca 100644 --- a/libi2pd/Profiling.cpp +++ b/libi2pd/Profiling.cpp @@ -7,6 +7,7 @@ */ #include +#include #include #include #include "Base.h" @@ -19,24 +20,26 @@ namespace i2p { namespace data { - i2p::fs::HashedStorage m_ProfilesStorage("peerProfiles", "p", "profile-", "txt"); + static i2p::fs::HashedStorage g_ProfilesStorage("peerProfiles", "p", "profile-", "txt"); + static std::unordered_map > g_Profiles; + static boost::posix_time::ptime GetTime () + { + return boost::posix_time::second_clock::local_time(); + } + RouterProfile::RouterProfile (): - m_LastUpdateTime (boost::posix_time::second_clock::local_time()), + m_LastUpdateTime (GetTime ()), m_IsUpdated (false), m_LastDeclineTime (0), m_LastUnreachableTime (0), m_NumTunnelsAgreed (0), m_NumTunnelsDeclined (0), m_NumTunnelsNonReplied (0), m_NumTimesTaken (0), m_NumTimesRejected (0) { } - boost::posix_time::ptime RouterProfile::GetTime () const - { - return boost::posix_time::second_clock::local_time(); - } - void RouterProfile::UpdateTime () { m_LastUpdateTime = GetTime (); + m_IsUpdated = true; } void RouterProfile::Save (const IdentHash& identHash) @@ -59,7 +62,7 @@ namespace data // save to file std::string ident = identHash.ToBase64 (); - std::string path = m_ProfilesStorage.Path(ident); + std::string path = g_ProfilesStorage.Path(ident); try { boost::property_tree::write_ini (path, pt); @@ -72,7 +75,7 @@ namespace data void RouterProfile::Load (const IdentHash& identHash) { std::string ident = identHash.ToBase64 (); - std::string path = m_ProfilesStorage.Path(ident); + std::string path = g_ProfilesStorage.Path(ident); boost::property_tree::ptree pt; if (!i2p::fs::Exists(path)) @@ -158,6 +161,7 @@ namespace data void RouterProfile::Unreachable () { m_LastUnreachableTime = i2p::util::GetSecondsSinceEpoch (); + UpdateTime (); } bool RouterProfile::IsLowPartcipationRate () const @@ -209,30 +213,68 @@ namespace data std::shared_ptr GetRouterProfile (const IdentHash& identHash) { + auto it = g_Profiles.find (identHash); + if (it != g_Profiles.end ()) + return it->second; auto profile = std::make_shared (); profile->Load (identHash); // if possible + g_Profiles.emplace (identHash, profile); return profile; } void InitProfilesStorage () { - m_ProfilesStorage.SetPlace(i2p::fs::GetDataDir()); - m_ProfilesStorage.Init(i2p::data::GetBase64SubstitutionTable(), 64); + g_ProfilesStorage.SetPlace(i2p::fs::GetDataDir()); + g_ProfilesStorage.Init(i2p::data::GetBase64SubstitutionTable(), 64); } + void PersistProfiles () + { + auto ts = GetTime (); + for (auto it = g_Profiles.begin (); it != g_Profiles.end ();) + { + if ((ts - it->second->GetLastUpdateTime ()).total_seconds () > PEER_PROFILE_PERSIST_INTERVAL) + { + if (it->second->IsUpdated ()) + it->second->Save (it->first); + it = g_Profiles.erase (it); + } + else + it++; + } + } + + void SaveProfiles () + { + auto ts = GetTime (); + for (auto it: g_Profiles) + if (it.second->IsUpdated () && (ts - it.second->GetLastUpdateTime ()).total_seconds () < PEER_PROFILE_EXPIRATION_TIMEOUT*3600) + it.second->Save (it.first); + g_Profiles.clear (); + } + void DeleteObsoleteProfiles () { + auto ts = GetTime (); + for (auto it = g_Profiles.begin (); it != g_Profiles.end ();) + { + if ((ts - it->second->GetLastUpdateTime ()).total_seconds () >= PEER_PROFILE_EXPIRATION_TIMEOUT*3600) + it = g_Profiles.erase (it); + else + it++; + } + struct stat st; std::time_t now = std::time(nullptr); std::vector files; - m_ProfilesStorage.Traverse(files); + g_ProfilesStorage.Traverse(files); for (const auto& path: files) { if (stat(path.c_str(), &st) != 0) { LogPrint(eLogWarning, "Profiling: Can't stat(): ", path); continue; } - if (((now - st.st_mtime) / 3600) >= PEER_PROFILE_EXPIRATION_TIMEOUT) { + if (now - st.st_mtime >= PEER_PROFILE_EXPIRATION_TIMEOUT*3600) { LogPrint(eLogDebug, "Profiling: Removing expired peer profile: ", path); i2p::fs::Remove(path); } diff --git a/libi2pd/Profiling.h b/libi2pd/Profiling.h index bccf19de..752d6190 100644 --- a/libi2pd/Profiling.h +++ b/libi2pd/Profiling.h @@ -29,10 +29,11 @@ namespace data const char PEER_PROFILE_USAGE_TAKEN[] = "taken"; const char PEER_PROFILE_USAGE_REJECTED[] = "rejected"; - const int PEER_PROFILE_EXPIRATION_TIMEOUT = 72; // in hours (3 days) - const int PEER_PROFILE_AUTOCLEAN_TIMEOUT = 24 * 3600; // in seconds (1 day) - const int PEER_PROFILE_AUTOCLEAN_VARIANCE = 3 * 3600; // in seconds (3 hours) + const int PEER_PROFILE_EXPIRATION_TIMEOUT = 36; // in hours (1.5 days) + const int PEER_PROFILE_AUTOCLEAN_TIMEOUT = 6 * 3600; // in seconds (6 hours) + const int PEER_PROFILE_AUTOCLEAN_VARIANCE = 3600; // in seconds (1 hour) const int PEER_PROFILE_DECLINED_RECENTLY_INTERVAL = 150; // in seconds (2.5 minutes) + const int PEER_PROFILE_PERSIST_INTERVAL = 3300; // in seconds (55 minutes) const int PEER_PROFILE_UNREACHABLE_INTERVAL = 2*3600; // on seconds (2 hours) class RouterProfile @@ -53,9 +54,11 @@ namespace data void Unreachable (); + boost::posix_time::ptime GetLastUpdateTime () const { return m_LastUpdateTime; }; + bool IsUpdated () const { return m_IsUpdated; }; + private: - boost::posix_time::ptime GetTime () const; void UpdateTime (); bool IsAlwaysDeclining () const { return !m_NumTunnelsAgreed && m_NumTunnelsDeclined >= 5; }; @@ -66,6 +69,7 @@ namespace data private: boost::posix_time::ptime m_LastUpdateTime; // TODO: use std::chrono + bool m_IsUpdated; uint64_t m_LastDeclineTime, m_LastUnreachableTime; // in seconds // participation uint32_t m_NumTunnelsAgreed; @@ -79,6 +83,8 @@ namespace data std::shared_ptr GetRouterProfile (const IdentHash& identHash); void InitProfilesStorage (); void DeleteObsoleteProfiles (); + void SaveProfiles (); + void PersistProfiles (); } } diff --git a/libi2pd/RouterInfo.h b/libi2pd/RouterInfo.h index ea0abf41..f31b3cc2 100644 --- a/libi2pd/RouterInfo.h +++ b/libi2pd/RouterInfo.h @@ -251,7 +251,7 @@ namespace data bool SaveToFile (const std::string& fullPath); std::shared_ptr GetProfile () const; - void SaveProfile () { if (m_Profile) m_Profile->Save (GetIdentHash ()); }; + void DropProfile () { m_Profile = nullptr; }; void Update (const uint8_t * buf, size_t len); void DeleteBuffer () { m_Buffer = nullptr; };