From f24054100ef3d5d0bab4d73c17ef72d3e61ae5fc Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 11 Feb 2016 00:00:00 +0000 Subject: [PATCH 01/23] * new i2p::fs implementation --- FS.cpp | 189 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ FS.h | 143 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 FS.cpp create mode 100644 FS.h diff --git a/FS.cpp b/FS.cpp new file mode 100644 index 00000000..758b81c8 --- /dev/null +++ b/FS.cpp @@ -0,0 +1,189 @@ +/* +* Copyright (c) 2013-2016, The PurpleI2P Project +* +* This file is part of Purple i2pd project and licensed under BSD3 +* +* See full license text in LICENSE file at top of project tree +*/ + +#include +#include + +#include "Base.h" +#include "FS.h" +#include "Log.h" + +namespace i2p { +namespace fs { + std::string appName = "i2pd"; + std::string dataDir = ""; +#ifdef _WIN32 + std::string dirSep = "\\"; +#else + std::string dirSep = "/"; +#endif + HashedStorage NetDB("netDb", "r", "routerInfo-", "dat"); + HashedStorage Peers("peerProfiles", "p", "profile-", "txt"); + ABookStorage ABook("addressbook", "b", "", "b32"); + + static const char T32[32] = { + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'k', 'k', 'l', 'm', 'n', 'o', 'p', + 'q', 'r', 't', 't', 'u', 'v', 'w', 'x', + 'y', 'z', '2', '3', '4', '5', '6', '7', + }; + static const char T64[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '-', '~' + }; + + const std::string & GetAppName () { + return appName; + } + + void SetAppName (const std::string& name) { + appName = name; + } + + const std::string & GetDataDir () { + return dataDir; + } + + void DetectDataDir(const std::string & cmdline_param, bool isService) { + if (cmdline_param != "") { + dataDir = cmdline_param; + return; + } +#if defined(WIN32) || defined(_WIN32) + char localAppData[MAX_PATH]; + SHGetFolderPath(NULL, CSIDL_APPDATA, 0, NULL, localAppData); + dataDir = std::string(localAppData) + "\\" + appName; + return; +#elif defined(MAC_OSX) + char *home = getenv("HOME"); + dataDir = (home != NULL && strlen(home) > 0) ? home : ""; + dataDir += "/Library/Application Support/" + appName; + return; +#else /* other unix */ + char *home = getenv("HOME"); + if (isService) { + dataDir = "/var/lib/" + appName; + } else if (home != NULL && strlen(home) > 0) { + dataDir = std::string(home) + "/." + appName; + } else { + dataDir = "/tmp/" + appName; + } + return; +#endif + } + + bool Init() { + if (boost::filesystem::exists(dataDir)) + boost::filesystem::create_directory(dataDir); + std::string destinations = DataDirPath("destinations"); + if (boost::filesystem::exists(destinations)) + boost::filesystem::create_directory(destinations); + + NetDB.SetRoot(dataDir); + NetDB.Init(T64, 64); + Peers.SetRoot(dataDir); + Peers.Init(T64, 64); + ABook.SetRoot(dataDir); + ABook.Init(T32, 32); + return true; + } + + bool ReadDir(const std::string & path, std::vector & files) { + if (!boost::filesystem::exists(path)) + return false; + boost::filesystem::directory_iterator it(path); + boost::filesystem::directory_iterator end; + + for ( ; it != end; it++) { + if (!boost::filesystem::is_regular_file(it->status())) + continue; + files.push_back(it->path().string()); + } + + return true; + } + + bool Exists(const std::string & path) { + return boost::filesystem::exists(path); + } + + bool Remove(const std::string & path) { + if (!boost::filesystem::exists(path)) + return false; + return boost::filesystem::remove(path); + } + + void HashedStorage::SetRoot(const std::string &path) { + root = path + i2p::fs::dirSep + name; + } + + bool HashedStorage::Init(const char * chars, size_t count) { + if (!boost::filesystem::exists(root)) { + boost::filesystem::create_directory(root); + } + + for (size_t i = 0; i < count; i++) { + auto p = root + i2p::fs::dirSep + prefix1 + chars[i]; + if (boost::filesystem::exists(p)) + continue; + if (boost::filesystem::create_directory(p)) + continue; /* ^ throws exception on failure */ + return false; + } + return true; + } + + std::string HashedStorage::Path(const std::string & ident) { + std::string safe_ident = ident; + std::replace(safe_ident.begin(), safe_ident.end(), '/', '-'); + std::replace(safe_ident.begin(), safe_ident.end(), '\\', '-'); + + std::stringstream t(""); + t << this->root << i2p::fs::dirSep; + t << prefix1 << safe_ident[0] << i2p::fs::dirSep; + t << prefix2 << safe_ident << "." << suffix; + + return t.str(); + } + + void HashedStorage::Remove(const std::string & ident) { + std::string path = Path(ident); + if (!boost::filesystem::exists(path)) + return; + boost::filesystem::remove(path); + } + + void HashedStorage::Traverse(std::vector & files) { + boost::filesystem::path p(root); + boost::filesystem::recursive_directory_iterator it(p); + boost::filesystem::recursive_directory_iterator end; + + for ( ; it != end; it++) { + if (!boost::filesystem::is_regular_file( it->status() )) + continue; + const std::string & t = it->path().string(); + files.push_back(t); + } + } + + std::string ABookStorage::IndexPath() { + std::string path = root + i2p::fs::dirSep + "addresses.csv"; + return path; + } + + HashedStorage & GetNetDB() { return NetDB; } + HashedStorage & GetPeerProfiles() { return Peers; } + ABookStorage & GetAddressBook() { return ABook; } +} // fs +} // i2p diff --git a/FS.h b/FS.h new file mode 100644 index 00000000..80b07353 --- /dev/null +++ b/FS.h @@ -0,0 +1,143 @@ +/* +* Copyright (c) 2013-2016, The PurpleI2P Project +* +* This file is part of Purple i2pd project and licensed under BSD3 +* +* See full license text in LICENSE file at top of project tree +*/ + +#include +#include +#include +#include + +namespace i2p { +namespace fs { + extern std::string dirSep; + + /** + * @brief Class to work with NetDb & Router profiles + * + * Usage: + * + * const char alphabet[8] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}; + * auto h = HashedStorage("name", "y", "z-", ".txt"); + * h.SetRoot("/tmp/hs-test"); + * h.Init(alphabet, 8); <- creates needed dirs, 8 is size of alphabet + * h.Path("abcd"); <- returns /tmp/hs-test/name/ya/z-abcd.txt + * h.Remove("abcd"); <- removes /tmp/hs-test/name/ya/z-abcd.txt, if it exists + * std::vector files; + * h.Traverse(files); <- finds all files in storage and saves in given vector + */ + class HashedStorage { + protected: + std::string root; + std::string name; + std::string prefix1; + std::string prefix2; + std::string suffix; + + public: + HashedStorage(const char *n, const char *p1, const char *p2, const char *s): + name(n), prefix1(p1), prefix2(p2), suffix(s) {}; + + bool Init(const char* chars, size_t cnt); + const std::string & GetRoot() { return this->root; } + void SetRoot(const std::string & path); + std::string Path(const std::string & ident); + void Remove(const std::string & ident); + void Traverse(std::vector & files); + }; + + /** @brief Slightly extended HashedStorage */ + class ABookStorage : public HashedStorage { + public: + ABookStorage(const char *n, const char *p1, const char *p2, const char *s): + HashedStorage(n, p1, p2, s) {}; + + std::string IndexPath(); + }; + + /** @brief Returns current application name, default 'i2pd' */ + const std::string & GetAppName (); + /** @brief Set applicaton name, affects autodetection of datadir */ + void SetAppName (const std::string& name); + + /** @brief Returns datadir path */ + const std::string & GetDataDir(); + + /** + * @brief Set datadir either from cmdline option or using autodetection + * @param cmdline_param Value of cmdline parameter --datadir= + * @param isService Value of cmdline parameter --service + * + * Examples of autodetected paths: + * + * Windows < Vista: C:\Documents and Settings\Username\Application Data\i2pd\ + * Windows >= Vista: C:\Users\Username\AppData\Roaming\i2pd\ + * Mac: /Library/Application Support/i2pd/ or ~/Library/Application Support/i2pd/ + * Unix: /var/lib/i2pd/ (system=1) >> ~/.i2pd/ or /tmp/i2pd/ + */ + void DetectDataDir(const std::string & cmdline_datadir, bool isService = false); + + /** + * @brief Create subdirectories inside datadir + */ + bool Init(); + + /** + * @brief Get list of files in directory + * @param path Path to directory + * @param files Vector to store found files + * @return true on success and false if directory not exists + */ + bool ReadDir(const std::string & path, std::vector & files); + + /** + * @brief Remove file with given path + * @param path Absolute path to file + * @return true on success, false if file not exists, throws exception on error + */ + bool Remove(const std::string & path); + + /** + * @brief Check existence of file + * @param path Absolute path to file + * @return true if file exists, false otherwise + */ + bool Exists(const std::string & path); + + template + void _ExpandPath(std::stringstream & path, T c) { + path << i2p::fs::dirSep << c; + } + + template + void _ExpandPath(std::stringstream & path, T c, Other ... other) { + _ExpandPath(path, c); + _ExpandPath(path, other ...); + } + + /** + * @brief Get path relative to datadir + * + * Examples (with datadir = "/tmp/i2pd"): + * + * i2p::fs::Path("test") -> '/tmp/i2pd/test' + * i2p::fs::Path("test", "file.txt") -> '/tmp/i2pd/test/file.txt' + */ + template + std::string DataDirPath(Other ... components) { + std::stringstream s(""); + s << i2p::fs::GetDataDir(); + _ExpandPath(s, components ...); + + return s.str(); + } + + /* accessors */ + HashedStorage & GetNetDB(); + HashedStorage & GetPeerProfiles(); + ABookStorage & GetAddressBook(); +} // fs +} // i2p From 6f4271c054c033785c42d745db8140fad0cf710c Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 11 Feb 2016 00:00:00 +0000 Subject: [PATCH 02/23] * update buildsystems --- build/CMakeLists.txt | 1 + filelist.mk | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build/CMakeLists.txt b/build/CMakeLists.txt index 646e8a6e..7227f4ea 100644 --- a/build/CMakeLists.txt +++ b/build/CMakeLists.txt @@ -23,6 +23,7 @@ set (LIBI2PD_SRC "${CMAKE_SOURCE_DIR}/I2NPProtocol.cpp" "${CMAKE_SOURCE_DIR}/Identity.cpp" "${CMAKE_SOURCE_DIR}/LeaseSet.cpp" + "${CMAKE_SOURCE_DIR}/FS.cpp" "${CMAKE_SOURCE_DIR}/Log.cpp" "${CMAKE_SOURCE_DIR}/NTCPSession.cpp" "${CMAKE_SOURCE_DIR}/NetDbRequests.cpp" diff --git a/filelist.mk b/filelist.mk index 1d46b8fc..4417ba9b 100644 --- a/filelist.mk +++ b/filelist.mk @@ -4,7 +4,7 @@ LIB_SRC = \ Reseed.cpp RouterContext.cpp RouterInfo.cpp Signature.cpp SSU.cpp \ SSUSession.cpp SSUData.cpp Streaming.cpp Identity.cpp TransitTunnel.cpp \ Transports.cpp Tunnel.cpp TunnelEndpoint.cpp TunnelPool.cpp TunnelGateway.cpp \ - Destination.cpp Base.cpp I2PEndian.cpp Config.cpp util.cpp api.cpp + Destination.cpp Base.cpp I2PEndian.cpp FS.cpp Config.cpp util.cpp api.cpp LIB_CLIENT_SRC = \ AddressBook.cpp BOB.cpp ClientContext.cpp I2PTunnel.cpp I2PService.cpp \ From 6d7449349159061e8254a6ac8c1d69e5ee5990f0 Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 11 Feb 2016 00:00:00 +0000 Subject: [PATCH 03/23] * i2p::fs migration: NetDb.* --- NetDb.cpp | 203 +++++++++++++++++++----------------------------------- NetDb.h | 5 +- 2 files changed, 73 insertions(+), 135 deletions(-) diff --git a/NetDb.cpp b/NetDb.cpp index fe1d6a45..9253dcdc 100644 --- a/NetDb.cpp +++ b/NetDb.cpp @@ -1,12 +1,13 @@ #include -#include "I2PEndian.h" #include #include #include #include #include +#include "I2PEndian.h" #include "Base.h" #include "Log.h" +#include "FS.h" #include "Timestamp.h" #include "I2NPProtocol.h" #include "Tunnel.h" @@ -22,7 +23,6 @@ namespace i2p { namespace data { - const char NetDb::m_NetDbPath[] = "netDb"; NetDb netdb; NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr) @@ -250,30 +250,6 @@ namespace data return it->second->SetUnreachable (unreachable); } - // TODO: Move to reseed and/or scheduled tasks. (In java version, scheduler fix this as well as sort RIs.) - bool NetDb::CreateNetDb(boost::filesystem::path directory) - { - LogPrint (eLogInfo, "NetDb: storage directory doesn't exist, trying to create it."); - if (!boost::filesystem::create_directory (directory)) - { - LogPrint (eLogError, "NetDb: failed to create directory ", directory); - return false; - } - - // list of chars might appear in base64 string - const char * chars = GetBase64SubstitutionTable (); // 64 bytes - for (int i = 0; i < 64; i++) - { - auto p = directory / (std::string ("r") + chars[i]); - if (!boost::filesystem::exists (p) && !boost::filesystem::create_directory (p)) - { - LogPrint (eLogError, "NetDb: failed to create directory ", p); - return false; - } - } - return true; - } - void NetDb::Reseed () { if (!m_Reseeder) @@ -288,145 +264,108 @@ namespace data LogPrint (eLogWarning, "NetDb: failed to reseed after 10 attempts"); } - void NetDb::Load () + bool NetDb::LoadRouterInfo (const std::string & path) { - boost::filesystem::path p(i2p::util::filesystem::GetDataDir() / m_NetDbPath); - if (!boost::filesystem::exists (p)) + auto r = std::make_shared(path); + if (r->GetRouterIdentity () && !r->IsUnreachable () && + (!r->UsesIntroducer () || m_LastLoad < r->GetTimestamp () + 3600*1000LL)) // 1 hour { - // seems netDb doesn't exist yet - if (!CreateNetDb(p)) return; + r->DeleteBuffer (); + r->ClearProperties (); // properties are not used for regular routers + m_RouterInfos[r->GetIdentHash ()] = r; + if (r->IsFloodfill ()) + m_Floodfills.push_back (r); + } else { + LogPrint(eLogWarning, "NetDb: Can't load RI from ", path, ", delete"); + i2p::fs::Remove(path); } + return true; + } + + void NetDb::Load () + { // make sure we cleanup netDb from previous attempts m_RouterInfos.clear (); m_Floodfills.clear (); - // load routers now - uint64_t ts = i2p::util::GetMillisecondsSinceEpoch (); - int numRouters = 0; - boost::filesystem::directory_iterator end; - for (boost::filesystem::directory_iterator it (p); it != end; ++it) - { - if (boost::filesystem::is_directory (it->status())) - { - for (boost::filesystem::directory_iterator it1 (it->path ()); it1 != end; ++it1) - { -#if BOOST_VERSION > 10500 - const std::string& fullPath = it1->path().string(); -#else - const std::string& fullPath = it1->path(); -#endif - auto r = std::make_shared(fullPath); - if (r->GetRouterIdentity () && !r->IsUnreachable () && - (!r->UsesIntroducer () || ts < r->GetTimestamp () + 3600*1000LL)) // 1 hour - { - r->DeleteBuffer (); - r->ClearProperties (); // properties are not used for regular routers - m_RouterInfos[r->GetIdentHash ()] = r; - if (r->IsFloodfill ()) - m_Floodfills.push_back (r); - numRouters++; - } - else - { - if (boost::filesystem::exists (fullPath)) - boost::filesystem::remove (fullPath); - } - } - } - } - LogPrint (eLogInfo, "NetDb: ", numRouters, " routers loaded (", m_Floodfills.size (), " floodfils)"); + m_LastLoad = i2p::util::GetSecondsSinceEpoch(); + std::vector files; + i2p::fs::GetNetDB().Traverse(files); + for (auto path : files) + LoadRouterInfo(path); + + LogPrint (eLogInfo, "NetDb: ", m_RouterInfos.size(), " routers loaded (", m_Floodfills.size (), " floodfils)"); } void NetDb::SaveUpdated () { - auto GetFilePath = [](const boost::filesystem::path& directory, const RouterInfo * routerInfo) - { - std::string s(routerInfo->GetIdentHashBase64()); - return directory / (std::string("r") + s[0]) / ("routerInfo-" + s + ".dat"); - }; - - boost::filesystem::path fullDirectory (i2p::util::filesystem::GetDataDir() / m_NetDbPath); - int count = 0, deletedCount = 0; + int updatedCount = 0, deletedCount = 0; auto total = m_RouterInfos.size (); - uint64_t ts = i2p::util::GetMillisecondsSinceEpoch (); + uint64_t ts = i2p::util::GetMillisecondsSinceEpoch(); + for (auto it: m_RouterInfos) { - if (it.second->IsUpdated ()) - { - std::string f = GetFilePath(fullDirectory, it.second.get()).string(); - it.second->SaveToFile (f); + std::string ident = it.second->GetIdentHashBase64(); + std::string path = i2p::fs::GetNetDB().Path(ident); + if (it.second->IsUpdated ()) { + it.second->SaveToFile (path); it.second->SetUpdated (false); it.second->SetUnreachable (false); it.second->DeleteBuffer (); - count++; + updatedCount++; + continue; } - else - { + // find & mark unreachable routers + if (it.second->UsesIntroducer () && ts > it.second->GetTimestamp () + 3600*1000LL) { // RouterInfo expires after 1 hour if uses introducer - if (it.second->UsesIntroducer () && ts > it.second->GetTimestamp () + 3600*1000LL) // 1 hour - it.second->SetUnreachable (true); - else if (total > 75 && ts > (i2p::context.GetStartupTime () + 600)*1000LL) // routers don't expire if less than 25 or uptime is less than 10 minutes - { - if (i2p::context.IsFloodfill ()) - { - if (ts > it.second->GetTimestamp () + 3600*1000LL) // 1 hours - { - it.second->SetUnreachable (true); - total--; - } + it.second->SetUnreachable (true); + } else if (total > 75 && ts > (i2p::context.GetStartupTime () + 600)*1000LL) { + // routers don't expire if less than 25 or uptime is less than 10 minutes + if (i2p::context.IsFloodfill ()) { + if (ts > it.second->GetTimestamp () + 3600*1000LL) { // 1 hour + it.second->SetUnreachable (true); + total--; } - else if (total > 300) - { - if (ts > it.second->GetTimestamp () + 30*3600*1000LL) // 30 hours - { - it.second->SetUnreachable (true); - total--; - } + } else if (total > 300) { + if (ts > it.second->GetTimestamp () + 30*3600*1000LL) { // 30 hours + it.second->SetUnreachable (true); + total--; } - else if (total > 120) - { - if (ts > it.second->GetTimestamp () + 72*3600*1000LL) // 72 hours - { - it.second->SetUnreachable (true); - total--; - } + } else if (total > 120) { + if (ts > it.second->GetTimestamp () + 72*3600*1000LL) { // 72 hours + it.second->SetUnreachable (true); + total--; } } - - if (it.second->IsUnreachable ()) - { - total--; - // delete RI file - if (boost::filesystem::exists (GetFilePath (fullDirectory, it.second.get ()))) - { - boost::filesystem::remove (GetFilePath (fullDirectory, it.second.get ())); - deletedCount++; - } - // delete from floodfills list - if (it.second->IsFloodfill ()) - { - std::unique_lock l(m_FloodfillsMutex); - m_Floodfills.remove (it.second); - } + } + + if (it.second->IsUnreachable ()) { + total--; + // delete RI file + i2p::fs::GetNetDB().Remove(ident); + deletedCount++; + // delete from floodfills list + if (it.second->IsFloodfill ()) { + std::unique_lock l(m_FloodfillsMutex); + m_Floodfills.remove (it.second); } - } - } - if (count > 0) - LogPrint (eLogInfo, "NetDb: ", count, " new/updated routers saved"); + } + } // m_RouterInfos iteration + if (updatedCount > 0) + LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers"); if (deletedCount > 0) { - LogPrint (eLogDebug, "NetDb: ", deletedCount, " routers deleted"); + LogPrint (eLogInfo, "NetDb: deleting ", deletedCount, " unreachable routers"); // clean up RouterInfos table std::unique_lock l(m_RouterInfosMutex); for (auto it = m_RouterInfos.begin (); it != m_RouterInfos.end ();) { - if (it->second->IsUnreachable ()) - { + if (it->second->IsUnreachable ()) { it->second->SaveProfile (); it = m_RouterInfos.erase (it); + continue; } - else - it++; + it++; } } } diff --git a/NetDb.h b/NetDb.h index 237e5be1..855faa3c 100644 --- a/NetDb.h +++ b/NetDb.h @@ -8,7 +8,6 @@ #include #include #include -#include #include "Base.h" #include "Queue.h" #include "I2NPProtocol.h" @@ -23,7 +22,6 @@ namespace i2p { namespace data { - class NetDb { public: @@ -69,8 +67,8 @@ namespace data private: - bool CreateNetDb(boost::filesystem::path directory); void Load (); + bool LoadRouterInfo (const std::string & path); void SaveUpdated (); void Run (); // exploratory thread void Explore (int numDestinations); @@ -90,6 +88,7 @@ namespace data std::list > m_Floodfills; bool m_IsRunning; + uint64_t m_LastLoad; std::thread * m_Thread; i2p::util::Queue > m_Queue; // of I2NPDatabaseStoreMsg From 2b137b43e6d305cdb9d440fbeb65e5abe49d86b4 Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 11 Feb 2016 00:00:00 +0000 Subject: [PATCH 04/23] * i2p::fs migration: I2PControl.* --- I2PControl.cpp | 12 +++++------- I2PControl.h | 3 --- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/I2PControl.cpp b/I2PControl.cpp index 1ea6c8aa..907fa2fd 100644 --- a/I2PControl.cpp +++ b/I2PControl.cpp @@ -13,6 +13,7 @@ #include #endif +#include "FS.h" #include "Log.h" #include "Config.h" #include "NetDb.h" @@ -39,15 +40,12 @@ namespace client // certificate / keys std::string i2pcp_crt; i2p::config::GetOption("i2pcontrol.cert", i2pcp_crt); std::string i2pcp_key; i2p::config::GetOption("i2pcontrol.key", i2pcp_key); - auto path = GetPath (); - // TODO: move this to i2p::fs::expand + if (i2pcp_crt.at(0) != '/') - i2pcp_crt.insert(0, (path / "/").string()); + i2pcp_crt = i2p::fs::DataDirPath(i2pcp_crt); if (i2pcp_key.at(0) != '/') - i2pcp_key.insert(0, (path / "/").string()); - if (!boost::filesystem::exists (i2pcp_crt) || - !boost::filesystem::exists (i2pcp_key)) - { + i2pcp_key = i2p::fs::DataDirPath(i2pcp_key); + if (!i2p::fs::Exists (i2pcp_crt) || !i2p::fs::Exists (i2pcp_key)) { LogPrint (eLogInfo, "I2PControl: creating new certificate for control connection"); CreateCertificate (i2pcp_crt.c_str(), i2pcp_key.c_str()); } else { diff --git a/I2PControl.h b/I2PControl.h index 18592068..714d3aa5 100644 --- a/I2PControl.h +++ b/I2PControl.h @@ -12,8 +12,6 @@ #include #include #include -#include -#include "util.h" namespace i2p { @@ -52,7 +50,6 @@ namespace client void HandleResponseSent (const boost::system::error_code& ecode, std::size_t bytes_transferred, std::shared_ptr socket, std::shared_ptr buf); - boost::filesystem::path GetPath () const { return i2p::util::filesystem::GetDefaultDataDir(); }; void CreateCertificate (const char *crt_path, const char *key_path); private: From bfcb6f577f4a6d63e778f1c46a6d7461a6a57033 Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 11 Feb 2016 00:00:00 +0000 Subject: [PATCH 05/23] * i2p::fs migration: Profiling.* --- Profiling.cpp | 186 ++++++++++++++++++++------------------------------ Profiling.h | 2 - 2 files changed, 73 insertions(+), 115 deletions(-) diff --git a/Profiling.cpp b/Profiling.cpp index 9bfd4de9..d868298f 100644 --- a/Profiling.cpp +++ b/Profiling.cpp @@ -1,8 +1,8 @@ -#include +#include #include #include #include "Base.h" -#include "util.h" +#include "FS.h" #include "Log.h" #include "Profiling.h" @@ -41,101 +41,69 @@ namespace data boost::property_tree::ptree pt; pt.put (PEER_PROFILE_LAST_UPDATE_TIME, boost::posix_time::to_simple_string (m_LastUpdateTime)); pt.put_child (PEER_PROFILE_SECTION_PARTICIPATION, participation); - pt.put_child (PEER_PROFILE_SECTION_USAGE, usage); + pt.put_child (PEER_PROFILE_SECTION_USAGE, usage); // save to file - auto path = i2p::util::filesystem::GetDefaultDataDir() / PEER_PROFILES_DIRECTORY; - if (!boost::filesystem::exists (path)) - { - // Create directory is necessary - if (!boost::filesystem::create_directory (path)) - { - LogPrint (eLogError, "Failed to create directory ", path); - return; - } - const char * chars = GetBase64SubstitutionTable (); // 64 bytes - for (int i = 0; i < 64; i++) - { - auto path1 = path / (std::string ("p") + chars[i]); - if (!boost::filesystem::exists (path1) && !boost::filesystem::create_directory (path1)) - { - LogPrint (eLogError, "Failed to create directory ", path1); - return; - } - } - } - std::string base64 = m_IdentHash.ToBase64 (); - path = path / (std::string ("p") + base64[0]); - auto filename = path / (std::string (PEER_PROFILE_PREFIX) + base64 + ".txt"); - try - { - boost::property_tree::write_ini (filename.string (), pt); - } - catch (std::exception& ex) - { - LogPrint (eLogError, "Can't write ", filename, ": ", ex.what ()); + std::string ident = m_IdentHash.ToBase64 (); + std::string path = i2p::fs::GetPeerProfiles().Path(ident); + + try { + boost::property_tree::write_ini (path, pt); + } catch (std::exception& ex) { + /* boost exception verbose enough */ + LogPrint (eLogError, "Profiling: ", ex.what ()); } - } + } void RouterProfile::Load () { - std::string base64 = m_IdentHash.ToBase64 (); - auto path = i2p::util::filesystem::GetDefaultDataDir() / PEER_PROFILES_DIRECTORY; - path /= std::string ("p") + base64[0]; - auto filename = path / (std::string (PEER_PROFILE_PREFIX) + base64 + ".txt"); - if (boost::filesystem::exists (filename)) - { - boost::property_tree::ptree pt; - try - { - boost::property_tree::read_ini (filename.string (), pt); - } - catch (std::exception& ex) - { - LogPrint (eLogError, "Can't read ", filename, ": ", ex.what ()); - return; - } - try - { - auto t = pt.get (PEER_PROFILE_LAST_UPDATE_TIME, ""); - if (t.length () > 0) - m_LastUpdateTime = boost::posix_time::time_from_string (t); - if ((GetTime () - m_LastUpdateTime).hours () < PEER_PROFILE_EXPIRATION_TIMEOUT) - { - try - { - // read participations - auto participations = pt.get_child (PEER_PROFILE_SECTION_PARTICIPATION); - m_NumTunnelsAgreed = participations.get (PEER_PROFILE_PARTICIPATION_AGREED, 0); - m_NumTunnelsDeclined = participations.get (PEER_PROFILE_PARTICIPATION_DECLINED, 0); - m_NumTunnelsNonReplied = participations.get (PEER_PROFILE_PARTICIPATION_NON_REPLIED, 0); - } - catch (boost::property_tree::ptree_bad_path& ex) - { - LogPrint (eLogWarning, "Missing section ", PEER_PROFILE_SECTION_PARTICIPATION); - } - try - { - // read usage - auto usage = pt.get_child (PEER_PROFILE_SECTION_USAGE); - m_NumTimesTaken = usage.get (PEER_PROFILE_USAGE_TAKEN, 0); - m_NumTimesRejected = usage.get (PEER_PROFILE_USAGE_REJECTED, 0); - } - catch (boost::property_tree::ptree_bad_path& ex) - { - LogPrint (eLogWarning, "Missing section ", PEER_PROFILE_SECTION_USAGE); - } + std::string ident = m_IdentHash.ToBase64 (); + std::string path = i2p::fs::GetPeerProfiles().Path(ident); + boost::property_tree::ptree pt; + + if (!i2p::fs::Exists(path)) { + LogPrint(eLogWarning, "Profiling: no profile yet for ", ident); + return; + } + + try { + boost::property_tree::read_ini (path, pt); + } catch (std::exception& ex) { + /* boost exception verbose enough */ + LogPrint (eLogError, "Profiling: ", ex.what ()); + return; + } + + try { + auto t = pt.get (PEER_PROFILE_LAST_UPDATE_TIME, ""); + if (t.length () > 0) + m_LastUpdateTime = boost::posix_time::time_from_string (t); + if ((GetTime () - m_LastUpdateTime).hours () < PEER_PROFILE_EXPIRATION_TIMEOUT) { + try { + // read participations + auto participations = pt.get_child (PEER_PROFILE_SECTION_PARTICIPATION); + m_NumTunnelsAgreed = participations.get (PEER_PROFILE_PARTICIPATION_AGREED, 0); + m_NumTunnelsDeclined = participations.get (PEER_PROFILE_PARTICIPATION_DECLINED, 0); + m_NumTunnelsNonReplied = participations.get (PEER_PROFILE_PARTICIPATION_NON_REPLIED, 0); + } catch (boost::property_tree::ptree_bad_path& ex) { + LogPrint (eLogWarning, "Profiling: Missing section ", PEER_PROFILE_SECTION_PARTICIPATION, " in profile for ", ident); } - else - *this = RouterProfile (m_IdentHash); - } - catch (std::exception& ex) - { - LogPrint (eLogError, "Can't read profile ", base64, " :", ex.what ()); - } - } - } - + try { + // read usage + auto usage = pt.get_child (PEER_PROFILE_SECTION_USAGE); + m_NumTimesTaken = usage.get (PEER_PROFILE_USAGE_TAKEN, 0); + m_NumTimesRejected = usage.get (PEER_PROFILE_USAGE_REJECTED, 0); + } catch (boost::property_tree::ptree_bad_path& ex) { + LogPrint (eLogWarning, "Missing section ", PEER_PROFILE_SECTION_USAGE, " in profile for ", ident); + } + } else { + *this = RouterProfile (m_IdentHash); + } + } catch (std::exception& ex) { + LogPrint (eLogError, "Profiling: Can't read profile ", ident, " :", ex.what ()); + } + } + void RouterProfile::TunnelBuildResponse (uint8_t ret) { UpdateTime (); @@ -186,29 +154,21 @@ namespace data void DeleteObsoleteProfiles () { - int num = 0; - auto ts = boost::posix_time::second_clock::local_time(); - boost::filesystem::path p (i2p::util::filesystem::GetDataDir()/PEER_PROFILES_DIRECTORY); - if (boost::filesystem::exists (p)) - { - boost::filesystem::directory_iterator end; - for (boost::filesystem::directory_iterator it (p); it != end; ++it) - { - if (boost::filesystem::is_directory (it->status())) - { - for (boost::filesystem::directory_iterator it1 (it->path ()); it1 != end; ++it1) - { - auto lastModified = boost::posix_time::from_time_t (boost::filesystem::last_write_time (it1->path ())); - if ((ts - lastModified).hours () >= PEER_PROFILE_EXPIRATION_TIMEOUT) - { - boost::filesystem::remove (it1->path ()); - num++; - } - } - } - } + struct stat st; + std::time_t now = std::time(nullptr); + + std::vector files; + i2p::fs::GetPeerProfiles().Traverse(files); + for (auto path: files) { + if (stat(path.c_str(), &st) != 0) { + LogPrint(eLogWarning, "Profiling: Can't stat(): ", path); + continue; + } + if (((now - st.st_mtime) / 3600) >= PEER_PROFILE_EXPIRATION_TIMEOUT) { + LogPrint(eLogDebug, "Profiling: removing expired peer profile: ", path); + i2p::fs::Remove(path); + } } - LogPrint (eLogInfo, num, " obsolete profiles deleted"); - } + } } } diff --git a/Profiling.h b/Profiling.h index 0690d6cb..3a65714d 100644 --- a/Profiling.h +++ b/Profiling.h @@ -9,8 +9,6 @@ namespace i2p { namespace data { - const char PEER_PROFILES_DIRECTORY[] = "peerProfiles"; - const char PEER_PROFILE_PREFIX[] = "profile-"; // sections const char PEER_PROFILE_SECTION_PARTICIPATION[] = "participation"; const char PEER_PROFILE_SECTION_USAGE[] = "usage"; From ddd8d4aeb2549a3ceb5d916d348a873613225030 Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 11 Feb 2016 00:00:00 +0000 Subject: [PATCH 06/23] * i2p::fs migration: AddressBook.* --- AddressBook.cpp | 219 ++++++++++++++++++------------------------------ AddressBook.h | 2 - 2 files changed, 81 insertions(+), 140 deletions(-) diff --git a/AddressBook.cpp b/AddressBook.cpp index 1c2f3e24..540fd74c 100644 --- a/AddressBook.cpp +++ b/AddressBook.cpp @@ -5,11 +5,11 @@ #include #include #include -#include #include #include "Base.h" #include "util.h" #include "Identity.h" +#include "FS.h" #include "Log.h" #include "NetDb.h" #include "ClientContext.h" @@ -19,143 +19,96 @@ namespace i2p { namespace client { - + // TODO: this is actually proxy class class AddressBookFilesystemStorage: public AddressBookStorage { public: - - AddressBookFilesystemStorage (); + AddressBookFilesystemStorage () {}; std::shared_ptr GetAddress (const i2p::data::IdentHash& ident) const; void AddAddress (std::shared_ptr address); void RemoveAddress (const i2p::data::IdentHash& ident); int Load (std::map& addresses); int Save (const std::map& addresses); - - private: - - boost::filesystem::path GetPath () const - { - return i2p::util::filesystem::GetDefaultDataDir() / "addressbook"; - } - boost::filesystem::path GetAddressPath (const i2p::data::IdentHash& ident) const - { - auto b32 = ident.ToBase32(); - return GetPath () / (std::string ("b") + b32[0]) / (b32 + ".b32"); - } }; - AddressBookFilesystemStorage::AddressBookFilesystemStorage () - { - auto path = GetPath (); - if (!boost::filesystem::exists (path)) - { - // Create directory is necessary - if (!boost::filesystem::create_directory (path)) - LogPrint (eLogError, "Addressbook: failed to create addressbook directory"); - } - - } - std::shared_ptr AddressBookFilesystemStorage::GetAddress (const i2p::data::IdentHash& ident) const { - auto filename = GetAddressPath (ident); - if (!boost::filesystem::exists (filename)) - { - boost::filesystem::create_directory (filename.parent_path ()); - // try to find in main folder - auto filename1 = GetPath () / (ident.ToBase32 () + ".b32"); - if (!boost::filesystem::exists (filename1)) - { - boost::system::error_code ec; - boost::filesystem::rename (filename1, filename, ec); - if (ec) - LogPrint (eLogError, "Addresbook: couldn't move file ", ec.message ()); - } - else - return nullptr; // address doesn't exist - } - std::ifstream f(filename.string (), std::ifstream::binary); - if (f.is_open ()) - { - f.seekg (0,std::ios::end); - size_t len = f.tellg (); - if (len < i2p::data::DEFAULT_IDENTITY_SIZE) - { - LogPrint (eLogError, "Addresbook: File ", filename, " is too short. ", len); - return nullptr; - } - f.seekg(0, std::ios::beg); - uint8_t * buf = new uint8_t[len]; - f.read((char *)buf, len); - auto address = std::make_shared(buf, len); - delete[] buf; - return address; + std::string filename = i2p::fs::GetAddressBook().Path(ident.ToBase32()); + std::ifstream f(filename, std::ifstream::binary); + if (!f.is_open ()) { + LogPrint(eLogDebug, "Addressbook: Requested, but not found: ", filename); + return nullptr; } - else + + f.seekg (0,std::ios::end); + size_t len = f.tellg (); + if (len < i2p::data::DEFAULT_IDENTITY_SIZE) { + LogPrint (eLogError, "Addresbook: File ", filename, " is too short: ", len); return nullptr; + } + f.seekg(0, std::ios::beg); + uint8_t * buf = new uint8_t[len]; + f.read((char *)buf, len); + auto address = std::make_shared(buf, len); + delete[] buf; + return address; } void AddressBookFilesystemStorage::AddAddress (std::shared_ptr address) { - auto filename = GetAddressPath (address->GetIdentHash ()); - std::ofstream f (filename.string (), std::ofstream::binary | std::ofstream::out); - if (!f.is_open ()) - { - // create subdirectory - if (boost::filesystem::create_directory (filename.parent_path ())) - f.open (filename.string (), std::ofstream::binary | std::ofstream::out); // and try to open again - } - if (f.is_open ()) - { - size_t len = address->GetFullLen (); - uint8_t * buf = new uint8_t[len]; - address->ToBuffer (buf, len); - f.write ((char *)buf, len); - delete[] buf; + std::string path = i2p::fs::GetAddressBook().Path( address->GetIdentHash().ToBase32() ); + std::ofstream f (path, std::ofstream::binary | std::ofstream::out); + if (!f.is_open ()) { + LogPrint (eLogError, "Addresbook: can't open file ", path); + return; } - else - LogPrint (eLogError, "Addresbook: can't open file ", filename); + size_t len = address->GetFullLen (); + uint8_t * buf = new uint8_t[len]; + address->ToBuffer (buf, len); + f.write ((char *)buf, len); + delete[] buf; } void AddressBookFilesystemStorage::RemoveAddress (const i2p::data::IdentHash& ident) { - auto filename = GetAddressPath (ident); - if (boost::filesystem::exists (filename)) - boost::filesystem::remove (filename); + i2p::fs::GetAddressBook().Remove( ident.ToBase32() ); } int AddressBookFilesystemStorage::Load (std::map& addresses) { int num = 0; - auto filename = GetPath () / "addresses.csv"; - std::ifstream f (filename.string (), std::ifstream::in); // in text mode - if (f.is_open ()) - { - addresses.clear (); - while (!f.eof ()) + std::string s; + std::string index = i2p::fs::GetAddressBook().IndexPath(); + std::ifstream f (index, std::ifstream::in); // in text mode + + if (f.is_open ()) { + LogPrint(eLogInfo, "Addressbook: using index file ", index); + } else { + LogPrint(eLogWarning, "Addressbook: Can't open ", index); + return 0; + } + + addresses.clear (); + while (!f.eof ()) { + getline(f, s); + if (!s.length()) + continue; // skip empty line + + std::size_t pos = s.find(','); + if (pos != std::string::npos) { - std::string s; - getline(f, s); - if (!s.length()) - continue; // skip empty line + std::string name = s.substr(0, pos++); + std::string addr = s.substr(pos); - size_t pos = s.find(','); - if (pos != std::string::npos) - { - std::string name = s.substr(0, pos++); - std::string addr = s.substr(pos); - - i2p::data::IdentHash ident; - ident.FromBase32 (addr); - addresses[name] = ident; - num++; - } - } - LogPrint (eLogInfo, "Addressbook: ", num, " addresses loaded"); + i2p::data::IdentHash ident; + ident.FromBase32 (addr); + addresses[name] = ident; + num++; + } } - else - LogPrint (eLogWarning, "Addressbook: ", filename, " not found"); + + LogPrint (eLogInfo, "Addressbook: ", num, " addresses loaded from storage"); return num; } @@ -167,24 +120,24 @@ namespace client } int num = 0; - auto filename = GetPath () / "addresses.csv"; - std::ofstream f (filename.string (), std::ofstream::out); // in text mode - if (f.is_open ()) - { - for (auto it: addresses) - { - f << it.first << "," << it.second.ToBase32 () << std::endl; - num++; - } - LogPrint (eLogInfo, "Addressbook: ", num, " addresses saved"); + std::string index = i2p::fs::GetAddressBook().IndexPath(); + std::ofstream f (index, std::ofstream::out); // in text mode + + if (!f.is_open ()) { + LogPrint (eLogWarning, "Addressbook: Can't open ", index); + return 0; + } + + for (auto it: addresses) { + f << it.first << "," << it.second.ToBase32 () << std::endl; + num++; } - else - LogPrint (eLogError, "Addresbook: can't open file ", filename); + LogPrint (eLogInfo, "Addressbook: ", num, " addresses saved"); return num; } //--------------------------------------------------------------------- - AddressBook::AddressBook (): m_Storage (nullptr), m_IsLoaded (false), m_IsDownloading (false), + AddressBook::AddressBook (): m_Storage(new AddressBookFilesystemStorage), m_IsLoaded (false), m_IsDownloading (false), m_DefaultSubscription (nullptr), m_SubscriptionsUpdateTimer (nullptr) { } @@ -239,11 +192,6 @@ namespace client m_Subscriptions.clear (); } - AddressBookStorage * AddressBook::CreateStorage () - { - return new AddressBookFilesystemStorage (); - } - bool AddressBook::GetIdentHash (const std::string& address, i2p::data::IdentHash& ident) { auto pos = address.find(".b32.i2p"); @@ -287,8 +235,6 @@ namespace client { auto ident = std::make_shared(); ident->FromBase64 (base64); - if (!m_Storage) - m_Storage = CreateStorage (); m_Storage->AddAddress (ident); m_Addresses[address] = ident->GetIdentHash (); LogPrint (eLogInfo, "Addressbook: added ", address," -> ", ToAddress(ident->GetIdentHash ())); @@ -296,15 +242,11 @@ namespace client void AddressBook::InsertAddress (std::shared_ptr address) { - if (!m_Storage) - m_Storage = CreateStorage (); m_Storage->AddAddress (address); } std::shared_ptr AddressBook::GetAddress (const std::string& address) { - if (!m_Storage) - m_Storage = CreateStorage (); i2p::data::IdentHash ident; if (!GetIdentHash (address, ident)) return nullptr; return m_Storage->GetAddress (ident); @@ -312,16 +254,14 @@ namespace client void AddressBook::LoadHosts () { - if (!m_Storage) - m_Storage = CreateStorage (); if (m_Storage->Load (m_Addresses) > 0) { m_IsLoaded = true; return; } - // try hosts.txt first - std::ifstream f (i2p::util::filesystem::GetFullPath ("hosts.txt").c_str (), std::ifstream::in); // in text mode + // then try hosts.txt + std::ifstream f (i2p::fs::DataDirPath("hosts.txt"), std::ifstream::in); // in text mode if (f.is_open ()) { LoadHostsFromStream (f); @@ -384,7 +324,7 @@ namespace client { if (!m_Subscriptions.size ()) { - std::ifstream f (i2p::util::filesystem::GetFullPath ("subscriptions.txt").c_str (), std::ifstream::in); // in text mode + std::ifstream f (i2p::fs::DataDirPath ("subscriptions.txt"), std::ifstream::in); // in text mode if (f.is_open ()) { std::string s; @@ -443,7 +383,10 @@ namespace client if (ecode != boost::asio::error::operation_aborted) { auto dest = i2p::client::context.GetSharedLocalDestination (); - if (!dest) return; + if (!dest) { + LogPrint(eLogWarning, "Addressbook: missing local destination, skip subscription update"); + return; + } if (m_IsLoaded && !m_IsDownloading && dest->IsReady () && !m_Subscriptions.empty ()) { // pick random subscription @@ -495,7 +438,7 @@ namespace client }); if (newDataReceived.wait_for (l, std::chrono::seconds (SUBSCRIPTION_REQUEST_TIMEOUT)) == std::cv_status::timeout) { - LogPrint (eLogError, "Subscription LeaseSet request timeout expired"); + LogPrint (eLogError, "Addressbook: Subscription LeaseSet request timeout expired"); i2p::client::context.GetSharedLocalDestination ()->CancelDestinationRequest (ident); } } @@ -595,7 +538,7 @@ namespace client LogPrint (eLogError, "Addressbook: Can't resolve ", u.host_); if (!success) - LogPrint (eLogError, "Addressbook: download failed"); + LogPrint (eLogError, "Addressbook: download hosts.txt from ", m_Link, " failed"); m_Book.DownloadComplete (success); } diff --git a/AddressBook.h b/AddressBook.h index b50b9d9b..9ddce82a 100644 --- a/AddressBook.h +++ b/AddressBook.h @@ -10,7 +10,6 @@ #include #include #include "Base.h" -#include "util.h" #include "Identity.h" #include "Log.h" @@ -65,7 +64,6 @@ namespace client void StartSubscriptions (); void StopSubscriptions (); - AddressBookStorage * CreateStorage (); void LoadHosts (); void LoadSubscriptions (); From 79bf44b3f52bf26bd994ab226196439fff2e4180 Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 11 Feb 2016 00:00:00 +0000 Subject: [PATCH 07/23] * i2p::fs migration: ClientContext, Destination, HTTPServer, Reseed, RouterContext --- ClientContext.cpp | 26 +++++++++++++------------- Destination.cpp | 43 ++++++++++++++++++++----------------------- HTTPServer.cpp | 3 ++- Reseed.cpp | 30 +++++++++++++++--------------- RouterContext.cpp | 9 +++++---- 5 files changed, 55 insertions(+), 56 deletions(-) diff --git a/ClientContext.cpp b/ClientContext.cpp index 6d05cde1..072e6e24 100644 --- a/ClientContext.cpp +++ b/ClientContext.cpp @@ -3,7 +3,7 @@ #include #include #include "Config.h" -#include "util.h" +#include "FS.h" #include "Log.h" #include "Identity.h" #include "ClientContext.h" @@ -152,10 +152,10 @@ namespace client m_SharedLocalDestination = nullptr; } - void ClientContext::LoadPrivateKeys (i2p::data::PrivateKeys& keys, const std::string& filename, i2p::data::SigningKeyType sigType) + void ClientContext::LoadPrivateKeys (i2p::data::PrivateKeys& keys, const std::string& filename, i2p::data::SigningKeyType sigType) { - std::string fullPath = i2p::util::filesystem::GetFullPath (filename); - std::ifstream s(fullPath.c_str (), std::ifstream::binary); + std::string fullPath = i2p::fs::DataDirPath (filename); + std::ifstream s(fullPath, std::ifstream::binary); if (s.is_open ()) { s.seekg (0, std::ios::end); @@ -256,14 +256,14 @@ namespace client void ClientContext::ReadTunnels () { boost::property_tree::ptree pt; - std::string pathTunnelsConfigFile = i2p::util::filesystem::GetTunnelsConfigFile().string(); - try - { - boost::property_tree::read_ini (pathTunnelsConfigFile, pt); - } - catch (std::exception& ex) - { - LogPrint (eLogWarning, "Clients: Can't read ", pathTunnelsConfigFile, ": ", ex.what ()); + std::string pathTunConf; + i2p::config::GetOption("tunconf", pathTunConf); + if (pathTunConf == "") + return; + try { + boost::property_tree::read_ini (pathTunConf, pt); + } catch (std::exception& ex) { + LogPrint (eLogWarning, "Clients: Can't read ", pathTunConf, ": ", ex.what ()); return; } @@ -351,7 +351,7 @@ namespace client numServerTunnels++; } else - LogPrint (eLogWarning, "Clients: Unknown section type=", type, " of ", name, " in ", pathTunnelsConfigFile); + LogPrint (eLogWarning, "Clients: Unknown section type=", type, " of ", name, " in ", pathTunConf); } catch (std::exception& ex) diff --git a/Destination.cpp b/Destination.cpp index e7e9b93c..3509fe80 100644 --- a/Destination.cpp +++ b/Destination.cpp @@ -2,8 +2,9 @@ #include #include #include + #include "Log.h" -#include "util.h" +#include "FS.h" #include "Crypto.h" #include "Timestamp.h" #include "NetDb.h" @@ -690,30 +691,26 @@ namespace client void ClientDestination::PersistTemporaryKeys () { - auto path = i2p::util::filesystem::GetDefaultDataDir() / "destinations"; - auto filename = path / (GetIdentHash ().ToBase32 () + ".dat"); - std::ifstream f(filename.string (), std::ifstream::binary); - if (f) - { - f.read ((char *)m_EncryptionPublicKey, 256); + std::string ident = GetIdentHash().ToBase32(); + std::string path = i2p::fs::DataDirPath("destinations", (ident + ".dat")); + std::ifstream f(path, std::ifstream::binary); + + if (f) { + f.read ((char *)m_EncryptionPublicKey, 256); f.read ((char *)m_EncryptionPrivateKey, 256); + return; } - if (!f) - { - LogPrint (eLogInfo, "Creating new temporary keys for address ", GetIdentHash ().ToBase32 ()); - i2p::crypto::GenerateElGamalKeyPair(m_EncryptionPrivateKey, m_EncryptionPublicKey); - if (!boost::filesystem::exists (path)) - { - if (!boost::filesystem::create_directory (path)) - LogPrint (eLogError, "Failed to create destinations directory"); - } - std::ofstream f1 (filename.string (), std::ofstream::binary | std::ofstream::out); - if (f1) - { - f1.write ((char *)m_EncryptionPublicKey, 256); - f1.write ((char *)m_EncryptionPrivateKey, 256); - } - } + + LogPrint (eLogInfo, "Destination: Creating new temporary keys for address ", ident, ".b32.i2p"); + i2p::crypto::GenerateElGamalKeyPair(m_EncryptionPrivateKey, m_EncryptionPublicKey); + + std::ofstream f1 (path, std::ofstream::binary | std::ofstream::out); + if (f1) { + f1.write ((char *)m_EncryptionPublicKey, 256); + f1.write ((char *)m_EncryptionPrivateKey, 256); + return; + } + LogPrint(eLogError, "Destinations: Can't save keys to ", path); } } } diff --git a/HTTPServer.cpp b/HTTPServer.cpp index f04b0b1c..85dbec42 100644 --- a/HTTPServer.cpp +++ b/HTTPServer.cpp @@ -4,6 +4,7 @@ #include #include #include "Base.h" +#include "FS.h" #include "Log.h" #include "Tunnel.h" #include "TransitTunnel.h" @@ -420,7 +421,7 @@ namespace util s << " (" << i2p::transport::transports.GetInBandwidth () <<" Bps)
\r\n"; s << "Sent: " << i2p::transport::transports.GetTotalSentBytes ()/1000 << "K"; s << " (" << i2p::transport::transports.GetOutBandwidth () <<" Bps)
\r\n"; - s << "Data path: " << i2p::util::filesystem::GetDataDir().string() << "
\r\n
\r\n"; + s << "Data path: " << i2p::fs::GetDataDir() << "
\r\n
\r\n"; s << "Our external address:" << "
\r\n" ; for (auto& address : i2p::context.GetRouterInfo().GetAddresses()) { diff --git a/Reseed.cpp b/Reseed.cpp index 80cc6a45..506b0a0f 100644 --- a/Reseed.cpp +++ b/Reseed.cpp @@ -2,15 +2,16 @@ #include #include #include -#include #include #include #include #include #include #include + #include "I2PEndian.h" #include "Reseed.h" +#include "FS.h" #include "Log.h" #include "Identity.h" #include "Crypto.h" @@ -346,23 +347,22 @@ namespace data void Reseeder::LoadCertificates () { - boost::filesystem::path reseedDir = i2p::util::filesystem::GetCertificatesDir() / "reseed"; - - if (!boost::filesystem::exists (reseedDir)) - { - LogPrint (eLogWarning, "Reseed: certificates not loaded, ", reseedDir, " doesn't exist"); + std::string certDir = i2p::fs::DataDirPath("certificates", "reseed"); + std::vector files; + int numCertificates = 0; + + if (!i2p::fs::ReadDir(certDir, files)) { + LogPrint(eLogWarning, "Reseed: Can't load reseed certificates from ", certDir); return; } - int numCertificates = 0; - boost::filesystem::directory_iterator end; // empty - for (boost::filesystem::directory_iterator it (reseedDir); it != end; ++it) - { - if (boost::filesystem::is_regular_file (it->status()) && it->path ().extension () == ".crt") - { - LoadCertificate (it->path ().string ()); - numCertificates++; - } + for (const std::string & file : files) { + if (file.compare(file.size() - 4, 4, ".crt") != 0) { + LogPrint(eLogWarning, "Reseed: ignoring file ", file); + continue; + } + LoadCertificate (file); + numCertificates++; } LogPrint (eLogInfo, "Reseed: ", numCertificates, " certificates loaded"); } diff --git a/RouterContext.cpp b/RouterContext.cpp index 90961948..67f99d29 100644 --- a/RouterContext.cpp +++ b/RouterContext.cpp @@ -5,6 +5,7 @@ #include "Timestamp.h" #include "I2NPProtocol.h" #include "NetDb.h" +#include "FS.h" #include "util.h" #include "version.h" #include "Log.h" @@ -64,7 +65,7 @@ namespace i2p void RouterContext::UpdateRouterInfo () { m_RouterInfo.CreateBuffer (m_Keys); - m_RouterInfo.SaveToFile (i2p::util::filesystem::GetFullPath (ROUTER_INFO)); + m_RouterInfo.SaveToFile (i2p::fs::DataDirPath (ROUTER_INFO)); m_LastUpdateTime = i2p::util::GetSecondsSinceEpoch (); } @@ -292,7 +293,7 @@ namespace i2p bool RouterContext::Load () { - std::ifstream fk (i2p::util::filesystem::GetFullPath (ROUTER_KEYS).c_str (), std::ifstream::binary | std::ifstream::in); + std::ifstream fk (i2p::fs::DataDirPath (ROUTER_KEYS), std::ifstream::in | std::ifstream::binary); if (!fk.is_open ()) return false; fk.seekg (0, std::ios::end); size_t len = fk.tellg(); @@ -312,7 +313,7 @@ namespace i2p delete[] buf; } - i2p::data::RouterInfo routerInfo(i2p::util::filesystem::GetFullPath (ROUTER_INFO)); // TODO + i2p::data::RouterInfo routerInfo(i2p::fs::DataDirPath (ROUTER_INFO)); // TODO m_RouterInfo.SetRouterIdentity (GetIdentity ()); m_RouterInfo.Update (routerInfo.GetBuffer (), routerInfo.GetBufferLen ()); m_RouterInfo.SetProperty ("coreVersion", I2P_VERSION); @@ -331,7 +332,7 @@ namespace i2p void RouterContext::SaveKeys () { // save in the same format as .dat files - std::ofstream fk (i2p::util::filesystem::GetFullPath (ROUTER_KEYS).c_str (), std::ofstream::binary | std::ofstream::out); + std::ofstream fk (i2p::fs::DataDirPath (ROUTER_KEYS), std::ofstream::binary | std::ofstream::out); size_t len = m_Keys.GetFullLen (); uint8_t * buf = new uint8_t[len]; m_Keys.ToBuffer (buf, len); From 97c136d04391e203cfe0a4404a126ce9ee28c455 Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 11 Feb 2016 00:00:00 +0000 Subject: [PATCH 08/23] * i2p::fs migration: Daemon, DaemonLinux, api (#290) --- Daemon.cpp | 29 ++++++++++++++--------------- DaemonLinux.cpp | 9 ++++----- api.cpp | 12 +++++++++--- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/Daemon.cpp b/Daemon.cpp index 242f4bbf..4b1d44b3 100644 --- a/Daemon.cpp +++ b/Daemon.cpp @@ -5,6 +5,7 @@ #include "Config.h" #include "Log.h" +#include "FS.h" #include "Base.h" #include "version.h" #include "Transports.h" @@ -14,7 +15,6 @@ #include "Tunnel.h" #include "NetDb.h" #include "Garlic.h" -#include "util.h" #include "Streaming.h" #include "Destination.h" #include "HTTPServer.h" @@ -63,9 +63,17 @@ namespace i2p i2p::config::Init(); i2p::config::ParseCmdline(argc, argv); - std::string config = i2p::util::filesystem::GetConfigFile().string(); - std::string tunconf = i2p::util::filesystem::GetTunnelsConfigFile().string(); - std::string datadir = i2p::util::filesystem::GetDataDir().string(); + std::string config; i2p::config::GetOption("conf", config); + std::string tunconf; i2p::config::GetOption("tunconf", tunconf); + std::string datadir; i2p::config::GetOption("datadir", datadir); + i2p::fs::DetectDataDir(datadir, IsService()); + i2p::fs::Init(); + + datadir = i2p::fs::GetDataDir(); + if (config == "") + config = i2p::fs::DataDirPath("i2p.conf"); + if (tunconf == "") + tunconf = i2p::fs::DataDirPath("tunnels.cfg"); i2p::config::ParseConfig(config); i2p::config::Finalize(); @@ -144,18 +152,9 @@ namespace i2p if (isDaemon && (logs == "" || logs == "stdout")) logs = "file"; - if (logs == "file") - { + if (logs == "file") { if (logfile == "") - { - // use autodetect of logfile - logfile = IsService () ? "/var/log" : i2p::util::filesystem::GetDataDir().string(); -#ifndef _WIN32 - logfile.append("/i2pd.log"); -#else - logfile.append("\\i2pd.log"); -#endif - } + logfile = i2p::fs::DataDirPath("i2pd.log"); StartLog (logfile); } else { // use stdout diff --git a/DaemonLinux.cpp b/DaemonLinux.cpp index 9382a878..2ccbfe38 100644 --- a/DaemonLinux.cpp +++ b/DaemonLinux.cpp @@ -9,8 +9,8 @@ #include #include "Config.h" +#include "FS.h" #include "Log.h" -#include "util.h" void handle_signal(int sig) { @@ -55,7 +55,7 @@ namespace i2p LogPrint(eLogError, "Daemon: could not create process group."); return false; } - std::string d(i2p::util::filesystem::GetDataDir().string ()); // make a copy + std::string d = i2p::fs::GetDataDir(); if (chdir(d.c_str()) != 0) { LogPrint(eLogError, "Daemon: could not chdir: ", strerror(errno)); @@ -75,8 +75,7 @@ namespace i2p // this code is c-styled and a bit ugly, but we need fd for locking pidfile std::string pidfile; i2p::config::GetOption("pidfile", pidfile); if (pidfile == "") { - pidfile = IsService () ? "/var/run" : i2p::util::filesystem::GetDataDir().string(); - pidfile.append("/i2pd.pid"); + pidfile = i2p::fs::DataDirPath("i2pd.pid"); } if (pidfile != "") { pidFH = open(pidfile.c_str(), O_RDWR | O_CREAT, 0600); @@ -115,7 +114,7 @@ namespace i2p bool DaemonLinux::stop() { - unlink(pidfile.c_str()); + i2p::fs::Remove(pidfile); return Daemon_Singleton::stop(); } diff --git a/api.cpp b/api.cpp index 0f4e1799..3e037c02 100644 --- a/api.cpp +++ b/api.cpp @@ -9,7 +9,7 @@ #include "Identity.h" #include "Destination.h" #include "Crypto.h" -#include "util.h" +#include "FS.h" #include "api.h" namespace i2p @@ -18,10 +18,16 @@ namespace api { void InitI2P (int argc, char* argv[], const char * appName) { - i2p::util::filesystem::SetAppName (appName); i2p::config::Init (); i2p::config::ParseCmdline (argc, argv); i2p::config::Finalize (); + + std::string datadir; i2p::config::GetOption("datadir", datadir); + + i2p::fs::SetAppName (appName); + i2p::fs::DetectDataDir(datadir, false); + i2p::fs::Init(); + i2p::crypto::InitCrypto (); i2p::context.Init (); } @@ -36,7 +42,7 @@ namespace api if (logStream) StartLog (logStream); else - StartLog (i2p::util::filesystem::GetFullPath (i2p::util::filesystem::GetAppName () + ".log")); + StartLog (i2p::fs::DataDirPath (i2p::fs::GetAppName () + ".log")); LogPrint(eLogInfo, "API: starting NetDB"); i2p::data::netdb.Start(); LogPrint(eLogInfo, "API: starting Transports"); From 4b8465613399a2e948f94f8c77227e6df12c76ca Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 11 Feb 2016 00:00:00 +0000 Subject: [PATCH 09/23] * i2p::fs migration: drop unused code from util.* (#314) --- util.cpp | 131 ------------------------------------------------------- util.h | 17 -------- 2 files changed, 148 deletions(-) diff --git a/util.cpp b/util.cpp index 1ea5fcd7..7c11d280 100644 --- a/util.cpp +++ b/util.cpp @@ -6,13 +6,7 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include "Config.h" #include "util.h" #include "Log.h" @@ -67,131 +61,6 @@ namespace i2p { namespace util { -namespace filesystem -{ - std::string appName ("i2pd"); - - void SetAppName (const std::string& name) - { - appName = name; - } - - std::string GetAppName () - { - return appName; - } - - const boost::filesystem::path &GetDataDir() - { - static boost::filesystem::path path; - - // TODO: datadir parameter is useless because GetDataDir is called before OptionParser - // and mapArgs is not initialized yet - /* - std::string datadir; i2p::config::GetOption("datadir", datadir); - if (datadir != "") - path = boost::filesystem::system_complete(datadir); - else */ - path = GetDefaultDataDir(); - - if (!boost::filesystem::exists( path )) - { - // Create data directory - if (!boost::filesystem::create_directory( path )) - { - LogPrint(eLogError, "FS: Failed to create data directory!"); - path = ""; - return path; - } - } - if (!boost::filesystem::is_directory(path)) - path = GetDefaultDataDir(); - return path; - } - - std::string GetFullPath (const std::string& filename) - { - std::string fullPath = GetDataDir ().string (); -#ifndef _WIN32 - fullPath.append ("/"); -#else - fullPath.append ("\\"); -#endif - fullPath.append (filename); - return fullPath; - } - - boost::filesystem::path GetConfigFile() - { - std::string config; i2p::config::GetOption("conf", config); - if (config != "") { - /* config file set with cmdline */ - boost::filesystem::path path(config); - return path; - } - /* else - try autodetect */ - boost::filesystem::path path("i2p.conf"); - path = GetDataDir() / path; - if (!boost::filesystem::exists(path)) - path = ""; /* reset */ - return path; - } - - boost::filesystem::path GetTunnelsConfigFile() - { - std::string tunconf; i2p::config::GetOption("tunconf", tunconf); - if (tunconf != "") { - /* config file set with cmdline */ - boost::filesystem::path path(tunconf); - return path; - } - /* else - try autodetect */ - boost::filesystem::path path("tunnels.cfg"); - path = GetDataDir() / path; - if (!boost::filesystem::exists(path)) - path = ""; /* reset */ - return path; - } - - boost::filesystem::path GetDefaultDataDir() - { - // Windows < Vista: C:\Documents and Settings\Username\Application Data\i2pd - // Windows >= Vista: C:\Users\Username\AppData\Roaming\i2pd - // Mac: ~/Library/Application Support/i2pd - // Unix: ~/.i2pd or /var/lib/i2pd is system=1 -#ifdef WIN32 - // Windows - char localAppData[MAX_PATH]; - SHGetFolderPath(NULL, CSIDL_APPDATA, 0, NULL, localAppData); - return boost::filesystem::path(std::string(localAppData) + "\\" + appName); -#else /* UNIX */ - bool service; i2p::config::GetOption("service", service); - if (service) // use system folder - return boost::filesystem::path(std::string ("/var/lib/") + appName); - boost::filesystem::path pathRet; - char* pszHome = getenv("HOME"); - if (pszHome == NULL || strlen(pszHome) == 0) - pathRet = boost::filesystem::path("/"); - else - pathRet = boost::filesystem::path(pszHome); -#ifdef MAC_OSX - // Mac - pathRet /= "Library/Application Support"; - boost::filesystem::create_directory(pathRet); - return pathRet / appName; -#else /* Other Unix */ - // Unix - return pathRet / (std::string (".") + appName); -#endif -#endif /* UNIX */ - } - - boost::filesystem::path GetCertificatesDir() - { - return GetDataDir () / "certificates"; - } -} - namespace http { std::string GetHttpContent (std::istream& response) diff --git a/util.h b/util.h index 0377ef8d..d1d7f41a 100644 --- a/util.h +++ b/util.h @@ -5,28 +5,11 @@ #include #include #include -#include -#include - -#define PAIRTYPE(t1, t2) std::pair namespace i2p { namespace util { - namespace filesystem - { - void SetAppName (const std::string& name); - std::string GetAppName (); - - const boost::filesystem::path &GetDataDir(); - std::string GetFullPath (const std::string& filename); - boost::filesystem::path GetDefaultDataDir(); - boost::filesystem::path GetConfigFile(); - boost::filesystem::path GetTunnelsConfigFile(); - boost::filesystem::path GetCertificatesDir(); - } - namespace http { const char ETAG[] = "ETag"; From 68cc75cada8bb5b6aacba5136b650a0bfb0235ee Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 18 Feb 2016 00:00:00 +0000 Subject: [PATCH 10/23] * Base.cpp : add T32 character set + accessor --- Base.cpp | 14 +++++++++++++- Base.h | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Base.cpp b/Base.cpp index 1479d62f..dbf887cf 100644 --- a/Base.cpp +++ b/Base.cpp @@ -6,6 +6,18 @@ namespace i2p { namespace data { + static const char T32[32] = { + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'k', 'k', 'l', 'm', 'n', 'o', 'p', + 'q', 'r', 't', 't', 'u', 'v', 'w', 'x', + 'y', 'z', '2', '3', '4', '5', '6', '7', + }; + + const char * GetBase32SubstitutionTable () + { + return T32; + } + static void iT64Build(void); /* @@ -16,7 +28,7 @@ namespace data * Direct Substitution Table */ - static char T64[64] = { + static const char T64[64] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', diff --git a/Base.h b/Base.h index d598542b..0f48a78a 100644 --- a/Base.h +++ b/Base.h @@ -12,6 +12,7 @@ namespace data { size_t ByteStreamToBase64 (const uint8_t * InBuffer, size_t InCount, char * OutBuffer, size_t len); size_t Base64ToByteStream (const char * InBuffer, size_t InCount, uint8_t * OutBuffer, size_t len ); + const char * GetBase32SubstitutionTable (); const char * GetBase64SubstitutionTable (); size_t Base32ToByteStream (const char * inBuf, size_t len, uint8_t * outBuf, size_t outLen); From f190ee951c96a991d687eb569315856b1ed59a7f Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 18 Feb 2016 00:00:00 +0000 Subject: [PATCH 11/23] * use characters sets from Base.cpp - remove ABook class --- FS.cpp | 34 ++++------------------------------ FS.h | 10 ---------- 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/FS.cpp b/FS.cpp index 758b81c8..4a1b670a 100644 --- a/FS.cpp +++ b/FS.cpp @@ -24,24 +24,6 @@ namespace fs { #endif HashedStorage NetDB("netDb", "r", "routerInfo-", "dat"); HashedStorage Peers("peerProfiles", "p", "profile-", "txt"); - ABookStorage ABook("addressbook", "b", "", "b32"); - - static const char T32[32] = { - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', - 'i', 'k', 'k', 'l', 'm', 'n', 'o', 'p', - 'q', 'r', 't', 't', 'u', 'v', 'w', 'x', - 'y', 'z', '2', '3', '4', '5', '6', '7', - }; - static const char T64[64] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', - 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', - 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', '-', '~' - }; const std::string & GetAppName () { return appName; @@ -90,12 +72,10 @@ namespace fs { if (boost::filesystem::exists(destinations)) boost::filesystem::create_directory(destinations); - NetDB.SetRoot(dataDir); - NetDB.Init(T64, 64); - Peers.SetRoot(dataDir); - Peers.Init(T64, 64); - ABook.SetRoot(dataDir); - ABook.Init(T32, 32); + NetDB.SetPlace(dataDir); + NetDB.Init(i2p::data::GetBase64SubstitutionTable(), 64); + Peers.SetPlace(dataDir); + Peers.Init(i2p::data::GetBase64SubstitutionTable(), 64); return true; } @@ -177,13 +157,7 @@ namespace fs { } } - std::string ABookStorage::IndexPath() { - std::string path = root + i2p::fs::dirSep + "addresses.csv"; - return path; - } - HashedStorage & GetNetDB() { return NetDB; } HashedStorage & GetPeerProfiles() { return Peers; } - ABookStorage & GetAddressBook() { return ABook; } } // fs } // i2p diff --git a/FS.h b/FS.h index 80b07353..bb5f44b8 100644 --- a/FS.h +++ b/FS.h @@ -49,15 +49,6 @@ namespace fs { void Traverse(std::vector & files); }; - /** @brief Slightly extended HashedStorage */ - class ABookStorage : public HashedStorage { - public: - ABookStorage(const char *n, const char *p1, const char *p2, const char *s): - HashedStorage(n, p1, p2, s) {}; - - std::string IndexPath(); - }; - /** @brief Returns current application name, default 'i2pd' */ const std::string & GetAppName (); /** @brief Set applicaton name, affects autodetection of datadir */ @@ -138,6 +129,5 @@ namespace fs { /* accessors */ HashedStorage & GetNetDB(); HashedStorage & GetPeerProfiles(); - ABookStorage & GetAddressBook(); } // fs } // i2p From 2b92a039bb81e32eb5d22e971f9cb3ae57b7e78b Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 18 Feb 2016 00:00:00 +0000 Subject: [PATCH 12/23] * FS.h : more comments --- FS.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/FS.h b/FS.h index bb5f44b8..9d7f0e9b 100644 --- a/FS.h +++ b/FS.h @@ -23,6 +23,8 @@ namespace fs { * const char alphabet[8] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}; * auto h = HashedStorage("name", "y", "z-", ".txt"); * h.SetRoot("/tmp/hs-test"); + * h.GetName() -> gives "name" + * h.GetRoot() -> gives "/tmp/hs-test/name" * h.Init(alphabet, 8); <- creates needed dirs, 8 is size of alphabet * h.Path("abcd"); <- returns /tmp/hs-test/name/ya/z-abcd.txt * h.Remove("abcd"); <- removes /tmp/hs-test/name/ya/z-abcd.txt, if it exists @@ -31,21 +33,27 @@ namespace fs { */ class HashedStorage { protected: - std::string root; - std::string name; - std::string prefix1; - std::string prefix2; - std::string suffix; + std::string root; /**< path to storage with it's name included */ + std::string name; /**< name of the storage */ + std::string prefix1; /**< hashed directory prefix */ + std::string prefix2; /**< prefix of file in storage */ + std::string suffix; /**< suffix of file in storage (extension) */ public: HashedStorage(const char *n, const char *p1, const char *p2, const char *s): name(n), prefix1(p1), prefix2(p2), suffix(s) {}; + /** create subdirs in storage */ bool Init(const char* chars, size_t cnt); const std::string & GetRoot() { return this->root; } + const std::string & GetName() { return this->name; } + /** set directory where to place storage directory */ void SetRoot(const std::string & path); + /** path to file with given ident */ std::string Path(const std::string & ident); + /** remove file by ident */ void Remove(const std::string & ident); + /** find all files in storage and store list in provided vector */ void Traverse(std::vector & files); }; From 464a2281062d0f3dd24bb31042049069ad7933a2 Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 18 Feb 2016 00:00:00 +0000 Subject: [PATCH 13/23] * FS.cpp : rename method --- FS.cpp | 2 +- FS.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/FS.cpp b/FS.cpp index 4a1b670a..c77cbe6d 100644 --- a/FS.cpp +++ b/FS.cpp @@ -104,7 +104,7 @@ namespace fs { return boost::filesystem::remove(path); } - void HashedStorage::SetRoot(const std::string &path) { + void HashedStorage::SetPlace(const std::string &path) { root = path + i2p::fs::dirSep + name; } diff --git a/FS.h b/FS.h index 9d7f0e9b..64b50940 100644 --- a/FS.h +++ b/FS.h @@ -22,7 +22,7 @@ namespace fs { * * const char alphabet[8] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}; * auto h = HashedStorage("name", "y", "z-", ".txt"); - * h.SetRoot("/tmp/hs-test"); + * h.SetPlace("/tmp/hs-test"); * h.GetName() -> gives "name" * h.GetRoot() -> gives "/tmp/hs-test/name" * h.Init(alphabet, 8); <- creates needed dirs, 8 is size of alphabet @@ -48,7 +48,7 @@ namespace fs { const std::string & GetRoot() { return this->root; } const std::string & GetName() { return this->name; } /** set directory where to place storage directory */ - void SetRoot(const std::string & path); + void SetPlace(const std::string & path); /** path to file with given ident */ std::string Path(const std::string & ident); /** remove file by ident */ From 138d57143afc19df2e12ca72f61d3a0d25dcf3b1 Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 18 Feb 2016 00:00:00 +0000 Subject: [PATCH 14/23] * FS.cpp : add const to accessors --- FS.cpp | 2 +- FS.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/FS.cpp b/FS.cpp index c77cbe6d..f6125954 100644 --- a/FS.cpp +++ b/FS.cpp @@ -124,7 +124,7 @@ namespace fs { return true; } - std::string HashedStorage::Path(const std::string & ident) { + std::string HashedStorage::Path(const std::string & ident) const { std::string safe_ident = ident; std::replace(safe_ident.begin(), safe_ident.end(), '/', '-'); std::replace(safe_ident.begin(), safe_ident.end(), '\\', '-'); diff --git a/FS.h b/FS.h index 64b50940..0c4e246a 100644 --- a/FS.h +++ b/FS.h @@ -45,12 +45,12 @@ namespace fs { /** create subdirs in storage */ bool Init(const char* chars, size_t cnt); - const std::string & GetRoot() { return this->root; } - const std::string & GetName() { return this->name; } + const std::string & GetRoot() const { return this->root; } + const std::string & GetName() const { return this->name; } /** set directory where to place storage directory */ void SetPlace(const std::string & path); /** path to file with given ident */ - std::string Path(const std::string & ident); + std::string Path(const std::string & ident) const; /** remove file by ident */ void Remove(const std::string & ident); /** find all files in storage and store list in provided vector */ From 85bd7a63c6695d60d86d0c35584b0cbd171135f3 Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 18 Feb 2016 00:00:00 +0000 Subject: [PATCH 15/23] * AddressBook : embed HashedStorage instance into AddressBookFilesystemStorage class --- AddressBook.cpp | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/AddressBook.cpp b/AddressBook.cpp index 540fd74c..fe5070f7 100644 --- a/AddressBook.cpp +++ b/AddressBook.cpp @@ -22,8 +22,12 @@ namespace client // TODO: this is actually proxy class class AddressBookFilesystemStorage: public AddressBookStorage { + private: + i2p::fs::HashedStorage storage; + std::string indexPath; + public: - AddressBookFilesystemStorage () {}; + AddressBookFilesystemStorage (); std::shared_ptr GetAddress (const i2p::data::IdentHash& ident) const; void AddAddress (std::shared_ptr address); void RemoveAddress (const i2p::data::IdentHash& ident); @@ -32,9 +36,17 @@ namespace client int Save (const std::map& addresses); }; + AddressBookFilesystemStorage::AddressBookFilesystemStorage(): + storage("addressbook", "b", "", "b32") + { + storage.SetPlace(i2p::fs::GetDataDir()); + storage.Init(i2p::data::GetBase32SubstitutionTable(), 32); + indexPath = storage.GetRoot() + i2p::fs::dirSep + "addresses.csv"; + } + std::shared_ptr AddressBookFilesystemStorage::GetAddress (const i2p::data::IdentHash& ident) const { - std::string filename = i2p::fs::GetAddressBook().Path(ident.ToBase32()); + std::string filename = storage.Path(ident.ToBase32()); std::ifstream f(filename, std::ifstream::binary); if (!f.is_open ()) { LogPrint(eLogDebug, "Addressbook: Requested, but not found: ", filename); @@ -57,7 +69,7 @@ namespace client void AddressBookFilesystemStorage::AddAddress (std::shared_ptr address) { - std::string path = i2p::fs::GetAddressBook().Path( address->GetIdentHash().ToBase32() ); + std::string path = storage.Path( address->GetIdentHash().ToBase32() ); std::ofstream f (path, std::ofstream::binary | std::ofstream::out); if (!f.is_open ()) { LogPrint (eLogError, "Addresbook: can't open file ", path); @@ -72,20 +84,19 @@ namespace client void AddressBookFilesystemStorage::RemoveAddress (const i2p::data::IdentHash& ident) { - i2p::fs::GetAddressBook().Remove( ident.ToBase32() ); + storage.Remove( ident.ToBase32() ); } int AddressBookFilesystemStorage::Load (std::map& addresses) { int num = 0; std::string s; - std::string index = i2p::fs::GetAddressBook().IndexPath(); - std::ifstream f (index, std::ifstream::in); // in text mode + std::ifstream f (indexPath, std::ifstream::in); // in text mode if (f.is_open ()) { - LogPrint(eLogInfo, "Addressbook: using index file ", index); + LogPrint(eLogInfo, "Addressbook: using index file ", indexPath); } else { - LogPrint(eLogWarning, "Addressbook: Can't open ", index); + LogPrint(eLogWarning, "Addressbook: Can't open ", indexPath); return 0; } @@ -120,11 +131,10 @@ namespace client } int num = 0; - std::string index = i2p::fs::GetAddressBook().IndexPath(); - std::ofstream f (index, std::ofstream::out); // in text mode + std::ofstream f (indexPath, std::ofstream::out); // in text mode if (!f.is_open ()) { - LogPrint (eLogWarning, "Addressbook: Can't open ", index); + LogPrint (eLogWarning, "Addressbook: Can't open ", indexPath); return 0; } From 2a4ba8d349a4de8f5a23f01807778568e2b2815e Mon Sep 17 00:00:00 2001 From: hagen Date: Thu, 18 Feb 2016 00:00:00 +0000 Subject: [PATCH 16/23] * Addressbook : move storage init code from constructor to Init() : was too early --- AddressBook.cpp | 9 +++++---- AddressBook.h | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/AddressBook.cpp b/AddressBook.cpp index fe5070f7..3b630a65 100644 --- a/AddressBook.cpp +++ b/AddressBook.cpp @@ -27,21 +27,21 @@ namespace client std::string indexPath; public: - AddressBookFilesystemStorage (); + AddressBookFilesystemStorage (): storage("addressbook", "b", "", "b32") {}; std::shared_ptr GetAddress (const i2p::data::IdentHash& ident) const; void AddAddress (std::shared_ptr address); void RemoveAddress (const i2p::data::IdentHash& ident); + bool Init (); int Load (std::map& addresses); int Save (const std::map& addresses); }; - AddressBookFilesystemStorage::AddressBookFilesystemStorage(): - storage("addressbook", "b", "", "b32") + bool AddressBookFilesystemStorage::Init() { storage.SetPlace(i2p::fs::GetDataDir()); - storage.Init(i2p::data::GetBase32SubstitutionTable(), 32); indexPath = storage.GetRoot() + i2p::fs::dirSep + "addresses.csv"; + return storage.Init(i2p::data::GetBase32SubstitutionTable(), 32); } std::shared_ptr AddressBookFilesystemStorage::GetAddress (const i2p::data::IdentHash& ident) const @@ -159,6 +159,7 @@ namespace client void AddressBook::Start () { + m_Storage->Init(); LoadHosts (); /* try storage, then hosts.txt, then download */ StartSubscriptions (); } diff --git a/AddressBook.h b/AddressBook.h index 9ddce82a..16403e1e 100644 --- a/AddressBook.h +++ b/AddressBook.h @@ -35,6 +35,7 @@ namespace client virtual void AddAddress (std::shared_ptr address) = 0; virtual void RemoveAddress (const i2p::data::IdentHash& ident) = 0; + virtual bool Init () = 0; virtual int Load (std::map& addresses) = 0; virtual int Save (const std::map& addresses) = 0; }; From 02310d4af67ebac7129117bc2920fd7e62d24917 Mon Sep 17 00:00:00 2001 From: hagen Date: Sat, 20 Feb 2016 01:00:00 +0000 Subject: [PATCH 17/23] * Family : use i2p::fs::ReadDir instead direct boost::filesystem call --- Family.cpp | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Family.cpp b/Family.cpp index a93e31ef..442f096d 100644 --- a/Family.cpp +++ b/Family.cpp @@ -1,7 +1,7 @@ #include -#include "util.h" #include #include +#include "FS.h" #include "Log.h" #include "Crypto.h" #include "Family.h" @@ -84,21 +84,24 @@ namespace data void Families::LoadCertificates () { - boost::filesystem::path familyDir = i2p::util::filesystem::GetCertificatesDir() / "family"; - - if (!boost::filesystem::exists (familyDir)) return; + std::string certDir = i2p::fs::DataDirPath("certificates", "family"); + std::vector files; int numCertificates = 0; - boost::filesystem::directory_iterator end; // empty - for (boost::filesystem::directory_iterator it (familyDir); it != end; ++it) - { - if (boost::filesystem::is_regular_file (it->status()) && it->path ().extension () == ".crt") - { - LoadCertificate (it->path ().string ()); - numCertificates++; - } + + if (!i2p::fs::ReadDir(certDir, files)) { + LogPrint(eLogWarning, "Reseed: Can't load reseed certificates from ", certDir); + return; + } + + for (const std::string & file : files) { + if (file.compare(file.size() - 4, 4, ".crt") != 0) { + LogPrint(eLogWarning, "Family: ignoring file ", file); + continue; + } + LoadCertificate (file); + numCertificates++; } - if (numCertificates > 0) - LogPrint (eLogInfo, "Family: ", numCertificates, " certificates loaded"); + LogPrint (eLogInfo, "Family: ", numCertificates, " certificates loaded"); } bool Families::VerifyFamily (const std::string& family, const IdentHash& ident, @@ -116,7 +119,6 @@ namespace data // TODO: process key return true; } - } } From d312d753e960fd7473ffe274f7917c6c3f15973f Mon Sep 17 00:00:00 2001 From: hagen Date: Sat, 20 Feb 2016 01:00:00 +0000 Subject: [PATCH 18/23] * Destination.cpp : fix lambda with 4.7 --- Destination.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Destination.cpp b/Destination.cpp index 5163f99d..3d7f34c1 100644 --- a/Destination.cpp +++ b/Destination.cpp @@ -460,7 +460,7 @@ namespace client { auto s = shared_from_this (); RequestLeaseSet (GetIdentHash (), - [s](std::shared_ptr leaseSet) + [s,this](std::shared_ptr leaseSet) { if (leaseSet) { From 33a33e3c712cb631c2babb137819475e31b0850c Mon Sep 17 00:00:00 2001 From: hagen Date: Sat, 20 Feb 2016 01:00:00 +0000 Subject: [PATCH 19/23] * i2p::util::http::GetHttpContent() : use std::transform instead boost --- util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util.cpp b/util.cpp index e871dac1..bd77e682 100644 --- a/util.cpp +++ b/util.cpp @@ -81,7 +81,7 @@ namespace http if (colon != std::string::npos) { std::string field = header.substr (0, colon); - boost::to_lower (field); + std::transform(field.begin(), field.end(), field.begin(), ::tolower); if (field == i2p::util::http::TRANSFER_ENCODING) isChunked = (header.find ("chunked", colon + 1) != std::string::npos); } From d3746e0119175f0d23c77fde7ac1d5013323a891 Mon Sep 17 00:00:00 2001 From: hagen Date: Sat, 20 Feb 2016 01:00:00 +0000 Subject: [PATCH 20/23] * FS.h : add include guards --- FS.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/FS.h b/FS.h index 0c4e246a..f80840bc 100644 --- a/FS.h +++ b/FS.h @@ -6,6 +6,9 @@ * See full license text in LICENSE file at top of project tree */ +#ifndef FS_H__ +#define FS_H__ + #include #include #include @@ -139,3 +142,5 @@ namespace fs { HashedStorage & GetPeerProfiles(); } // fs } // i2p + +#endif // /* FS_H__ */ From b69fbdda9a6f8dc0891a6505789279342f917359 Mon Sep 17 00:00:00 2001 From: hagen Date: Sat, 20 Feb 2016 01:00:00 +0000 Subject: [PATCH 21/23] * NetDb : move storage from FS.cpp to NetDb.cpp --- FS.cpp | 4 ---- FS.h | 1 - NetDb.cpp | 11 ++++++----- NetDb.h | 4 ++-- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/FS.cpp b/FS.cpp index f6125954..ac3d0935 100644 --- a/FS.cpp +++ b/FS.cpp @@ -22,7 +22,6 @@ namespace fs { #else std::string dirSep = "/"; #endif - HashedStorage NetDB("netDb", "r", "routerInfo-", "dat"); HashedStorage Peers("peerProfiles", "p", "profile-", "txt"); const std::string & GetAppName () { @@ -72,8 +71,6 @@ namespace fs { if (boost::filesystem::exists(destinations)) boost::filesystem::create_directory(destinations); - NetDB.SetPlace(dataDir); - NetDB.Init(i2p::data::GetBase64SubstitutionTable(), 64); Peers.SetPlace(dataDir); Peers.Init(i2p::data::GetBase64SubstitutionTable(), 64); return true; @@ -157,7 +154,6 @@ namespace fs { } } - HashedStorage & GetNetDB() { return NetDB; } HashedStorage & GetPeerProfiles() { return Peers; } } // fs } // i2p diff --git a/FS.h b/FS.h index f80840bc..a3f071e5 100644 --- a/FS.h +++ b/FS.h @@ -138,7 +138,6 @@ namespace fs { } /* accessors */ - HashedStorage & GetNetDB(); HashedStorage & GetPeerProfiles(); } // fs } // i2p diff --git a/NetDb.cpp b/NetDb.cpp index 428f1c36..1678850b 100644 --- a/NetDb.cpp +++ b/NetDb.cpp @@ -7,7 +7,6 @@ #include "I2PEndian.h" #include "Base.h" #include "Log.h" -#include "FS.h" #include "Timestamp.h" #include "I2NPProtocol.h" #include "Tunnel.h" @@ -25,7 +24,7 @@ namespace data { NetDb netdb; - NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr) + NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr), m_Storage("netDb", "r", "routerInfo-", "dat") { } @@ -37,6 +36,8 @@ namespace data void NetDb::Start () { + m_Storage.SetPlace(i2p::fs::GetDataDir()); + m_Storage.Init(i2p::data::GetBase64SubstitutionTable(), 64); m_Families.LoadCertificates (); Load (); if (m_RouterInfos.size () < 25) // reseed if # of router less than 50 @@ -313,7 +314,7 @@ namespace data m_LastLoad = i2p::util::GetSecondsSinceEpoch(); std::vector files; - i2p::fs::GetNetDB().Traverse(files); + m_Storage.Traverse(files); for (auto path : files) LoadRouterInfo(path); @@ -329,7 +330,7 @@ namespace data for (auto it: m_RouterInfos) { std::string ident = it.second->GetIdentHashBase64(); - std::string path = i2p::fs::GetNetDB().Path(ident); + std::string path = m_Storage.Path(ident); if (it.second->IsUpdated ()) { it.second->SaveToFile (path); it.second->SetUpdated (false); @@ -376,7 +377,7 @@ namespace data if (it.second->IsUnreachable ()) { total--; // delete RI file - i2p::fs::GetNetDB().Remove(ident); + m_Storage.Remove(ident); deletedCount++; // delete from floodfills list if (it.second->IsFloodfill ()) { diff --git a/NetDb.h b/NetDb.h index 0c59ff0f..7efbfcf2 100644 --- a/NetDb.h +++ b/NetDb.h @@ -9,6 +9,7 @@ #include #include #include "Base.h" +#include "FS.h" #include "Queue.h" #include "I2NPProtocol.h" #include "RouterInfo.h" @@ -97,11 +98,10 @@ namespace data GzipInflator m_Inflator; Reseeder * m_Reseeder; Families m_Families; + i2p::fs::HashedStorage m_Storage; friend class NetDbRequests; NetDbRequests m_Requests; - - static const char m_NetDbPath[]; }; extern NetDb netdb; From 0d15eceacb3eee65327eb7a221491b6faa6c1357 Mon Sep 17 00:00:00 2001 From: hagen Date: Sat, 20 Feb 2016 01:00:00 +0000 Subject: [PATCH 22/23] * Profiling : move storage from FS.cpp to Profiling.cpp --- FS.cpp | 5 ----- FS.h | 3 --- NetDb.cpp | 1 + Profiling.cpp | 14 +++++++++++--- Profiling.h | 1 + 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/FS.cpp b/FS.cpp index ac3d0935..9416d6b4 100644 --- a/FS.cpp +++ b/FS.cpp @@ -22,7 +22,6 @@ namespace fs { #else std::string dirSep = "/"; #endif - HashedStorage Peers("peerProfiles", "p", "profile-", "txt"); const std::string & GetAppName () { return appName; @@ -71,8 +70,6 @@ namespace fs { if (boost::filesystem::exists(destinations)) boost::filesystem::create_directory(destinations); - Peers.SetPlace(dataDir); - Peers.Init(i2p::data::GetBase64SubstitutionTable(), 64); return true; } @@ -153,7 +150,5 @@ namespace fs { files.push_back(t); } } - - HashedStorage & GetPeerProfiles() { return Peers; } } // fs } // i2p diff --git a/FS.h b/FS.h index a3f071e5..833258b9 100644 --- a/FS.h +++ b/FS.h @@ -136,9 +136,6 @@ namespace fs { return s.str(); } - - /* accessors */ - HashedStorage & GetPeerProfiles(); } // fs } // i2p diff --git a/NetDb.cpp b/NetDb.cpp index 1678850b..6401fcc8 100644 --- a/NetDb.cpp +++ b/NetDb.cpp @@ -38,6 +38,7 @@ namespace data { m_Storage.SetPlace(i2p::fs::GetDataDir()); m_Storage.Init(i2p::data::GetBase64SubstitutionTable(), 64); + InitProfilesStorage (); m_Families.LoadCertificates (); Load (); if (m_RouterInfos.size () < 25) // reseed if # of router less than 50 diff --git a/Profiling.cpp b/Profiling.cpp index d868298f..be675502 100644 --- a/Profiling.cpp +++ b/Profiling.cpp @@ -10,6 +10,8 @@ namespace i2p { namespace data { + i2p::fs::HashedStorage m_ProfilesStorage("peerProfiles", "p", "profile-", "txt"); + RouterProfile::RouterProfile (const IdentHash& identHash): m_IdentHash (identHash), m_LastUpdateTime (boost::posix_time::second_clock::local_time()), m_NumTunnelsAgreed (0), m_NumTunnelsDeclined (0), m_NumTunnelsNonReplied (0), @@ -45,7 +47,7 @@ namespace data // save to file std::string ident = m_IdentHash.ToBase64 (); - std::string path = i2p::fs::GetPeerProfiles().Path(ident); + std::string path = m_ProfilesStorage.Path(ident); try { boost::property_tree::write_ini (path, pt); @@ -58,7 +60,7 @@ namespace data void RouterProfile::Load () { std::string ident = m_IdentHash.ToBase64 (); - std::string path = i2p::fs::GetPeerProfiles().Path(ident); + std::string path = m_ProfilesStorage.Path(ident); boost::property_tree::ptree pt; if (!i2p::fs::Exists(path)) { @@ -152,13 +154,19 @@ namespace data return profile; } + void InitProfilesStorage () + { + m_ProfilesStorage.SetPlace(i2p::fs::GetDataDir()); + m_ProfilesStorage.Init(i2p::data::GetBase64SubstitutionTable(), 64); + } + void DeleteObsoleteProfiles () { struct stat st; std::time_t now = std::time(nullptr); std::vector files; - i2p::fs::GetPeerProfiles().Traverse(files); + m_ProfilesStorage.Traverse(files); for (auto path: files) { if (stat(path.c_str(), &st) != 0) { LogPrint(eLogWarning, "Profiling: Can't stat(): ", path); diff --git a/Profiling.h b/Profiling.h index 3a65714d..26d5c2f7 100644 --- a/Profiling.h +++ b/Profiling.h @@ -60,6 +60,7 @@ namespace data }; std::shared_ptr GetRouterProfile (const IdentHash& identHash); + void InitProfilesStorage (); void DeleteObsoleteProfiles (); } } From dc344d465835f4adb06a0b2e2a0b4aeb525386c1 Mon Sep 17 00:00:00 2001 From: hagen Date: Mon, 22 Feb 2016 12:57:25 +0000 Subject: [PATCH 23/23] * add comment --- Destination.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Destination.cpp b/Destination.cpp index 3d7f34c1..484c0679 100644 --- a/Destination.cpp +++ b/Destination.cpp @@ -460,6 +460,7 @@ namespace client { auto s = shared_from_this (); RequestLeaseSet (GetIdentHash (), + // "this" added due to bug in gcc 4.7-4.8 [s,this](std::shared_ptr leaseSet) { if (leaseSet)