From 86355cfe09278ff158bf45192451da0017dd72b2 Mon Sep 17 00:00:00 2001 From: Meeh Date: Fri, 31 Jan 2014 07:22:11 +0100 Subject: [PATCH] Adding reseed support. v1, only http so far. --- Makefile | 4 +-- NetDb.cpp | 56 +++++++++----------------------- NetDb.h | 3 +- Reseed.cpp | 70 ++++++++++++++++++++++++++++++++++++++++ Reseed.h | 28 ++++++++++++++++ util.cpp | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ util.h | 12 ++++++- 7 files changed, 223 insertions(+), 45 deletions(-) create mode 100644 Reseed.cpp create mode 100644 Reseed.h diff --git a/Makefile b/Makefile index 5a3e6336..5c2e61fb 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,9 @@ CFLAGS = -g -Wall -std=c++0x OBJECTS = obj/i2p.o obj/base64.o obj/NTCPSession.o obj/RouterInfo.o obj/Transports.o \ obj/RouterContext.o obj/NetDb.o obj/LeaseSet.o obj/Tunnel.o obj/TunnelEndpoint.o \ obj/TunnelGateway.o obj/TransitTunnel.o obj/I2NPProtocol.o obj/Log.o obj/Garlic.o \ - obj/HTTPServer.o obj/Streaming.o obj/Identity.o obj/SSU.o obj/util.o + obj/HTTPServer.o obj/Streaming.o obj/Identity.o obj/SSU.o obj/util.o obj/Reseed.o INCFLAGS = -LDFLAGS = -Wl,-rpath,/usr/local/lib -lcryptopp -lboost_system -lboost_filesystem -lpthread +LDFLAGS = -Wl,-rpath,/usr/local/lib -lcryptopp -lboost_system -lboost_filesystem -lboost_regex -lpthread LIBS = all: i2p diff --git a/NetDb.cpp b/NetDb.cpp index 278e8286..4d4ce103 100644 --- a/NetDb.cpp +++ b/NetDb.cpp @@ -13,6 +13,7 @@ #include "RouterContext.h" #include "Garlic.h" #include "NetDb.h" +#include "Reseed.h" namespace i2p { @@ -207,12 +208,23 @@ namespace data void NetDb::Load (const char * directory) { + Load(directory, false); + } + + void NetDb::Load (const char * directory, bool reseed) + { + i2p::data::Reseeder *reseeder = new i2p::data::Reseeder(); boost::filesystem::path p (directory); if (!boost::filesystem::exists (p)) { if (!CreateNetDb(directory)) return; + reseeder->reseedNow(); + } + if (reseed) + { + reseeder->reseedNow(); + m_reseedRetries++; } - // TODO: Reseed if needed. int numRouters = 0; boost::filesystem::directory_iterator end; for (boost::filesystem::directory_iterator it (p); it != end; ++it) @@ -232,6 +244,8 @@ namespace data } } LogPrint (numRouters, " routers loaded"); + if (numRouters < 100 && m_reseedRetries < 10) + Load(directory, true); // Reseed } void NetDb::SaveUpdated (const char * directory) @@ -601,45 +615,5 @@ namespace data return r; } - //TODO: Move to reseed. - //TODO: Implement v1 & v2 reseeding. Lightweight zip library is needed for v2. - // orignal: zip is part of crypto++, see implementation of DatabaseStoreMsg - //TODO: Implement SU3, utils. - void NetDb::DownloadRouterInfo (const std::string& address, const std::string& filename) - { - try - { - boost::asio::ip::tcp::iostream site(address, "http"); - if (!site) - { - //site.expires_from_now (boost::posix_time::seconds (10)); // wait for 10 seconds - site << "GET " << filename << "HTTP/1.0\nHost: " << address << "\nAccept: */*\nConnection: close\n\n"; - // read response - std::string version, statusMessage; - site >> version; // HTTP version - int status; - site >> status; // status - std::getline (site, statusMessage); - if (status == 200) // OK - { - std::string header; - while (header != "\n") - std::getline (site, header); - // read content - std::stringstream ss; - ss << site.rdbuf(); - AddRouterInfo ((uint8_t *)ss.str ().c_str (), ss.str ().size ()); - } - else - LogPrint ("HTTP response ", status); - } - else - LogPrint ("Can't connect to ", address); - } - catch (std::exception& ex) - { - LogPrint ("Failed to download ", filename, " : ", ex.what ()); - } - } } } diff --git a/NetDb.h b/NetDb.h index 1397e9e4..ddf50d06 100644 --- a/NetDb.h +++ b/NetDb.h @@ -78,8 +78,8 @@ namespace data bool CreateNetDb(const char * directory); void Load (const char * directory); + void Load (const char * directory, bool reseed); void SaveUpdated (const char * directory); - void DownloadRouterInfo (const std::string& address, const std::string& filename); // for reseed void Run (); // exploratory thread void Explore (); const RouterInfo * GetClosestFloodfill (const IdentHash& destination, const std::set& excluded) const; @@ -95,6 +95,7 @@ namespace data std::map m_RequestedDestinations; bool m_IsRunning; + int m_reseedRetries = 0; std::thread * m_Thread; i2p::util::Queue m_Queue; // of I2NPDatabaseStoreMsg }; diff --git a/Reseed.cpp b/Reseed.cpp new file mode 100644 index 00000000..cc61e4ef --- /dev/null +++ b/Reseed.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include "Reseed.h" +#include "Log.h" +#include "util.h" + + +namespace i2p +{ +namespace data +{ + + Reseeder::Reseeder() + { + } + + Reseeder::~Reseeder() + { + } + + bool Reseeder::reseedNow() + { + try + { + std::string reseedHost = httpReseedHostList[(rand() % httpReseedHostList.size())]; + LogPrint("Reseeding from ", reseedHost); + std::string content = i2p::util::http::httpRequest(reseedHost); + if (content == "") + { + LogPrint("Reseed failed"); + return false; + } + boost::regex e("<\\s*A\\s+[^>]*href\\s*=\\s*\"([^\"]*)\"", + boost::regex::normal | boost::regbase::icase); + boost::sregex_token_iterator i(content.begin(), content.end(), e, 1); + boost::sregex_token_iterator j; + std::string name; + std::string routerInfo; + std::string tmpUrl; + std::string filename; + while (i != j) + { + name = *i++; + LogPrint("Downloading ", name); + tmpUrl = reseedHost; + tmpUrl.append(name); + routerInfo = i2p::util::http::httpRequest(tmpUrl); + filename = "netDb/r"; + filename += name.at(11); + filename.append("/"); + filename.append(name.c_str()); + std::ofstream outfile (filename, std::ios::binary); + outfile << routerInfo; + outfile.close(); + } + return true; + } + catch (std::exception& ex) + { + //TODO: error reporting + return false; + } + return false; + } + +} +} + diff --git a/Reseed.h b/Reseed.h new file mode 100644 index 00000000..ac5a7cd7 --- /dev/null +++ b/Reseed.h @@ -0,0 +1,28 @@ +#ifndef RESEED_H +#define RESEED_H + +#include +#include + +namespace i2p +{ +namespace data +{ + + class Reseeder + { + public: + Reseeder(); + ~Reseeder(); + bool reseedNow(); + private: + std::vector httpReseedHostList = { + "http://193.150.121.66/netDb/", + "http://netdb.i2p2.no/" + }; + }; + +} +} + +#endif \ No newline at end of file diff --git a/util.cpp b/util.cpp index 676fddd1..a101dfe4 100644 --- a/util.cpp +++ b/util.cpp @@ -1,4 +1,11 @@ +#include +#include +#include +#include +#include +#include #include "util.h" +#include "Log.h" namespace i2p { @@ -40,6 +47,94 @@ const char* GetCharArg(const std::string& strArg, const std::string& nDefault) return nDefault.c_str(); } +namespace http +{ +//const std::string& filename +//AddRouterInfo ((uint8_t *)ss.str ().c_str (), ss.str ().size ()); + //TODO: Move to reseed. + //TODO: Implement v1 & v2 reseeding. Lightweight zip library is needed for v2. + // orignal: zip is part of crypto++, see implementation of DatabaseStoreMsg + //TODO: Implement SU3, utils. + std::string httpRequest(const std::string& address) + { + try + { + i2p::util::http::url u(address); + boost::asio::ip::tcp::iostream site; + site.expires_from_now (boost::posix_time::seconds(30)); + site.connect(u.host_, "http"); + if (site) + { + //site.expires_from_now (boost::posix_time::seconds (10)); // wait for 10 seconds + // User-Agent is needed to get the server list routerInfo files. + site << "GET " << u.path_ << " HTTP/1.0\r\nHost: " << u.host_ + << "\r\nAccept: */*\r\n" << "User-Agent: Wget/1.11.4\r\n" << "Connection: close\r\n\r\n"; + // read response + std::string version, statusMessage; + site >> version; // HTTP version + int status; + site >> status; // status + std::getline (site, statusMessage); + if (status == 200) // OK + { + std::string header; + while (std::getline(site, header) && header != "\r"){} + std::stringstream ss; + ss << site.rdbuf(); + return ss.str(); + } + else + { + LogPrint ("HTTP response ", status); + return ""; + } + } + else + { + LogPrint ("Can't connect to ", address); + return ""; + } + } + catch (std::exception& ex) + { + LogPrint ("Failed to download ", address, " : ", ex.what ()); + return ""; + } + } + + url::url(const std::string& url_s) + { + parse(url_s); + } + + void url::parse(const std::string& url_s) + { + const std::string prot_end("://"); + std::string::const_iterator prot_i = search(url_s.begin(), url_s.end(), + prot_end.begin(), prot_end.end()); + protocol_.reserve(distance(url_s.begin(), prot_i)); + transform(url_s.begin(), prot_i, + back_inserter(protocol_), + std::ptr_fun(tolower)); // protocol is icase + if( prot_i == url_s.end() ) + return; + advance(prot_i, prot_end.length()); + std::string::const_iterator path_i = find(prot_i, url_s.end(), '/'); + host_.reserve(distance(prot_i, path_i)); + transform(prot_i, path_i, + back_inserter(host_), + std::ptr_fun(tolower)); // host is icase + std::string::const_iterator query_i = find(path_i, url_s.end(), '?'); + path_.assign(path_i, query_i); + if( query_i != url_s.end() ) + ++query_i; + query_.assign(query_i, url_s.end()); + } + +} + + + } // Namespace end } diff --git a/util.h b/util.h index fb3597e6..7643e0d0 100644 --- a/util.h +++ b/util.h @@ -12,7 +12,17 @@ namespace util void OptionParser(int argc, const char* const argv[]); int GetIntArg(const std::string& strArg, int nDefault); const char* GetCharArg(const std::string& strArg, const std::string& nDefault); - + namespace http + { + std::string httpRequest(const std::string& address); + struct url { + url(const std::string& url_s); // omitted copy, ==, accessors, ... + private: + void parse(const std::string& url_s); + public: + std::string protocol_, host_, path_, query_; + }; + } } }