From 86355cfe09278ff158bf45192451da0017dd72b2 Mon Sep 17 00:00:00 2001 From: Meeh Date: Fri, 31 Jan 2014 07:22:11 +0100 Subject: [PATCH 1/4] Adding reseed support. v1, only http so far. --- Makefile | 4 +-- NetDb.cpp | 56 +++++++++----------------------- NetDb.h | 3 +- Reseed.cpp | 70 ++++++++++++++++++++++++++++++++++++++++ Reseed.h | 28 ++++++++++++++++ util.cpp | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ util.h | 12 ++++++- 7 files changed, 223 insertions(+), 45 deletions(-) create mode 100644 Reseed.cpp create mode 100644 Reseed.h diff --git a/Makefile b/Makefile index 5a3e6336..5c2e61fb 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,9 @@ CFLAGS = -g -Wall -std=c++0x OBJECTS = obj/i2p.o obj/base64.o obj/NTCPSession.o obj/RouterInfo.o obj/Transports.o \ obj/RouterContext.o obj/NetDb.o obj/LeaseSet.o obj/Tunnel.o obj/TunnelEndpoint.o \ obj/TunnelGateway.o obj/TransitTunnel.o obj/I2NPProtocol.o obj/Log.o obj/Garlic.o \ - obj/HTTPServer.o obj/Streaming.o obj/Identity.o obj/SSU.o obj/util.o + obj/HTTPServer.o obj/Streaming.o obj/Identity.o obj/SSU.o obj/util.o obj/Reseed.o INCFLAGS = -LDFLAGS = -Wl,-rpath,/usr/local/lib -lcryptopp -lboost_system -lboost_filesystem -lpthread +LDFLAGS = -Wl,-rpath,/usr/local/lib -lcryptopp -lboost_system -lboost_filesystem -lboost_regex -lpthread LIBS = all: i2p diff --git a/NetDb.cpp b/NetDb.cpp index 278e8286..4d4ce103 100644 --- a/NetDb.cpp +++ b/NetDb.cpp @@ -13,6 +13,7 @@ #include "RouterContext.h" #include "Garlic.h" #include "NetDb.h" +#include "Reseed.h" namespace i2p { @@ -207,12 +208,23 @@ namespace data void NetDb::Load (const char * directory) { + Load(directory, false); + } + + void NetDb::Load (const char * directory, bool reseed) + { + i2p::data::Reseeder *reseeder = new i2p::data::Reseeder(); boost::filesystem::path p (directory); if (!boost::filesystem::exists (p)) { if (!CreateNetDb(directory)) return; + reseeder->reseedNow(); + } + if (reseed) + { + reseeder->reseedNow(); + m_reseedRetries++; } - // TODO: Reseed if needed. int numRouters = 0; boost::filesystem::directory_iterator end; for (boost::filesystem::directory_iterator it (p); it != end; ++it) @@ -232,6 +244,8 @@ namespace data } } LogPrint (numRouters, " routers loaded"); + if (numRouters < 100 && m_reseedRetries < 10) + Load(directory, true); // Reseed } void NetDb::SaveUpdated (const char * directory) @@ -601,45 +615,5 @@ namespace data return r; } - //TODO: Move to reseed. - //TODO: Implement v1 & v2 reseeding. Lightweight zip library is needed for v2. - // orignal: zip is part of crypto++, see implementation of DatabaseStoreMsg - //TODO: Implement SU3, utils. - void NetDb::DownloadRouterInfo (const std::string& address, const std::string& filename) - { - try - { - boost::asio::ip::tcp::iostream site(address, "http"); - if (!site) - { - //site.expires_from_now (boost::posix_time::seconds (10)); // wait for 10 seconds - site << "GET " << filename << "HTTP/1.0\nHost: " << address << "\nAccept: */*\nConnection: close\n\n"; - // read response - std::string version, statusMessage; - site >> version; // HTTP version - int status; - site >> status; // status - std::getline (site, statusMessage); - if (status == 200) // OK - { - std::string header; - while (header != "\n") - std::getline (site, header); - // read content - std::stringstream ss; - ss << site.rdbuf(); - AddRouterInfo ((uint8_t *)ss.str ().c_str (), ss.str ().size ()); - } - else - LogPrint ("HTTP response ", status); - } - else - LogPrint ("Can't connect to ", address); - } - catch (std::exception& ex) - { - LogPrint ("Failed to download ", filename, " : ", ex.what ()); - } - } } } diff --git a/NetDb.h b/NetDb.h index 1397e9e4..ddf50d06 100644 --- a/NetDb.h +++ b/NetDb.h @@ -78,8 +78,8 @@ namespace data bool CreateNetDb(const char * directory); void Load (const char * directory); + void Load (const char * directory, bool reseed); void SaveUpdated (const char * directory); - void DownloadRouterInfo (const std::string& address, const std::string& filename); // for reseed void Run (); // exploratory thread void Explore (); const RouterInfo * GetClosestFloodfill (const IdentHash& destination, const std::set& excluded) const; @@ -95,6 +95,7 @@ namespace data std::map m_RequestedDestinations; bool m_IsRunning; + int m_reseedRetries = 0; std::thread * m_Thread; i2p::util::Queue m_Queue; // of I2NPDatabaseStoreMsg }; diff --git a/Reseed.cpp b/Reseed.cpp new file mode 100644 index 00000000..cc61e4ef --- /dev/null +++ b/Reseed.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include "Reseed.h" +#include "Log.h" +#include "util.h" + + +namespace i2p +{ +namespace data +{ + + Reseeder::Reseeder() + { + } + + Reseeder::~Reseeder() + { + } + + bool Reseeder::reseedNow() + { + try + { + std::string reseedHost = httpReseedHostList[(rand() % httpReseedHostList.size())]; + LogPrint("Reseeding from ", reseedHost); + std::string content = i2p::util::http::httpRequest(reseedHost); + if (content == "") + { + LogPrint("Reseed failed"); + return false; + } + boost::regex e("<\\s*A\\s+[^>]*href\\s*=\\s*\"([^\"]*)\"", + boost::regex::normal | boost::regbase::icase); + boost::sregex_token_iterator i(content.begin(), content.end(), e, 1); + boost::sregex_token_iterator j; + std::string name; + std::string routerInfo; + std::string tmpUrl; + std::string filename; + while (i != j) + { + name = *i++; + LogPrint("Downloading ", name); + tmpUrl = reseedHost; + tmpUrl.append(name); + routerInfo = i2p::util::http::httpRequest(tmpUrl); + filename = "netDb/r"; + filename += name.at(11); + filename.append("/"); + filename.append(name.c_str()); + std::ofstream outfile (filename, std::ios::binary); + outfile << routerInfo; + outfile.close(); + } + return true; + } + catch (std::exception& ex) + { + //TODO: error reporting + return false; + } + return false; + } + +} +} + diff --git a/Reseed.h b/Reseed.h new file mode 100644 index 00000000..ac5a7cd7 --- /dev/null +++ b/Reseed.h @@ -0,0 +1,28 @@ +#ifndef RESEED_H +#define RESEED_H + +#include +#include + +namespace i2p +{ +namespace data +{ + + class Reseeder + { + public: + Reseeder(); + ~Reseeder(); + bool reseedNow(); + private: + std::vector httpReseedHostList = { + "http://193.150.121.66/netDb/", + "http://netdb.i2p2.no/" + }; + }; + +} +} + +#endif \ No newline at end of file diff --git a/util.cpp b/util.cpp index 676fddd1..a101dfe4 100644 --- a/util.cpp +++ b/util.cpp @@ -1,4 +1,11 @@ +#include +#include +#include +#include +#include +#include #include "util.h" +#include "Log.h" namespace i2p { @@ -40,6 +47,94 @@ const char* GetCharArg(const std::string& strArg, const std::string& nDefault) return nDefault.c_str(); } +namespace http +{ +//const std::string& filename +//AddRouterInfo ((uint8_t *)ss.str ().c_str (), ss.str ().size ()); + //TODO: Move to reseed. + //TODO: Implement v1 & v2 reseeding. Lightweight zip library is needed for v2. + // orignal: zip is part of crypto++, see implementation of DatabaseStoreMsg + //TODO: Implement SU3, utils. + std::string httpRequest(const std::string& address) + { + try + { + i2p::util::http::url u(address); + boost::asio::ip::tcp::iostream site; + site.expires_from_now (boost::posix_time::seconds(30)); + site.connect(u.host_, "http"); + if (site) + { + //site.expires_from_now (boost::posix_time::seconds (10)); // wait for 10 seconds + // User-Agent is needed to get the server list routerInfo files. + site << "GET " << u.path_ << " HTTP/1.0\r\nHost: " << u.host_ + << "\r\nAccept: */*\r\n" << "User-Agent: Wget/1.11.4\r\n" << "Connection: close\r\n\r\n"; + // read response + std::string version, statusMessage; + site >> version; // HTTP version + int status; + site >> status; // status + std::getline (site, statusMessage); + if (status == 200) // OK + { + std::string header; + while (std::getline(site, header) && header != "\r"){} + std::stringstream ss; + ss << site.rdbuf(); + return ss.str(); + } + else + { + LogPrint ("HTTP response ", status); + return ""; + } + } + else + { + LogPrint ("Can't connect to ", address); + return ""; + } + } + catch (std::exception& ex) + { + LogPrint ("Failed to download ", address, " : ", ex.what ()); + return ""; + } + } + + url::url(const std::string& url_s) + { + parse(url_s); + } + + void url::parse(const std::string& url_s) + { + const std::string prot_end("://"); + std::string::const_iterator prot_i = search(url_s.begin(), url_s.end(), + prot_end.begin(), prot_end.end()); + protocol_.reserve(distance(url_s.begin(), prot_i)); + transform(url_s.begin(), prot_i, + back_inserter(protocol_), + std::ptr_fun(tolower)); // protocol is icase + if( prot_i == url_s.end() ) + return; + advance(prot_i, prot_end.length()); + std::string::const_iterator path_i = find(prot_i, url_s.end(), '/'); + host_.reserve(distance(prot_i, path_i)); + transform(prot_i, path_i, + back_inserter(host_), + std::ptr_fun(tolower)); // host is icase + std::string::const_iterator query_i = find(path_i, url_s.end(), '?'); + path_.assign(path_i, query_i); + if( query_i != url_s.end() ) + ++query_i; + query_.assign(query_i, url_s.end()); + } + +} + + + } // Namespace end } diff --git a/util.h b/util.h index fb3597e6..7643e0d0 100644 --- a/util.h +++ b/util.h @@ -12,7 +12,17 @@ namespace util void OptionParser(int argc, const char* const argv[]); int GetIntArg(const std::string& strArg, int nDefault); const char* GetCharArg(const std::string& strArg, const std::string& nDefault); - + namespace http + { + std::string httpRequest(const std::string& address); + struct url { + url(const std::string& url_s); // omitted copy, ==, accessors, ... + private: + void parse(const std::string& url_s); + public: + std::string protocol_, host_, path_, query_; + }; + } } } From 83bd3c21ca6fc92d3d87567113f226a811effd56 Mon Sep 17 00:00:00 2001 From: Meeh Date: Fri, 31 Jan 2014 07:31:06 +0100 Subject: [PATCH 2/4] Adding more reseed hosts Ignore zip files --- Reseed.cpp | 4 ++++ Reseed.h | 10 +++++++++- util.cpp | 6 +----- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Reseed.cpp b/Reseed.cpp index cc61e4ef..05825f6a 100644 --- a/Reseed.cpp +++ b/Reseed.cpp @@ -40,9 +40,13 @@ namespace data std::string routerInfo; std::string tmpUrl; std::string filename; + std::string ignoreFileSuffix = ".zip"; while (i != j) { + //TODO: Ugly code, try to clean up. name = *i++; + if (name.find(ignoreFileSuffix)!=std::string::npos) + continue; LogPrint("Downloading ", name); tmpUrl = reseedHost; tmpUrl.append(name); diff --git a/Reseed.h b/Reseed.h index ac5a7cd7..96ce9c55 100644 --- a/Reseed.h +++ b/Reseed.h @@ -18,7 +18,15 @@ namespace data private: std::vector httpReseedHostList = { "http://193.150.121.66/netDb/", - "http://netdb.i2p2.no/" + "http://netdb.i2p2.no/", + "http://reseed.i2p-projekt.de/", + "http://cowpuncher.drollette.com/netdb/", + "http://i2p.mooo.com/netDb/", + "http://reseed.info/", + "http://reseed.pkol.de/", + "http://uk.reseed.i2p2.no/", + "http://i2p-netdb.innovatio.no/", + "http://ieb9oopo.mooo.com" }; }; diff --git a/util.cpp b/util.cpp index a101dfe4..5834e2ab 100644 --- a/util.cpp +++ b/util.cpp @@ -49,11 +49,7 @@ const char* GetCharArg(const std::string& strArg, const std::string& nDefault) namespace http { -//const std::string& filename -//AddRouterInfo ((uint8_t *)ss.str ().c_str (), ss.str ().size ()); - //TODO: Move to reseed. - //TODO: Implement v1 & v2 reseeding. Lightweight zip library is needed for v2. - // orignal: zip is part of crypto++, see implementation of DatabaseStoreMsg + //TODO: Implement v2 reseeding. Lightweight zip library is needed. //TODO: Implement SU3, utils. std::string httpRequest(const std::string& address) { From 9415a49fbd618ad0d770022628a5ac6784ddbb17 Mon Sep 17 00:00:00 2001 From: Meeh Date: Fri, 31 Jan 2014 07:35:21 +0100 Subject: [PATCH 3/4] documentation update --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 3117f52a..dc4e0a8c 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,13 @@ First, build it. * $ cd i2pd * $ make -Now, copy your netDb folder from your Java I2P config dir. (The one with r0, r1, r2, ... folders in it) to the source folder where your i2p binary is. - Next, find out your public ip. (find it for example at http://www.whatismyip.com/) Then, run it with: $ ./i2p --host=YOUR_PUBLIC_IP +The client should now reseed by itself. Other options: * --port= - The port to listen on From 849ab03e1c261d96c23db1ca094aa2ab0af984a4 Mon Sep 17 00:00:00 2001 From: Meeh Date: Fri, 31 Jan 2014 07:43:56 +0100 Subject: [PATCH 4/4] Cleanup and final comments --- Reseed.cpp | 12 ++++++------ util.cpp | 3 --- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Reseed.cpp b/Reseed.cpp index 05825f6a..1442b1b4 100644 --- a/Reseed.cpp +++ b/Reseed.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include "Reseed.h" #include "Log.h" @@ -11,7 +10,8 @@ namespace i2p { namespace data { - + //TODO: Implement v2 reseeding. Lightweight zip library is needed. + //TODO: Implement SU3, utils. Reseeder::Reseeder() { } @@ -32,10 +32,11 @@ namespace data LogPrint("Reseed failed"); return false; } - boost::regex e("<\\s*A\\s+[^>]*href\\s*=\\s*\"([^\"]*)\"", - boost::regex::normal | boost::regbase::icase); + boost::regex e("<\\s*A\\s+[^>]*href\\s*=\\s*\"([^\"]*)\"", boost::regex::normal | boost::regbase::icase); boost::sregex_token_iterator i(content.begin(), content.end(), e, 1); boost::sregex_token_iterator j; + //TODO: Ugly code, try to clean up. + //TODO: Try to reduce N number of variables std::string name; std::string routerInfo; std::string tmpUrl; @@ -43,7 +44,6 @@ namespace data std::string ignoreFileSuffix = ".zip"; while (i != j) { - //TODO: Ugly code, try to clean up. name = *i++; if (name.find(ignoreFileSuffix)!=std::string::npos) continue; @@ -52,7 +52,7 @@ namespace data tmpUrl.append(name); routerInfo = i2p::util::http::httpRequest(tmpUrl); filename = "netDb/r"; - filename += name.at(11); + filename += name.at(11); // first char in id filename.append("/"); filename.append(name.c_str()); std::ofstream outfile (filename, std::ios::binary); diff --git a/util.cpp b/util.cpp index 5834e2ab..a33414ac 100644 --- a/util.cpp +++ b/util.cpp @@ -49,8 +49,6 @@ const char* GetCharArg(const std::string& strArg, const std::string& nDefault) namespace http { - //TODO: Implement v2 reseeding. Lightweight zip library is needed. - //TODO: Implement SU3, utils. std::string httpRequest(const std::string& address) { try @@ -61,7 +59,6 @@ namespace http site.connect(u.host_, "http"); if (site) { - //site.expires_from_now (boost::posix_time::seconds (10)); // wait for 10 seconds // User-Agent is needed to get the server list routerInfo files. site << "GET " << u.path_ << " HTTP/1.0\r\nHost: " << u.host_ << "\r\nAccept: */*\r\n" << "User-Agent: Wget/1.11.4\r\n" << "Connection: close\r\n\r\n";