From 9a6d478eb1e264646a5593f6d6fa9993759dab09 Mon Sep 17 00:00:00 2001 From: orignal Date: Thu, 18 Feb 2016 13:19:31 -0500 Subject: [PATCH 1/9] handle compressed addressbook --- AddressBook.cpp | 26 +++++++++++++++++++++++--- AddressBook.h | 1 + Base.cpp | 32 ++++++++++++++++++++++++++++++++ Base.h | 8 +++++++- util.h | 1 + 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/AddressBook.cpp b/AddressBook.cpp index 6570e172..ca96882e 100644 --- a/AddressBook.cpp +++ b/AddressBook.cpp @@ -507,6 +507,7 @@ namespace client << "Host: " << u.host_ << "\r\n" << "Accept: */*\r\n" << "User-Agent: Wget/1.11.4\r\n" + //<< "Accept-Encoding: gzip\r\n" << "Connection: close\r\n"; if (m_Etag.length () > 0) // etag request << i2p::util::http::IF_NONE_MATCH << ": \"" << m_Etag << "\"\r\n"; @@ -545,7 +546,7 @@ namespace client response >> status; // status if (status == 200) // OK { - bool isChunked = false; + bool isChunked = false, isGzip = false; std::string header, statusMessage; std::getline (response, statusMessage); // read until new line meaning end of header @@ -563,6 +564,8 @@ namespace client m_LastModified = header.substr (colon + 1); else if (field == i2p::util::http::TRANSFER_ENCODING) isChunked = !header.compare (colon + 1, std::string::npos, "chunked"); + else if (field == i2p::util::http::CONTENT_ENCODING) + isGzip = !header.compare (colon + 1, std::string::npos, "gzip"); } } LogPrint (eLogInfo, "Addressbook: ", m_Link, " ETag: ", m_Etag, " Last-Modified: ", m_LastModified); @@ -570,13 +573,13 @@ namespace client { success = true; if (!isChunked) - m_Book.LoadHostsFromStream (response); + success = ProcessResponse (response, isGzip); else { // merge chunks std::stringstream merged; i2p::util::http::MergeChunkedResponse (response, merged); - m_Book.LoadHostsFromStream (merged); + success = ProcessResponse (merged, isGzip); } } } @@ -599,6 +602,23 @@ namespace client m_Book.DownloadComplete (success); } + + bool AddressBookSubscription::ProcessResponse (std::stringstream& s, bool isGzip) + { + if (isGzip) + { + std::stringstream uncompressed; + i2p::data::GzipInflator inflator; + inflator.Inflate (s, uncompressed); + if (!uncompressed.fail ()) + m_Book.LoadHostsFromStream (uncompressed); + else + return false; + } + else + m_Book.LoadHostsFromStream (s); + return true; + } } } diff --git a/AddressBook.h b/AddressBook.h index f5f5270e..19d8744c 100644 --- a/AddressBook.h +++ b/AddressBook.h @@ -92,6 +92,7 @@ namespace client private: void Request (); + bool ProcessResponse (std::stringstream& s, bool isGzip = false); private: diff --git a/Base.cpp b/Base.cpp index 1479d62f..813b8dfa 100644 --- a/Base.cpp +++ b/Base.cpp @@ -302,6 +302,38 @@ namespace data } } + bool GzipInflator::Inflate (const uint8_t * in, size_t inLen, std::ostream& s) + { + m_IsDirty = true; + uint8_t * out = new uint8_t[GZIP_CHUNK_SIZE]; + m_Inflator.next_in = const_cast(in); + m_Inflator.avail_in = inLen; + int ret; + do + { + m_Inflator.next_out = out; + m_Inflator.avail_out = GZIP_CHUNK_SIZE; + ret = inflate (&m_Inflator, Z_NO_FLUSH); + if (ret < 0) + { + LogPrint (eLogError, "Decompression error ", ret); + inflateEnd (&m_Inflator); + s.setstate(std::ios_base::failbit); + break; + } + else + s.write ((char *)out, GZIP_CHUNK_SIZE - m_Inflator.avail_out); + } + while (!m_Inflator.avail_out); // more data to read + delete[] out; + return ret == Z_STREAM_END || ret < 0; + } + + void GzipInflator::Inflate (const std::stringstream& in, std::ostream& out) + { + Inflate ((const uint8_t *)in.str ().c_str (), in.str ().length (), out); + } + GzipDeflator::GzipDeflator (): m_IsDirty (false) { memset (&m_Deflator, 0, sizeof (m_Deflator)); diff --git a/Base.h b/Base.h index d598542b..b5ac04c3 100644 --- a/Base.h +++ b/Base.h @@ -5,6 +5,8 @@ #include #include #include +#include +#include namespace i2p { @@ -92,6 +94,7 @@ namespace data }; }; + const size_t GZIP_CHUNK_SIZE = 16384; class GzipInflator { public: @@ -100,7 +103,10 @@ namespace data ~GzipInflator (); size_t Inflate (const uint8_t * in, size_t inLen, uint8_t * out, size_t outLen); - + bool Inflate (const uint8_t * in, size_t inLen, std::ostream& s); + // return true when finshed or error, s failbit will be set in case of error + void Inflate (const std::stringstream& in, std::ostream& out); + private: z_stream m_Inflator; diff --git a/util.h b/util.h index 0377ef8d..15765f37 100644 --- a/util.h +++ b/util.h @@ -34,6 +34,7 @@ namespace util const char IF_MODIFIED_SINCE[] = "If-Modified-Since"; const char LAST_MODIFIED[] = "Last-Modified"; const char TRANSFER_ENCODING[] = "Transfer-Encoding"; + const char CONTENT_ENCODING[] = "Content-Encoding"; std::string GetHttpContent (std::istream& response); void MergeChunkedResponse (std::istream& response, std::ostream& merged); From e2aa2709acedcc58aa9ddeba62078e6b7caf93e5 Mon Sep 17 00:00:00 2001 From: orignal Date: Thu, 18 Feb 2016 15:57:43 -0500 Subject: [PATCH 2/9] family added --- Family.cpp | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Family.h | 32 ++++++++++++++++++++++ NetDb.cpp | 1 + NetDb.h | 2 ++ filelist.mk | 3 ++- 5 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 Family.cpp create mode 100644 Family.h diff --git a/Family.cpp b/Family.cpp new file mode 100644 index 00000000..211d2f54 --- /dev/null +++ b/Family.cpp @@ -0,0 +1,76 @@ +#include +#include +#include "util.h" +#include "Log.h" +#include "Family.h" + +namespace i2p +{ +namespace data +{ + Families::Families () + { + } + + Families::~Families () + { + } + + void Families::LoadCertificate (const std::string& filename) + { + SSL_CTX * ctx = SSL_CTX_new (TLSv1_method ()); + int ret = SSL_CTX_use_certificate_file (ctx, filename.c_str (), SSL_FILETYPE_PEM); + if (ret) + { + SSL * ssl = SSL_new (ctx); + X509 * cert = SSL_get_certificate (ssl); + // verify + if (cert) + { + // extract issuer name + char name[100]; + X509_NAME_oneline (X509_get_issuer_name(cert), name, 100); + auto pkey = X509_get_pubkey (cert); + int keyType = EVP_PKEY_type(pkey->type); + switch (keyType) + { + case EVP_PKEY_DSA: + // TODO: + break; + case EVP_PKEY_EC: + { + //EC_KEY * ecKey = EVP_PKEY_get0_EC_KEY (pkey); + break; + } + default: + LogPrint (eLogWarning, "Family: Certificate key type ", keyType, " is not supported"); + } + } + SSL_free (ssl); + } + else + LogPrint (eLogError, "Family: Can't open certificate file ", filename); + SSL_CTX_free (ctx); + } + + void Families::LoadCertificates () + { + boost::filesystem::path familyDir = i2p::util::filesystem::GetCertificatesDir() / "family"; + + if (!boost::filesystem::exists (familyDir)) return; + int numCertificates = 0; + boost::filesystem::directory_iterator end; // empty + for (boost::filesystem::directory_iterator it (familyDir); it != end; ++it) + { + if (boost::filesystem::is_regular_file (it->status()) && it->path ().extension () == ".crt") + { + LoadCertificate (it->path ().string ()); + numCertificates++; + } + } + if (numCertificates > 0) + LogPrint (eLogInfo, "Family: ", numCertificates, " certificates loaded"); + } +} +} + diff --git a/Family.h b/Family.h new file mode 100644 index 00000000..ca8dac3f --- /dev/null +++ b/Family.h @@ -0,0 +1,32 @@ +#ifndef FAMILY_H__ +#define FAMILY_H_ + +#include +#include +#include +#include "Signature.h" + +namespace i2p +{ +namespace data +{ + class Families + { + public: + + Families (); + ~Families (); + void LoadCertificates (); + + private: + + void LoadCertificate (const std::string& filename); + + private: + + std::map > m_SigningKeys; + }; +} +} + +#endif diff --git a/NetDb.cpp b/NetDb.cpp index b4b4ef08..df464664 100644 --- a/NetDb.cpp +++ b/NetDb.cpp @@ -37,6 +37,7 @@ namespace data void NetDb::Start () { + m_Families.LoadCertificates (); Load (); if (m_RouterInfos.size () < 25) // reseed if # of router less than 50 Reseed (); diff --git a/NetDb.h b/NetDb.h index cad00aa7..33fd6e27 100644 --- a/NetDb.h +++ b/NetDb.h @@ -18,6 +18,7 @@ #include "TunnelPool.h" #include "Reseed.h" #include "NetDbRequests.h" +#include "Family.h" namespace i2p { @@ -95,6 +96,7 @@ namespace data GzipInflator m_Inflator; Reseeder * m_Reseeder; + Families m_Families; friend class NetDbRequests; NetDbRequests m_Requests; diff --git a/filelist.mk b/filelist.mk index 1d46b8fc..166be50e 100644 --- a/filelist.mk +++ b/filelist.mk @@ -4,7 +4,8 @@ LIB_SRC = \ Reseed.cpp RouterContext.cpp RouterInfo.cpp Signature.cpp SSU.cpp \ SSUSession.cpp SSUData.cpp Streaming.cpp Identity.cpp TransitTunnel.cpp \ Transports.cpp Tunnel.cpp TunnelEndpoint.cpp TunnelPool.cpp TunnelGateway.cpp \ - Destination.cpp Base.cpp I2PEndian.cpp Config.cpp util.cpp api.cpp + Destination.cpp Base.cpp I2PEndian.cpp Config.cpp Family.cpp util.cpp \ + api.cpp LIB_CLIENT_SRC = \ AddressBook.cpp BOB.cpp ClientContext.cpp I2PTunnel.cpp I2PService.cpp \ From 47bf0ef591eb2b4b32be1e1c194b8d1227986f97 Mon Sep 17 00:00:00 2001 From: orignal Date: Thu, 18 Feb 2016 16:28:43 -0500 Subject: [PATCH 3/9] free pkey after usage --- Family.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Family.cpp b/Family.cpp index 211d2f54..6c96812c 100644 --- a/Family.cpp +++ b/Family.cpp @@ -45,6 +45,7 @@ namespace data default: LogPrint (eLogWarning, "Family: Certificate key type ", keyType, " is not supported"); } + EVP_PKEY_free (pkey); } SSL_free (ssl); } From 3053a9b6a0bc417202b1720d4e73b6ef22d7f969 Mon Sep 17 00:00:00 2001 From: orignal Date: Thu, 18 Feb 2016 20:35:14 -0500 Subject: [PATCH 4/9] enable i2p gzip compression --- AddressBook.cpp | 4 ++++ Base.cpp | 5 +++-- Base.h | 2 +- util.h | 3 ++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/AddressBook.cpp b/AddressBook.cpp index ca96882e..006d503c 100644 --- a/AddressBook.cpp +++ b/AddressBook.cpp @@ -508,6 +508,7 @@ namespace client << "Accept: */*\r\n" << "User-Agent: Wget/1.11.4\r\n" //<< "Accept-Encoding: gzip\r\n" + << "X-Accept-Encoding: x-i2p-gzip;q=1.0, identity;q=0.5, deflate;q=0, gzip;q=0, *;q=0\r\n" << "Connection: close\r\n"; if (m_Etag.length () > 0) // etag request << i2p::util::http::IF_NONE_MATCH << ": \"" << m_Etag << "\"\r\n"; @@ -557,6 +558,7 @@ namespace client if (colon != std::string::npos) { std::string field = header.substr (0, colon); + colon++; header.resize (header.length () - 1); // delete \r if (field == i2p::util::http::ETAG) m_Etag = header.substr (colon + 1); @@ -566,6 +568,8 @@ namespace client isChunked = !header.compare (colon + 1, std::string::npos, "chunked"); else if (field == i2p::util::http::CONTENT_ENCODING) isGzip = !header.compare (colon + 1, std::string::npos, "gzip"); + else if (field == i2p::util::http::CONTENT_ENCODING1) // Content-encoding + isGzip = !header.compare (colon + 1, std::string::npos, "x-i2p-gzip"); } } LogPrint (eLogInfo, "Addressbook: ", m_Link, " ETag: ", m_Etag, " Last-Modified: ", m_LastModified); diff --git a/Base.cpp b/Base.cpp index 813b8dfa..cb6faf2d 100644 --- a/Base.cpp +++ b/Base.cpp @@ -329,9 +329,10 @@ namespace data return ret == Z_STREAM_END || ret < 0; } - void GzipInflator::Inflate (const std::stringstream& in, std::ostream& out) + void GzipInflator::Inflate (std::stringstream& in, std::ostream& out) { - Inflate ((const uint8_t *)in.str ().c_str (), in.str ().length (), out); + auto str = in.str ().substr (in.tellg ()); + Inflate ((const uint8_t *)str.c_str (), str.length (), out); } GzipDeflator::GzipDeflator (): m_IsDirty (false) diff --git a/Base.h b/Base.h index b5ac04c3..19dc31da 100644 --- a/Base.h +++ b/Base.h @@ -105,7 +105,7 @@ namespace data size_t Inflate (const uint8_t * in, size_t inLen, uint8_t * out, size_t outLen); bool Inflate (const uint8_t * in, size_t inLen, std::ostream& s); // return true when finshed or error, s failbit will be set in case of error - void Inflate (const std::stringstream& in, std::ostream& out); + void Inflate (std::stringstream& in, std::ostream& out); private: diff --git a/util.h b/util.h index 15765f37..3b05c170 100644 --- a/util.h +++ b/util.h @@ -35,7 +35,8 @@ namespace util const char LAST_MODIFIED[] = "Last-Modified"; const char TRANSFER_ENCODING[] = "Transfer-Encoding"; const char CONTENT_ENCODING[] = "Content-Encoding"; - + const char CONTENT_ENCODING1[] = "Content-encoding"; + std::string GetHttpContent (std::istream& response); void MergeChunkedResponse (std::istream& response, std::ostream& merged); std::string urlDecode(const std::string& data); From 094d9193b9b0662863578495cafed599dab65237 Mon Sep 17 00:00:00 2001 From: orignal Date: Thu, 18 Feb 2016 22:34:14 -0500 Subject: [PATCH 5/9] start addressbook first --- ClientContext.cpp | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/ClientContext.cpp b/ClientContext.cpp index 6d05cde1..9967a33a 100644 --- a/ClientContext.cpp +++ b/ClientContext.cpp @@ -37,6 +37,8 @@ namespace client m_SharedLocalDestination->Start (); } + m_AddressBook.Start (); + std::shared_ptr localDestination; bool httproxy; i2p::config::GetOption("httpproxy.enabled", httproxy); if (httproxy) { @@ -94,25 +96,19 @@ namespace client m_BOBCommandChannel = new BOBCommandChannel (bobAddr, bobPort); m_BOBCommandChannel->Start (); } - - m_AddressBook.Start (); } void ClientContext::Stop () { - if (m_HttpProxy) { - LogPrint(eLogInfo, "Clients: stopping HTTP Proxy"); - m_HttpProxy->Stop(); - delete m_HttpProxy; - m_HttpProxy = nullptr; - } + LogPrint(eLogInfo, "Clients: stopping HTTP Proxy"); + m_HttpProxy->Stop(); + delete m_HttpProxy; + m_HttpProxy = nullptr; - if (m_SocksProxy) { - LogPrint(eLogInfo, "Clients: stopping SOCKS Proxy"); - m_SocksProxy->Stop(); - delete m_SocksProxy; - m_SocksProxy = nullptr; - } + LogPrint(eLogInfo, "Clients: stopping SOCKS Proxy"); + m_SocksProxy->Stop(); + delete m_SocksProxy; + m_SocksProxy = nullptr; for (auto& it: m_ClientTunnels) { From 76b49f6985bf4c7b80c667c94c301c64dec10576 Mon Sep 17 00:00:00 2001 From: orignal Date: Thu, 18 Feb 2016 22:34:55 -0500 Subject: [PATCH 6/9] uncompress stream by chunks --- Base.cpp | 11 ++++++++--- Base.h | 3 +-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Base.cpp b/Base.cpp index cb6faf2d..3017d910 100644 --- a/Base.cpp +++ b/Base.cpp @@ -329,10 +329,15 @@ namespace data return ret == Z_STREAM_END || ret < 0; } - void GzipInflator::Inflate (std::stringstream& in, std::ostream& out) + void GzipInflator::Inflate (std::istream& in, std::ostream& out) { - auto str = in.str ().substr (in.tellg ()); - Inflate ((const uint8_t *)str.c_str (), str.length (), out); + uint8_t * buf = new uint8_t[GZIP_CHUNK_SIZE]; + while (!in.eof ()) + { + in.read ((char *)buf, GZIP_CHUNK_SIZE); + Inflate (buf, in.gcount (), out); + } + delete[] buf; } GzipDeflator::GzipDeflator (): m_IsDirty (false) diff --git a/Base.h b/Base.h index 19dc31da..e6f9567d 100644 --- a/Base.h +++ b/Base.h @@ -6,7 +6,6 @@ #include #include #include -#include namespace i2p { @@ -105,7 +104,7 @@ namespace data size_t Inflate (const uint8_t * in, size_t inLen, uint8_t * out, size_t outLen); bool Inflate (const uint8_t * in, size_t inLen, std::ostream& s); // return true when finshed or error, s failbit will be set in case of error - void Inflate (std::stringstream& in, std::ostream& out); + void Inflate (std::istream& in, std::ostream& out); private: From f1d4818045320816ce4220abb229bb1e536d5236 Mon Sep 17 00:00:00 2001 From: orignal Date: Thu, 18 Feb 2016 22:39:09 -0500 Subject: [PATCH 7/9] Family.cpp added --- build/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/build/CMakeLists.txt b/build/CMakeLists.txt index 646e8a6e..f7e884f9 100644 --- a/build/CMakeLists.txt +++ b/build/CMakeLists.txt @@ -45,6 +45,7 @@ set (LIBI2PD_SRC "${CMAKE_SOURCE_DIR}/Base.cpp" "${CMAKE_SOURCE_DIR}/util.cpp" "${CMAKE_SOURCE_DIR}/Datagram.cpp" + "${CMAKE_SOURCE_DIR}/Family.cpp" "${CMAKE_SOURCE_DIR}/Signature.cpp" "${CMAKE_SOURCE_DIR}/api.cpp" ) From c5b6da7201717039ce8cdac30bb4ea6fc69e7e0d Mon Sep 17 00:00:00 2001 From: orignal Date: Fri, 19 Feb 2016 10:04:52 -0500 Subject: [PATCH 8/9] case-insensitive http responses --- AddressBook.cpp | 6 +++--- util.cpp | 1 + util.h | 15 ++++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/AddressBook.cpp b/AddressBook.cpp index 006d503c..f08a4041 100644 --- a/AddressBook.cpp +++ b/AddressBook.cpp @@ -558,6 +558,7 @@ namespace client if (colon != std::string::npos) { std::string field = header.substr (0, colon); + boost::to_lower (field); // field are not case-sensitive colon++; header.resize (header.length () - 1); // delete \r if (field == i2p::util::http::ETAG) @@ -567,9 +568,8 @@ namespace client else if (field == i2p::util::http::TRANSFER_ENCODING) isChunked = !header.compare (colon + 1, std::string::npos, "chunked"); else if (field == i2p::util::http::CONTENT_ENCODING) - isGzip = !header.compare (colon + 1, std::string::npos, "gzip"); - else if (field == i2p::util::http::CONTENT_ENCODING1) // Content-encoding - isGzip = !header.compare (colon + 1, std::string::npos, "x-i2p-gzip"); + isGzip = !header.compare (colon + 1, std::string::npos, "gzip") || + !header.compare (colon + 1, std::string::npos, "x-i2p-gzip"); } } LogPrint (eLogInfo, "Addressbook: ", m_Link, " ETag: ", m_Etag, " Last-Modified: ", m_LastModified); diff --git a/util.cpp b/util.cpp index 1ea5fcd7..e25ddb3f 100644 --- a/util.cpp +++ b/util.cpp @@ -212,6 +212,7 @@ namespace http if (colon != std::string::npos) { std::string field = header.substr (0, colon); + boost::to_lower (field); if (field == i2p::util::http::TRANSFER_ENCODING) isChunked = (header.find ("chunked", colon + 1) != std::string::npos); } diff --git a/util.h b/util.h index 3b05c170..81f178f8 100644 --- a/util.h +++ b/util.h @@ -29,14 +29,15 @@ namespace util namespace http { - const char ETAG[] = "ETag"; + // in (lower case) + const char ETAG[] = "etag"; // ETag + const char LAST_MODIFIED[] = "last-modified"; // Last-Modified + const char TRANSFER_ENCODING[] = "transfer-encoding"; // Transfer-Encoding + const char CONTENT_ENCODING[] = "content-encoding"; // Content-Encoding + // out const char IF_NONE_MATCH[] = "If-None-Match"; - const char IF_MODIFIED_SINCE[] = "If-Modified-Since"; - const char LAST_MODIFIED[] = "Last-Modified"; - const char TRANSFER_ENCODING[] = "Transfer-Encoding"; - const char CONTENT_ENCODING[] = "Content-Encoding"; - const char CONTENT_ENCODING1[] = "Content-encoding"; - + const char IF_MODIFIED_SINCE[] = "If-Modified-Since"; + std::string GetHttpContent (std::istream& response); void MergeChunkedResponse (std::istream& response, std::ostream& merged); std::string urlDecode(const std::string& data); From cb64072f7b2e48155b83f9067480c9e38c48a28a Mon Sep 17 00:00:00 2001 From: orignal Date: Fri, 19 Feb 2016 11:18:01 -0500 Subject: [PATCH 9/9] fixed windows build --- Family.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Family.cpp b/Family.cpp index 6c96812c..5470a92c 100644 --- a/Family.cpp +++ b/Family.cpp @@ -1,6 +1,6 @@ -#include -#include #include "util.h" +#include +#include #include "Log.h" #include "Family.h"