merge chunked responses

AddressBook.cpp

@@ -290,7 +290,7 @@ namespace client
 		
 	}
 
-	void AddressBook::LoadHostsFromStream (std::istream& f, bool isChunked)
+	void AddressBook::LoadHostsFromStream (std::istream& f)
 	{
 		std::unique_lock<std::mutex> l(m_AddressBookMutex);
 		int numAddresses = 0;
@@ -432,7 +432,17 @@ namespace client
 						}	
 					}
 					if (!response.eof ())	
-						m_Book.LoadHostsFromStream (response, isChunked);
+					{
+						if (!isChunked)
+							m_Book.LoadHostsFromStream (response);
+						else
+						{
+							// merge chunks
+							std::stringstream merged;
+							i2p::util::http::MergeChunkedResponse (response, merged);
+							m_Book.LoadHostsFromStream (merged);
+						}	
+					}	
 				}
 				else
 					LogPrint (eLogWarning, "Adressbook HTTP response ", status);

AddressBook.h

@@ -44,7 +44,7 @@ namespace client
 			void InsertAddress (const std::string& address, const std::string& base64); // for jump service
 			void InsertAddress (const i2p::data::IdentityEx& address);
 
-			void LoadHostsFromStream (std::istream& f, bool isChunked = false);
+			void LoadHostsFromStream (std::istream& f);
 			void SetIsDownloading (bool isDownloading) {  m_IsDownloading = isDownloading; };
 			
 		private:

util.cpp

@@ -239,7 +239,7 @@ namespace http
 			if (site)
 			{
 				// User-Agent is needed to get the server list routerInfo files.
-				site << "GET " << u.path_ << " HTTP/1.0\r\nHost: " << u.host_
+				site << "GET " << u.path_ << " HTTP/1.1\r\nHost: " << u.host_
 				<< "\r\nAccept: */*\r\n" << "User-Agent: Wget/1.11.4\r\n" << "Connection: close\r\n\r\n";
 				// read response
 				std::string version, statusMessage;
@@ -249,9 +249,24 @@ namespace http
 				std::getline (site, statusMessage);
 				if (status == 200) // OK
 				{
+					bool isChunked = false;
 					std::string header;
-					while (std::getline(site, header) && header != "\r"){}
+					while (!site.eof () && header != "\r")
+					{
+						std::getline(site, header);
+						auto colon = header.find (':');
+						if (colon != std::string::npos)
+						{
+							std::string field = header.substr (0, colon);
+							if (field == i2p::util::http::TRANSFER_ENCODING)
+								isChunked = (header.find ("chunked", colon + 1) != std::string::npos);
+						}	
+					}
+	
 					std::stringstream ss;
+					if (isChunked)
+						MergeChunkedResponse (site, ss);
+					else	
 						ss << site.rdbuf();
 					return ss.str();
 				}
@@ -274,6 +289,24 @@ namespace http
 		}
 	}
 
+	void MergeChunkedResponse (std::istream& response, std::ostream& merged)
+	{
+		while (!response.eof ())
+		{	
+			std::string hexLen;
+			int len;
+			std::getline (response, hexLen);
+			std::istringstream iss (hexLen);
+			iss >> std::hex >> len;
+			if (!len) break;
+			char * buf = new char[len];
+			response.read (buf, len);
+			merged.write (buf, len);
+			delete[] buf;
+			std::getline (response, hexLen); // read \r\n after chunk
+		}
+	}	
+	
 	int httpRequestViaI2pProxy(const std::string& address, std::string &content)
 	{
 		content = "";

util.h

@@ -3,6 +3,7 @@
 
 #include <map>
 #include <string>
+#include <iostream>
 #include <boost/asio.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
@@ -45,6 +46,7 @@ namespace util
 		const char TRANSFER_ENCODING[] = "Transfer-Encoding";
 
 		std::string httpRequest(const std::string& address);
+		void MergeChunkedResponse (std::istream& response, std::ostream& merged);
 		int httpRequestViaI2pProxy(const std::string& address, std::string &content); // return http code
 		
 		struct url {