process and save new RouterInfo

I2NPProtocol.cpp

@@ -1,9 +1,9 @@
 #include <string.h>
 #include <endian.h>
-#include <cryptopp/gzip.h>
 #include <cryptopp/sha.h>
 #include <cryptopp/modes.h>
 #include <cryptopp/aes.h>
+#include <cryptopp/gzip.h>
 #include "ElGamal.h"
 #include "Timestamp.h"
 #include "RouterContext.h"
@@ -129,16 +129,16 @@ namespace i2p
 		memcpy (msg->key, context.GetRouterInfo ().GetIdentHash (), 32);
 		msg->type = 0;
 		msg->replyToken = 0;
-		msg->size = 0;
 		
 		CryptoPP::Gzip compressor;
 		compressor.Put ((uint8_t *)context.GetRouterInfo ().GetBuffer (), context.GetRouterInfo ().GetBufferLen ());
 		compressor.MessageEnd();
 		int size = compressor.MaxRetrievable ();
-		msg->size = htobe16 (size);
 		uint8_t * buf = m->GetPayload () + sizeof (I2NPDatabaseStoreMsg);
+		*(uint16_t *)buf = htobe16 (size); // size
+		buf += 2;
 		compressor.Get (buf, size); 
-		m->len += sizeof (I2NPDatabaseStoreMsg) + size; // payload size
+		m->len += sizeof (I2NPDatabaseStoreMsg) + 2 + size; // payload size
 		FillI2NPMessageHeader (m, eI2NPDatabaseStore);
 		
 		return m;
@@ -147,21 +147,31 @@ namespace i2p
 	void HandleDatabaseStoreMsg (uint8_t * buf, size_t len)
 	{		
 		I2NPDatabaseStoreMsg * msg = (I2NPDatabaseStoreMsg *)buf;
+		size_t offset = sizeof (I2NPDatabaseStoreMsg);
+		if (msg->replyToken)
+			offset += 36;
 		if (msg->type)
 		{
 			LogPrint ("LeaseSet");
-			i2p::data::netdb.AddLeaseSet (buf + sizeof (I2NPDatabaseStoreMsg)-2, len - sizeof (I2NPDatabaseStoreMsg)+2);
+			i2p::data::netdb.AddLeaseSet (buf + offset, len - offset);
 		}	
 		else
 		{
 			LogPrint ("RouterInfo");
+			size_t size = be16toh (*(uint16_t *)(buf + offset));
+			if (size > 2048)
+			{
+				LogPrint ("Invalid RouterInfo length ", (int)size);
+				return;
+			}	
+			offset += 2;
 			CryptoPP::Gunzip decompressor;
-			decompressor.Put (buf + sizeof (I2NPDatabaseStoreMsg), be16toh (msg->size));
+			decompressor.Put (buf + offset, size);
 			decompressor.MessageEnd();
-			uint8_t uncompressed[1024];
+			uint8_t uncompressed[2048];
-			int size = decompressor.MaxRetrievable ();
+			int uncomressedSize = decompressor.MaxRetrievable ();
-			decompressor.Get (uncompressed, size);
+			decompressor.Get (uncompressed, uncomressedSize);
-			i2p::data::netdb.AddRouterInfo (uncompressed, size);
+			i2p::data::netdb.AddRouterInfo (uncompressed, uncomressedSize);
 		}	
 	}	
 
@@ -421,9 +431,6 @@ namespace i2p
 			case eI2NPGarlic:
 				LogPrint ("Garlic");
 			break;	
-			case eI2NPDatabaseStore:
-				LogPrint ("DatabaseStore");
-				HandleDatabaseStoreMsg (buf, size);
 			break;	
 			case eI2NPDatabaseSearchReply:
 				LogPrint ("DatabaseSearchReply");
@@ -449,20 +456,23 @@ namespace i2p
 	{
 		if (msg)
 		{	
-			if (msg->GetHeader ()->typeID == eI2NPTunnelData)
+			switch (msg->GetHeader ()->typeID)
-			{
-				LogPrint ("TunnelData");
-				i2p::tunnel::tunnels.PostTunnelData (msg);
-			}
-			else if (msg->GetHeader ()->typeID == eI2NPTunnelGateway)
-			{
-				LogPrint ("TunnelGateway");
-				HandleTunnelGatewayMsg (msg);
-			}	
-			else
 			{	
-				HandleI2NPMessage (msg->GetBuffer (), msg->GetLength ());
+				case eI2NPTunnelData:
-				DeleteI2NPMessage (msg);
+					LogPrint ("TunnelData");
+					i2p::tunnel::tunnels.PostTunnelData (msg);
+				break;	
+				case eI2NPTunnelGateway:
+					LogPrint ("TunnelGateway");
+					HandleTunnelGatewayMsg (msg);
+				break;
+				case eI2NPDatabaseStore:
+					LogPrint ("DatabaseStore");
+					i2p::data::netdb.PostDatabaseStoreMsg (msg);
+				break;	
+				default:
+					HandleI2NPMessage (msg->GetBuffer (), msg->GetLength ());
+					DeleteI2NPMessage (msg);
 			}	
 		}	
 	}	

I2NPProtocol.h

@@ -24,7 +24,6 @@ namespace i2p
 		uint8_t key[32];
 		uint8_t type;
 		uint32_t replyToken;	
-		uint16_t size;		
 	};
 
 	

NetDb.cpp

@@ -1,5 +1,7 @@
+#include <fstream>
 #include <boost/filesystem.hpp>
 #include "Log.h"
+#include "Timestamp.h"
 #include "I2NPProtocol.h"
 #include "Tunnel.h"
 #include "RouterContext.h"
@@ -12,9 +14,8 @@ namespace data
 	
 	NetDb netdb;
 
-	NetDb::NetDb (): m_IsRunning (false), m_Thread (0)
+	NetDb::NetDb (): m_IsRunning (false), m_Thread (0), m_LastFloodfill (0)
 	{
-		Load ("netDb");
 	}
 	
 	NetDb::~NetDb ()
@@ -28,6 +29,7 @@ namespace data
 
 	void NetDb::Start ()
 	{
+		Load ("netDb");
 		m_Thread = new std::thread (std::bind (&NetDb::Run, this));
 	}
 	
@@ -44,11 +46,38 @@ namespace data
 	
 	void NetDb::Run ()
 	{
+		uint32_t lastTs = 0;
 		m_IsRunning = true;
 		while (m_IsRunning)
 		{	
-			sleep (10);
+			I2NPMessage * msg = m_Queue.GetNextWithTimeout (10000); // 10 sec
-			Explore ();
+			if (msg)
+			{	
+				while (msg)
+				{
+					if (msg->GetHeader ()->typeID == eI2NPDatabaseStore)
+					{	
+						i2p::HandleDatabaseStoreMsg (msg->GetPayload (), msg->GetLength ()); // TODO
+						i2p::DeleteI2NPMessage (msg);
+					}	
+					else // WTF?
+					{
+						LogPrint ("NetDb: unexpected message type ", msg->GetHeader ()->typeID);
+						i2p::HandleI2NPMessage (msg);
+					}	
+					msg = m_Queue.Get ();
+				}	
+			}
+			else // if no new DatabaseStore coming, explore it
+				Explore ();
+
+			uint32_t ts = i2p::util::GetSecondsSinceEpoch ();
+			if (ts - lastTs >= 60) // save routers every minute
+			{
+				if (lastTs)
+					SaveUpdated ("netDb");
+				lastTs = ts;
+			}	
 		}	
 	}	
 	
@@ -98,6 +127,22 @@ namespace data
 			LogPrint (directory, " doesn't exist");
 	}	
 
+	void NetDb::SaveUpdated (const char * directory)
+	{	
+		int count = 0;
+		for (auto it: m_RouterInfos)
+			if (it.second->IsUpdated ())
+			{
+				std::ofstream r (std::string (directory) + "/routerInfo-" + 
+					it.second->GetIdentHashBase64 () + ".dat");
+				r.write ((char *)it.second->GetBuffer (), it.second->GetBufferLen ());
+				it.second->SetUpdated (false);
+				count++;
+			}
+		if (count > 0)
+			LogPrint (count," new routers saved");
+	}
+	
 	void NetDb::RequestDestination (const uint8_t * destination, const uint8_t * router)
 	{
 		i2p::tunnel::OutboundTunnel * outbound = i2p::tunnel::tunnels.GetNextOutboundTunnel ();
@@ -122,12 +167,16 @@ namespace data
 		if (!memcmp (m_Exploratory, key, 32))
 		{
 			if (m_RouterInfos.find (std::string ((const char *)router, 32)) == m_RouterInfos.end ())
-				LogPrint ("Found new router");
+			{	
+				LogPrint ("Found new router. Requesting RouterInfo ...");
+				if (m_LastFloodfill)
+					RequestDestination (router, m_LastFloodfill->GetIdentHash ());
+			}	
 			else
 				LogPrint ("Bayan");
 		}
-		else
+	//	else
-			RequestDestination (key, router);
+	//		RequestDestination (key, router);
 	}	
 
 	void NetDb::Explore ()
@@ -136,15 +185,15 @@ namespace data
 		i2p::tunnel::InboundTunnel * inbound = i2p::tunnel::tunnels.GetNextInboundTunnel ();
 		if (outbound && inbound)
 		{
-			const RouterInfo * floodFill = GetRandomNTCPRouter (true);
+			m_LastFloodfill = GetRandomNTCPRouter (true);
-			if (floodFill)
+			if (m_LastFloodfill)
 			{
 				LogPrint ("Exploring new routers ...");
 				CryptoPP::RandomNumberGenerator& rnd = i2p::context.GetRandomNumberGenerator ();
 				rnd.GenerateBlock (m_Exploratory, 32);
 				I2NPMessage * msg = i2p::CreateDatabaseLookupMsg (m_Exploratory, inbound->GetNextIdentHash (), 
 					inbound->GetNextTunnelID (), true);
-				outbound->SendTunnelDataMsg (floodFill->GetIdentHash (), 0, msg);
+				outbound->SendTunnelDataMsg (m_LastFloodfill->GetIdentHash (), 0, msg);
 			}	
 		}
 	}	
@@ -175,5 +224,10 @@ namespace data
 		}	
 		return nullptr;
 	}	
+
+	void NetDb::PostDatabaseStoreMsg (I2NPMessage * msg)
+	{
+		if (msg) m_Queue.Put (msg);	
+	}	
 }
 }

NetDb.h

@@ -5,6 +5,8 @@
 #include <map>
 #include <string>
 #include <thread>
+#include "Queue.h"
+#include "I2NPProtocol.h"
 #include "RouterInfo.h"
 #include "LeaseSet.h"
 
@@ -32,9 +34,12 @@ namespace data
 			const RouterInfo * GetRandomNTCPRouter (bool floodfillOnly = false) const;
 			const RouterInfo * GetRandomRouter () const;
 
+			void PostDatabaseStoreMsg (I2NPMessage * msg);
+			
 		private:
 
 			void Load (const char * directory);
+			void SaveUpdated (const char * directory);
 			void Run (); // exploratory thread
 			void Explore ();
 			
@@ -46,6 +51,8 @@ namespace data
 			bool m_IsRunning;
 			std::thread * m_Thread;	
 			uint8_t m_Exploratory[32];
+			const RouterInfo * m_LastFloodfill;
+			i2p::util::Queue<I2NPMessage> m_Queue; // of I2NPDatabaseStoreMsg
 	};
 
 	extern NetDb netdb;

RouterInfo.cpp

@@ -16,12 +16,14 @@ namespace i2p
 {
 namespace data
 {		
-	RouterInfo::RouterInfo (const char * filename)
+	RouterInfo::RouterInfo (const char * filename):
+		m_IsUpdated (false)
 	{
 		ReadFromFile (filename);
 	}	
 
-	RouterInfo::RouterInfo (const uint8_t * buf, int len)
+	RouterInfo::RouterInfo (const uint8_t * buf, int len):
+		m_IsUpdated (true)
 	{
 		memcpy (m_Buffer, buf, len);
 		m_BufferLen = len;
@@ -122,6 +124,8 @@ namespace data
 		}		
 		
 		CryptoPP::SHA256().CalculateDigest(m_IdentHash, (uint8_t *)&m_RouterIdentity, sizeof (m_RouterIdentity));
+		size_t l = i2p::data::ByteStreamToBase64 (m_IdentHash, 32, m_IdentHashBase64, 48);
+		m_IdentHashBase64[l] = 0;
 	}	
 
 	void RouterInfo::WriteToStream (std::ostream& s)
@@ -179,6 +183,7 @@ namespace data
 
 	void RouterInfo::CreateBuffer ()
 	{
+		m_Timestamp = i2p::util::GetMillisecondsSinceEpoch (); // refresh timstamp
 		std::stringstream s;
 		WriteToStream (s);
 		m_BufferLen = s.str ().size ();

RouterInfo.h

@@ -51,6 +51,7 @@ namespace data
 			const RouterIdentity& GetRouterIdentity () const { return m_RouterIdentity; };
 			void SetRouterIdentity (const RouterIdentity& identity);
 			const uint8_t * GetIdentHash () const { return m_IdentHash; };
+			const char * GetIdentHashBase64 () const { return m_IdentHashBase64; };
 			const std::vector<Address>& GetAddresses () const { return m_Addresses; };
 			Address * GetNTCPAddress ();
 			
@@ -64,6 +65,9 @@ namespace data
 			const char * GetBuffer () const  { return m_Buffer; };
 			int GetBufferLen () const { return m_BufferLen; };
 
+			bool IsUpdated () const { return m_IsUpdated; };
+			void SetUpdated (bool updated) { m_IsUpdated = updated; }; 
+			
 		private:
 
 			void ReadFromFile (const char * filename);
@@ -77,11 +81,13 @@ namespace data
 
 			RouterIdentity m_RouterIdentity;
 			uint8_t m_IdentHash[32];
+			char m_IdentHashBase64[48];
 			char m_Buffer[2048];
 			int m_BufferLen;
 			uint64_t m_Timestamp;
 			std::vector<Address> m_Addresses;
 			std::map<std::string, std::string> m_Properties;
+			bool m_IsUpdated;
 	};	
 }	
 }