0.9.28: better enforcement of DHT upload rate limit for non-locally generated requests.

the limit applies to: DHT replies, refreshes of stored items, checking for status/tracker and proxy server.
local requests (eg. dhtgets from the UI) are excluded from this limit, so user experience is not affected.
in other words: we limit only the band contributed back to twister network for maintenance tasks.

this should improve resilience against some sorts of denial-of-service attacks and also
prevents saturating the uplink as observed recently.

libtorrent/include/libtorrent/aux_/session_impl.hpp

@@ -310,7 +310,7 @@ namespace libtorrent
 			void dht_putDataSigned(std::string const &username, std::string const &resource, bool multi,
 			         entry const &p, std::string const &sig_p, std::string const &sig_user, bool local);
 
-			void dht_getData(std::string const &username, std::string const &resource, bool multi);
+			void dht_getData(std::string const &username, std::string const &resource, bool multi, bool local);
 			entry dht_getLocalData() const;
 
 

libtorrent/include/libtorrent/kademlia/dht_get.hpp

@@ -74,7 +74,7 @@ public:
 		, std::string const &targetUser, std::string const &targetResource, bool multi
 		, data_callback const& dcallback
 		, nodes_callback const& ncallback
-		, bool justToken);
+		, bool justToken, bool dontDrop );
 
 	virtual char const* name() const { return "getData"; }
 
@@ -98,6 +98,7 @@ private:
 	bool m_done:1;
 	bool m_got_data:1;
 	bool m_justToken:1;
+	bool m_dontDrop:1;
 
 	friend class dht_get_observer;
 };

libtorrent/include/libtorrent/kademlia/dht_tracker.hpp

@@ -101,7 +101,7 @@ namespace libtorrent { namespace dht
 
 		void getData(std::string const &username, std::string const &resource, bool multi,
 			     boost::function<void(entry::list_type const&)> fdata,
-			     boost::function<void(bool, bool)> fdone);
+			     boost::function<void(bool, bool)> fdone, bool local);
 
 		void dht_status(session_status& s);
 		void network_stats(int& sent, int& received);

libtorrent/include/libtorrent/kademlia/node.hpp

@@ -231,7 +231,7 @@ public:
 
 	void getData(std::string const &username, std::string const &resource, bool multi,
 		     boost::function<void(entry::list_type const&)> fdata,
-		     boost::function<void(bool, bool)> fdone);
+		     boost::function<void(bool, bool)> fdone, bool local);
 
 	bool verify_token(std::string const& token, char const* info_hash
 		, udp::endpoint const& addr);

libtorrent/include/libtorrent/kademlia/observer.hpp

@@ -76,6 +76,7 @@ struct observer : boost::noncopyable
 		, m_port(0)
 		, m_transaction_id()
 		, flags(0)
+		, m_dont_drop(true)
 	{
 		TORRENT_ASSERT(a);
 #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS
@@ -171,6 +172,7 @@ public:
 	bool m_was_abandoned:1;
 	bool m_in_use:1;
 #endif
+	bool m_dont_drop:1;
 };
 
 typedef boost::intrusive_ptr<observer> observer_ptr;

libtorrent/include/libtorrent/session.hpp

@@ -441,7 +441,7 @@ namespace libtorrent
 		void dht_putDataSigned(std::string const &username, std::string const &resource, bool multi,
 		                       entry const &p, std::string const &sig_p, std::string const &sig_user, bool local);
 
-		void dht_getData(std::string const &username, std::string const &resource, bool multi);
+		void dht_getData(std::string const &username, std::string const &resource, bool multi, bool local);
 		entry dht_getLocalData() const;
 
 #ifndef TORRENT_NO_DEPRECATE

libtorrent/src/kademlia/dht_get.cpp

@@ -222,7 +222,8 @@ dht_get::dht_get(
 	, bool multi
 	, data_callback const& dcallback
 	, nodes_callback const& ncallback
-	, bool justToken)
+	, bool justToken
+	, bool dontDrop)
 	: traversal_algorithm(node, node_id())
 	, m_data_callback(dcallback)
 	, m_nodes_callback(ncallback)
@@ -233,6 +234,7 @@ dht_get::dht_get(
 	, m_done(false)
 	, m_got_data(false)
 	, m_justToken(justToken)
+	, m_dontDrop(dontDrop)
 {
 	m_target["n"] = m_targetUser;
 	m_target["r"] = m_targetResource;
@@ -276,6 +278,7 @@ bool dht_get::invoke(observer_ptr o)
 	entry& target = a["target"];
 	target = m_target;
 	if (m_justToken) a["justtoken"] = 1;
+	o->m_dont_drop = m_dontDrop;
 	return m_node.m_rpc.invoke(e, o->target_ep(), o);
 }
 

libtorrent/src/kademlia/dht_tracker.cpp

@@ -441,9 +441,9 @@ namespace libtorrent { namespace dht
 
 	void dht_tracker::getData(std::string const &username, std::string const &resource, bool multi,
 				  boost::function<void(entry::list_type const&)> fdata,
-				  boost::function<void(bool, bool)> fdone)
+				  boost::function<void(bool, bool)> fdone, bool local)
 	{
-		m_dht.getData(username, resource, multi, fdata, fdone);
+		m_dht.getData(username, resource, multi, fdata, fdone, local);
 	}
 
 

libtorrent/src/kademlia/node.cpp

@@ -239,8 +239,9 @@ void node_impl::incoming(msg const& m)
 	if (!y_ent || y_ent->string_length() == 0)
 	{
 		entry e;
-		incoming_error(e, "missing 'y' entry");
+		incoming_error(e, "missing 'z' entry");
-		m_sock->send_packet(e, m.addr, 0);
+		// [MF] silently ignore bad packet
+		//m_sock->send_packet(e, m.addr, 0);
 		return;
 	}
 
@@ -514,7 +515,7 @@ void node_impl::putDataSigned(std::string const &username, std::string const &re
         // for info-hash id. then send putData to them.
         boost::intrusive_ptr<dht_get> ta(new dht_get(*this, username, resource, multi,
              boost::bind(&nop),
-             boost::bind(&putData_fun, _1, boost::ref(*this), p, sig_p, sig_user), true));
+             boost::bind(&putData_fun, _1, boost::ref(*this), p, sig_p, sig_user), true, local));
     
         if( local ) {
             // store it locally so it will be automatically refreshed with the rest
@@ -548,7 +549,7 @@ void node_impl::putDataSigned(std::string const &username, std::string const &re
 
 void node_impl::getData(std::string const &username, std::string const &resource, bool multi,
 			boost::function<void(entry::list_type const&)> fdata,
-			boost::function<void(bool, bool)> fdone)
+			boost::function<void(bool, bool)> fdone, bool local)
 {
 #ifdef TORRENT_DHT_VERBOSE_LOGGING
 	TORRENT_LOG(node) << "getData [ username: " << info_hash << " res: " << resource << " ]" ;
@@ -557,7 +558,7 @@ void node_impl::getData(std::string const &username, std::string const &resource
 	// for info-hash id. callback is used to return data.
 	boost::intrusive_ptr<dht_get> ta(new dht_get(*this, username, resource, multi,
 		 fdata,
-		 boost::bind(&getDataDone_fun, _1, _2, _3, boost::ref(*this), fdone), false));
+		 boost::bind(&getDataDone_fun, _1, _2, _3, boost::ref(*this), fdone), false, local));
 	ta->start();
 }
 
@@ -698,7 +699,9 @@ bool node_impl::refresh_storage() {
                 boost::intrusive_ptr<dht_get> ta(new dht_get(*this, username, resource, multi,
                                                              boost::bind(&putData_confirm, _1, boost::ref(item)),
                                                              boost::bind(&putData_fun, _1, boost::ref(*this),
-                                                                         entryP, item.sig_p, item.sig_user), item.confirmed));
+                                                                         entryP, item.sig_p, item.sig_user),
+                                                             item.confirmed,
+                                                             item.local_add_time));
                 ta->start();
                 did_something = true;
             }

libtorrent/src/kademlia/rpc_manager.cpp

@@ -492,7 +492,7 @@ bool rpc_manager::invoke(entry& e, udp::endpoint target_addr
 		<< e["q"].string() << " -> " << target_addr;
 #endif
 
-	if (m_sock->send_packet(e, target_addr, 1))
+	if (m_sock->send_packet(e, target_addr, (o->m_dont_drop) ? 1 : 0)) // udp_socket::dont_drop = 1
 	{
 		m_transactions.push_back(o);
 #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS

libtorrent/src/session.cpp

@@ -332,6 +332,9 @@ namespace libtorrent
 #define TORRENT_ASYNC_CALL3(x, a1, a2, a3) \
 	m_impl->m_io_service.dispatch(boost::bind(&session_impl:: x, m_impl.get(), a1, a2, a3))
 
+#define TORRENT_ASYNC_CALL4(x, a1, a2, a3, a4) \
+	m_impl->m_io_service.dispatch(boost::bind(&session_impl:: x, m_impl.get(), a1, a2, a3, a4))
+
 #define TORRENT_ASYNC_CALL6(x, a1, a2, a3, a4, a5, a6) \
 	m_impl->m_io_service.dispatch(boost::bind(&session_impl:: x, m_impl.get(), a1, a2, a3, a4, a5, a6))
 
@@ -862,10 +865,10 @@ namespace libtorrent
 #endif
 	}
 
-	void session::dht_getData(std::string const &username, std::string const &resource, bool multi)
+	void session::dht_getData(std::string const &username, std::string const &resource, bool multi, bool local)
 	{
 #ifndef TORRENT_DISABLE_DHT
-		TORRENT_ASYNC_CALL3(dht_getData, username, resource, multi);
+		TORRENT_ASYNC_CALL4(dht_getData, username, resource, multi, local);
 #endif
 	}
 

libtorrent/src/session_impl.cpp

@@ -5814,11 +5814,12 @@ retry:
 	    }
 	}
 
-	void session_impl::dht_getData(std::string const &username, std::string const &resource, bool multi)
+	void session_impl::dht_getData(std::string const &username, std::string const &resource, bool multi, bool local)
 	{
 	    if (m_dht) m_dht->getData(username, resource, multi,
 				      boost::bind( post_dht_getData, this, _1),
-				      boost::bind( getDataDone_fun, this, username, resource, multi, _1, _2));
+				      boost::bind( getDataDone_fun, this, username, resource, multi, _1, _2),
+				      local);
     }
 
 	entry session_impl::dht_getLocalData() const

src/clientversion.h

@@ -8,7 +8,7 @@
 // These need to be macros, as version.cpp's and bitcoin-qt.rc's voodoo requires it
 #define CLIENT_VERSION_MAJOR       0
 #define CLIENT_VERSION_MINOR       9
-#define CLIENT_VERSION_REVISION    27
+#define CLIENT_VERSION_REVISION    28
 #define CLIENT_VERSION_BUILD       0
 
 // Set to true for release, false for prerelease or test build

src/dhtproxy.cpp

@@ -228,7 +228,7 @@ namespace DhtProxy
             sha1_hash ih = dhtTargetHash(username, resource, multi ? "m" : "s");
             if( !req.stopReq ) {
                 dhtgetPeerReqAdd(ih, pfrom);
-                dhtGetData(username, resource, multi);
+                dhtGetData(username, resource, multi, false);
             } else {
                 dhtgetPeerReqRemove(ih, pfrom);
             }

src/twister.cpp

@@ -381,6 +381,9 @@ void ThreadWaitExtIP()
     settings.active_limit         = 25;
     settings.unchoke_slots_limit  = 20;
     settings.auto_manage_interval = 30;
+    // dht upload rate limit (enforced only for non-locally generated requests)
+    // limits: DHT replies, refreshes of stored items, checking for status/tracker and proxy server.
+    settings.dht_upload_rate_limit = 16000;
     ses->set_settings(settings);
 
     printf("libtorrent + dht started\n");
@@ -785,7 +788,7 @@ void ThreadSessionAlerts()
                                                    t->string().c_str());
 #endif
                                             neighborCheck[ih] = false;
-                                            dhtGetData(n->string(), r->string(), t->string() == "m");
+                                            dhtGetData(n->string(), r->string(), t->string() == "m", false);
                                         } else if( neighborCheck[ih] ) {
                                             sha1_hash ihStatus = dhtTargetHash(n->string(), "status", "s");
 
@@ -796,7 +799,7 @@ void ThreadSessionAlerts()
                                                         n->string().c_str(), "status", "s");
 #endif
                                                 statusCheck[ihStatus] = GetTime();
-                                                dhtGetData(n->string(), "status", false);
+                                                dhtGetData(n->string(), "status", false, false);
                                             }
                                         }
                                     }
@@ -832,7 +835,7 @@ void ThreadSessionAlerts()
 #endif
                             sha1_hash ihStatus = dhtTargetHash(dd->m_username, "status", "s");
                             statusCheck[ihStatus] = GetTime();
-                            dhtGetData(dd->m_username, "status", false);
+                            dhtGetData(dd->m_username, "status", false, false);
                         }
                     }
                     if( statusCheck.count(ih) ) {
@@ -1546,7 +1549,7 @@ entry formatSpamPost(const string &msg, const string &username, uint64_t utcTime
 }
 
 
-void dhtGetData(std::string const &username, std::string const &resource, bool multi)
+void dhtGetData(std::string const &username, std::string const &resource, bool multi, bool local)
 {
     if( DhtProxy::fEnabled ) {
         printf("dhtGetData: not allowed - using proxy (bug!)\n");
@@ -1557,7 +1560,7 @@ void dhtGetData(std::string const &username, std::string const &resource, bool m
         printf("dhtGetData: libtorrent session not ready\n");
         return;
     }
-    ses->dht_getData(username,resource,multi);
+    ses->dht_getData(username,resource,multi,local);
 }
 
 void dhtPutData(std::string const &username, std::string const &resource, bool multi,
@@ -1685,7 +1688,7 @@ Value dhtget(const Array& params, bool fHelp)
     vector<CNode*> dhtProxyNodes;
     if( !DhtProxy::fEnabled ) {
         dhtgetMapAdd(ih, &am);
-        dhtGetData(strUsername, strResource, multi);
+        dhtGetData(strUsername, strResource, multi, true);
     } else {
         DhtProxy::dhtgetMapAdd(ih, &am);
         dhtProxyNodes = DhtProxy::dhtgetStartRequest(strUsername, strResource, multi);

src/twister.h

@@ -47,7 +47,7 @@ int getDhtNodes(boost::int64_t *dht_global_nodes = NULL);
 void updateSeenHashtags(std::string &message, int64_t msgTime);
 
 // interface to dht api of the libtorrent current session
-void dhtGetData(std::string const &username, std::string const &resource, bool multi);
+void dhtGetData(std::string const &username, std::string const &resource, bool multi, bool local);
 void dhtPutData(std::string const &username, std::string const &resource, bool multi,
                 libtorrent::entry const &value, std::string const &sig_user,
                 boost::int64_t timeutc, int seq);