implement new peek / hashcash extension in our libtorrent fork

discussion here: https://groups.google.com/d/msg/twister-dev/oDKUr9oOBHg/6rzqqKoUCQAJ

libtorrent/include/libtorrent/add_torrent_params.hpp

@@ -94,6 +94,7 @@ namespace libtorrent
 			, max_connections(-1)
 			, upload_limit(-1)
 			, download_limit(-1)
+			, peek_single_piece(-1)
 		{
 		}
 
@@ -321,6 +322,9 @@ namespace libtorrent
 		int max_connections;
 		int upload_limit;
 		int download_limit;
+
+		// PEEK single piece with hashcash
+		int peek_single_piece;
 	};
 }
 

libtorrent/include/libtorrent/aux_/session_impl.hpp

@@ -105,6 +105,9 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <mach/mach_host.h>
 #endif
 
+#define HASHCASH_MIN_NBITS 16  // 16 bits ~ 10 ms @ i7 3.50GHz
+#define HASHCASH_MAX_NBITS 31
+
 namespace libtorrent
 {
 
@@ -1143,6 +1146,10 @@ namespace libtorrent
 			// the number of torrents that have apply_ip_filter
 			// set to false. This is typically 0
 			int m_non_filtered_torrents;
+			
+			// hashcash PEEK
+			int m_hashcash_nbits;
+			int m_hashcash_reqs;
 
 #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING
 			boost::shared_ptr<logger> create_log(std::string const& name

libtorrent/include/libtorrent/bt_peer_connection.hpp

@@ -132,6 +132,9 @@ namespace libtorrent
 			msg_cancel,
 			// DHT extension
 			msg_dht_port,
+			// hashcash PEEK extension
+			msg_hashcash_nbits,
+			msg_hashcash_nonce,
 			// FAST extension
 			msg_suggest_piece = 0xd,
 			msg_have_all,
@@ -198,6 +201,10 @@ namespace libtorrent
 		// DHT extension
 		void on_dht_port(int received);
 
+		// PEEK extension
+		void on_hashcash_nbits(int received);
+		void on_hashcash_nonce(int received);
+
 		// FAST extension
 		void on_suggest_piece(int received);
 		void on_have_all(int received);
@@ -239,6 +246,10 @@ namespace libtorrent
 		// DHT extension
 		void write_dht_port(int listen_port);
 
+		// PEEK extension
+		void write_hashcash_nbits(int nbits);
+		void write_hashcash_nonce(const char *nonce, int size);
+
 		// FAST extension
 		void write_have_all();
 		void write_have_none();
@@ -399,6 +410,7 @@ private:
 		bool m_supports_extensions:1;
 		bool m_supports_dht_port:1;
 		bool m_supports_fast:1;
+		bool m_supports_peek:1;
 
 #ifndef TORRENT_DISABLE_ENCRYPTION
 		// this is set to true after the encryption method has been
@@ -437,6 +449,7 @@ private:
 		// this is set to true when the client's
 		// bitfield is sent to this peer
 		bool m_sent_bitfield;
+		bool m_sent_hashcash_nonce;
 #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS
 
 		bool m_in_constructor;

libtorrent/include/libtorrent/peer_connection.hpp

@@ -487,13 +487,18 @@ namespace libtorrent
 		void incoming_cancel(peer_request const& r);
 
 		void incoming_dht_port(int listen_port);
-		
+
+		void incoming_hashcash_nbits(int nbits);
+		void incoming_hashcash_nonce(const char *ptr, int size);
+
 		void incoming_reject_request(peer_request const& r);
 		void incoming_have_all();
 		void incoming_have_none();
 		void incoming_allowed_fast(int index);
 		void incoming_suggest(int index);
 
+		void recheck_request_blocks();
+
 		void set_has_metadata(bool m) { m_has_metadata = m; }
 		bool has_metadata() const { return m_has_metadata; }
 
@@ -504,9 +509,12 @@ namespace libtorrent
 		void send_interested();
 		void send_not_interested();
 		void send_suggest(int piece);
+		void send_hashcash_nonce(int piece);
 
 		void snub_peer();
 
+		void sent_hashcash_nbits(int nbits) { m_sent_hashcash_nbits = nbits; }
+
 		bool can_request_time_critical() const;
 
 		void make_time_critical(piece_block const& block);
@@ -1069,6 +1077,17 @@ namespace libtorrent
 		// at the remote end.
 		boost::uint8_t m_desired_queue_size;
 
+		// the number of bits the remote end requires
+		// for PEEK hashcash
+		int m_hashcash_nbits;
+
+		// the number of bits we've sent to remote
+		int m_sent_hashcash_nbits;
+
+		// the nonce received from the remote end
+		// for PEEK hashcash
+		std::vector<char> m_hashcash_nonce;
+
 		// if this is true, the disconnection
 		// timestamp is not updated when the connection
 		// is closed. This means the time until we can
@@ -1175,6 +1194,9 @@ namespace libtorrent
 		// set to true when we've sent the first round of suggests
 		bool m_sent_suggests:1;
 
+		// set to true when we've sent the hashcash nonce for peek piece
+		bool m_sent_hashcash_nonce:1;
+
 		// set to true while we're trying to holepunch
 		bool m_holepunch_mode:1;
 

libtorrent/include/libtorrent/torrent.hpp

@@ -1406,6 +1406,9 @@ namespace libtorrent
 		// set to false until we've loaded resume data
 		bool m_resume_data_loaded;
 #endif
+	public:
+		// PEEK extension with hashcash
+		int m_peek_single_piece;
 	};
 }
 

libtorrent/src/bt_peer_connection.cpp

@@ -82,7 +82,9 @@ namespace libtorrent
 		&bt_peer_connection::on_piece,
 		&bt_peer_connection::on_cancel,
 		&bt_peer_connection::on_dht_port,
-		0, 0, 0,
+		&bt_peer_connection::on_hashcash_nbits,
+		&bt_peer_connection::on_hashcash_nonce,
+		0,
 		// FAST extension messages
 		&bt_peer_connection::on_suggest_piece,
 		&bt_peer_connection::on_have_all,
@@ -113,12 +115,14 @@ namespace libtorrent
 #endif
 		, m_supports_dht_port(false)
 		, m_supports_fast(false)
+		, m_supports_peek(false)
 #ifndef TORRENT_DISABLE_ENCRYPTION
 		, m_encrypted(false)
 		, m_rc4_encrypted(false)
 		, m_sync_bytes_read(0)
 #endif
 		, m_sent_bitfield(false)
+		, m_sent_hashcash_nonce(false)
 #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS
 		, m_in_constructor(true)
 		, m_sent_handshake(false)
@@ -233,6 +237,9 @@ namespace libtorrent
 		if (m_supports_dht_port && m_ses.m_dht)
 			write_dht_port(m_ses.m_external_udp_port);
 #endif
+		if (m_supports_peek) {
+			write_hashcash_nbits(m_ses.m_hashcash_nbits);
+		}
 	}
 
 	void bt_peer_connection::write_dht_port(int listen_port)
@@ -250,6 +257,45 @@ namespace libtorrent
 		send_buffer(msg, sizeof(msg));
 	}
 
+	void bt_peer_connection::write_hashcash_nbits(int nbits)
+	{
+		INVARIANT_CHECK;
+
+		TORRENT_ASSERT(m_sent_handshake && m_sent_bitfield);
+
+#ifdef TORRENT_VERBOSE_LOGGING
+		peer_log("==> HASHCASH_NBITS [ %d ]", nbits);
+#endif
+		char msg[] = {0,0,0,3, msg_hashcash_nbits, 0, 0};
+		char* ptr = msg + 5;
+		detail::write_uint16(nbits, ptr);
+		send_buffer(msg, sizeof(msg));
+		sent_hashcash_nbits(nbits);
+	}
+
+	void bt_peer_connection::write_hashcash_nonce(const char *nonce, int size)
+	{
+		INVARIANT_CHECK;
+
+		TORRENT_ASSERT(m_sent_handshake && m_sent_bitfield);
+
+#ifdef TORRENT_VERBOSE_LOGGING
+		peer_log("==> HASHCASH_NONCE [ size: %d ]", size);
+#endif
+		const int packet_size = size + 5;
+
+		char* msg = TORRENT_ALLOCA(char, packet_size);
+		if (msg == 0) return; // out of memory
+		unsigned char* ptr = (unsigned char*)msg;
+
+		detail::write_int32(packet_size - 4, ptr);
+		detail::write_uint8(msg_hashcash_nonce, ptr);
+		memcpy(ptr, nonce, size);
+
+		send_buffer(msg, packet_size);
+	}
+
+
 	void bt_peer_connection::write_have_all()
 	{
 		INVARIANT_CHECK;
@@ -730,6 +776,9 @@ namespace libtorrent
 		// we support FAST extension
 		*(ptr + 7) |= 0x04;
 
+		// we support PEEK/hashcash extension
+		*(ptr + 7) |= 0x10;
+
 #ifdef TORRENT_VERBOSE_LOGGING	
 		std::string bitmask;
 		for (int k = 0; k < 8; ++k)
@@ -1272,6 +1321,96 @@ namespace libtorrent
 		}
 	}
 
+	void bt_peer_connection::on_hashcash_nbits(int received)
+	{
+		INVARIANT_CHECK;
+
+		TORRENT_ASSERT(received > 0);
+		m_statistics.received_bytes(0, received);
+		if (packet_size() != 3)
+		{
+			disconnect(errors::invalid_message, 2);
+			return;
+		}
+		if (!packet_finished()) return;
+
+		buffer::const_interval recv_buffer = receive_buffer();
+
+		const char* ptr = recv_buffer.begin + 1;
+		int nbits = detail::read_uint16(ptr);
+		incoming_hashcash_nbits(nbits);
+		
+		if (!m_sent_hashcash_nonce) {
+			boost::shared_ptr<torrent> t = associated_torrent().lock();
+			if (!t) return;
+			if (t->m_peek_single_piece >= 0) {
+#ifdef TORRENT_VERBOSE_LOGGING
+				peer_log("+++ computing hashcash nonce");
+#endif
+				hasher h;
+				sha1_hash const& info_hash = t->info_hash();
+				sha1_hash max_hash = sha1_hash::max();
+				max_hash >>= nbits;
+				char tmp[8];
+				const ptime start_hashcase = time_now_hires();
+
+				// TODO: move to another thread (this is not the
+				//       proper place for cpu-burning code)
+				// TODO2: support longer nonces (>32 bits).
+				for(int nonce = 0; nonce < 0x7fff0000l; nonce++) {
+					h.reset();
+					h.update((const char*)info_hash.begin(), 20);
+					char* ptr = tmp;
+					detail::write_int32(t->m_peek_single_piece, ptr);
+					detail::write_int32(nonce, ptr);
+					h.update(tmp, sizeof(tmp));
+
+					sha1_hash finalhash = h.final();
+					if (finalhash < max_hash) {
+						write_hashcash_nonce(tmp+4,sizeof(tmp)-4);
+						m_sent_hashcash_nonce = true;
+						recheck_request_blocks();
+						break;
+					}
+					if( (nonce % 10000) == 0 &&
+						total_milliseconds(time_now_hires() - start_hashcase) > 1000) {
+#ifdef TORRENT_VERBOSE_LOGGING
+						peer_log("+++ hashcash timeout!");
+#endif
+						break;
+					}
+				}
+#ifdef TORRENT_VERBOSE_LOGGING
+				peer_log("+++ computing hashcash done");
+#endif
+			}
+		}
+	}
+
+	void bt_peer_connection::on_hashcash_nonce(int received)
+	{
+		INVARIANT_CHECK;
+
+		TORRENT_ASSERT(received > 0);
+		m_statistics.received_bytes(0, received);
+
+		if (packet_size() > 10)
+		{
+			disconnect(errors::invalid_message, 2);
+			return;
+		}
+
+		if (!packet_finished()) return;
+
+		buffer::const_interval recv_buffer = receive_buffer();
+
+		const char* ptr = recv_buffer.begin + 1;
+		int size = packet_size()-1;
+		incoming_hashcash_nonce(ptr, size);
+	}
+
+
+
 	void bt_peer_connection::on_suggest_piece(int received)
 	{
 		INVARIANT_CHECK;
@@ -2981,6 +3120,7 @@ namespace libtorrent
 				, extensions.c_str()
 				, (recv_buffer[7] & 0x01) ? "DHT " : ""
 				, (recv_buffer[7] & 0x04) ? "FAST " : ""
+				, (recv_buffer[7] & 0x10) ? "PEEK " : ""
 				, (recv_buffer[5] & 0x10) ? "extension " : "");
 #endif
 
@@ -2995,6 +3135,9 @@ namespace libtorrent
 			if (recv_buffer[7] & 0x04)
 				m_supports_fast = true;
 
+			if (recv_buffer[7] & 0x10)
+				m_supports_peek = true;
+
 			// ok, now we have got enough of the handshake. Is this connection
 			// attached to a torrent?
 			if (!t)
@@ -3183,6 +3326,9 @@ namespace libtorrent
 				if (m_supports_dht_port && m_ses.m_dht)
 					write_dht_port(m_ses.m_external_udp_port);
 #endif
+				if (m_supports_peek) {
+					write_hashcash_nbits(m_ses.m_hashcash_nbits);
+				}
 			}
 
 			TORRENT_ASSERT(!packet_finished());

libtorrent/src/peer_connection.cpp

@@ -36,6 +36,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <boost/limits.hpp>
 #include <boost/bind.hpp>
 #include <boost/cstdint.hpp>
+#include <boost/lexical_cast.hpp>
 #include <stdarg.h> // for va_start, va_end
 
 #include "libtorrent/peer_connection.hpp"
@@ -166,6 +167,8 @@ namespace libtorrent
 		, m_rtt(0)
 		, m_prefer_whole_pieces(0)
 		, m_desired_queue_size(8)
+		, m_hashcash_nbits(0)
+		, m_sent_hashcash_nbits(HASHCASH_MAX_NBITS)
 		, m_fast_reconnect(false)
 		, m_outgoing(outgoing)
 		, m_received_listen_port(false)
@@ -504,6 +507,10 @@ namespace libtorrent
 		if (!t->ready_for_connections()) return;
 
 		bool interested = false;
+		if (t->m_peek_single_piece >= 0) {
+			interested = t->m_peek_single_piece < m_have_piece.size() &&
+			             m_have_piece[t->m_peek_single_piece];
+		} else
 		if (!t->is_upload_only())
 		{
 			piece_picker const& p = t->picker();
@@ -1400,6 +1407,20 @@ namespace libtorrent
 		}
 	}
 
+	void peer_connection::recheck_request_blocks()
+	{
+		INVARIANT_CHECK;
+
+		boost::shared_ptr<torrent> t = m_torrent.lock();
+		TORRENT_ASSERT(t);
+
+		if (is_disconnecting()) return;
+
+		request_a_block(*t, *this);
+		send_block_requests();
+	}
+
+
 	// -----------------------------
 	// -------- INTERESTED ---------
 	// -----------------------------
@@ -2031,10 +2052,40 @@ namespace libtorrent
 			&& m_peer_interested
 			&& r.length <= t->block_size())
 		{
+			bool hashcash_valid = false;
+
+			if (m_choked && m_hashcash_nonce.size() ) {
+				hasher h;
+				sha1_hash const& info_hash = t->info_hash();
+				sha1_hash max_hash = sha1_hash::max();
+				max_hash >>= m_sent_hashcash_nbits;
+				char tmp[4];
+
+				h.reset();
+				h.update((const char*)info_hash.begin(), 20);
+				char* ptr = tmp;
+				detail::write_int32(r.piece, ptr);
+				h.update(tmp, sizeof(tmp));
+				h.update(m_hashcash_nonce.data(), (int)m_hashcash_nonce.size());
+
+				sha1_hash finalhash = h.final();
+				if (finalhash < max_hash) {
+					hashcash_valid = true;
+					m_ses.m_hashcash_reqs++;
+#ifdef TORRENT_VERBOSE_LOGGING
+					peer_log("+++ hashcash authorized");
+#endif
+				} else {
+#ifdef TORRENT_VERBOSE_LOGGING
+					peer_log("+++ hashcash failed");
+#endif
+				}
+			}
+
 			// if we have choked the client
 			// ignore the request
 			if (m_choked && std::find(m_accept_fast.begin(), m_accept_fast.end()
-				, r.piece) == m_accept_fast.end())
+				, r.piece) == m_accept_fast.end() && !hashcash_valid)
 			{
 #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING
 				peer_log("*** REJECTING REQUEST [ peer choked and piece not in allowed fast set ]");
@@ -2323,6 +2374,49 @@ namespace libtorrent
 			return;
 		}
 
+		if (t->m_peek_single_piece >= 0) {
+			std::string errmsg;
+			int hash_ok;
+			hash_ok = acceptSignedPost((char const*)data.get(), p.length,
+										t->torrent_file().name(), p.piece, errmsg, NULL);
+
+			if( hash_ok && p.piece == t->m_peek_single_piece ) {
+				lazy_entry v;
+				int pos;
+				libtorrent::error_code ec;
+				if (lazy_bdecode(data.get(), data.get() + p.length,
+				                 v, ec, &pos) == 0
+				    && v.type() == lazy_entry::dict_t ) {
+
+					// fake a dht encapsulation to reuse dht_reply_data_alert mechanism
+					entry target;
+					target["n"] = t->torrent_file().name();
+					target["r"] = "post" + boost::lexical_cast<std::string>(p.piece);
+					target["t"] = "s";
+					entry p;
+					p["target"] = target;
+					p["v"] = v;
+					entry e;
+					e["p"] = p;
+					e["sig_p"] = "peek";
+
+					entry::list_type lst;
+					lst.push_back(e);
+					if( t->alerts().should_post<dht_reply_data_alert>() ) {
+#ifdef TORRENT_VERBOSE_LOGGING
+						peer_log("*** post alert of fake dhtget reply (%s,%s,%s)"
+								, target["n"].string().c_str()
+								, target["r"].string().c_str()
+								, target["t"].string().c_str() );
+#endif
+						t->alerts().post_alert(dht_reply_data_alert(lst));
+					}
+				}
+			}
+			disconnect(errors::no_error);
+			return;
+		}
+
 		// if we're already seeding, don't bother,
 		// just ignore it
 		if (t->is_seed())
@@ -2665,6 +2759,36 @@ namespace libtorrent
 #endif
 	}
 
+
+	void peer_connection::incoming_hashcash_nbits(int nbits)
+	{
+		INVARIANT_CHECK;
+
+#ifdef TORRENT_VERBOSE_LOGGING
+		peer_log("<== HASHCASH BITS [ %d ]", nbits);
+#endif
+		m_hashcash_nbits = nbits;
+	}
+
+	void peer_connection::incoming_hashcash_nonce(const char *ptr, int size)
+	{
+		INVARIANT_CHECK;
+
+#ifdef TORRENT_VERBOSE_LOGGING
+		peer_log("<== HASHCASH NONCE [ size: %d ]", size);
+#endif
+		if( m_hashcash_nonce.size() ) {
+			// just one hashcash nonce per connection allowed
+			disconnect(errors::invalid_message);
+			return;
+		}
+
+		m_hashcash_nonce.clear();
+		while(size--) {
+			m_hashcash_nonce.push_back(*ptr++);
+		}
+	}
+
 	// -----------------------------
 	// --------- HAVE ALL ----------
 	// -----------------------------
@@ -2827,6 +2951,11 @@ namespace libtorrent
 				return;
 		}
 
+		// if peek mode and fast piece is not what we want, return
+		if (t->m_peek_single_piece >=0
+			&& index != t->m_peek_single_piece)
+			return;
+
 		// if we don't have the metadata, we'll verify
 		// this piece index later
 		m_allowed_fast.push_back(index);

libtorrent/src/policy.cpp

@@ -283,6 +283,14 @@ namespace libtorrent
 		bitfield const* bits = &c.get_bitfield();
 		bitfield fast_mask;
 		
+		if (t.m_peek_single_piece >= 0) {
+			// build a bitmask containing just the peek piece
+			fast_mask.resize(c.get_bitfield().size(), false);
+			if (t.m_peek_single_piece < c.get_bitfield().size()
+			    && (*bits)[t.m_peek_single_piece])
+				fast_mask.set_bit(t.m_peek_single_piece);
+			bits = &fast_mask;
+		} else
 		if (c.has_peer_choked())
 		{
 			// if we are choked we can only pick pieces from the

libtorrent/src/session_impl.cpp

@@ -681,6 +681,8 @@ namespace aux {
 		, m_need_auto_manage(false)
 #if (defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS) && defined BOOST_HAS_PTHREADS
 		, m_network_thread(0)
+		, m_hashcash_nbits(HASHCASH_MIN_NBITS)
+		, m_hashcash_reqs(0)
 #endif
 	{
 #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS
@@ -3563,6 +3565,17 @@ retry:
 			}
 		}
 
+		// --------------------------------------------------------------
+		// adjust hashcash: too much reqs last second => increase difficulty
+		// --------------------------------------------------------------
+		if (m_hashcash_reqs > 50) {
+			m_hashcash_nbits = std::min(m_hashcash_nbits+1, HASHCASH_MAX_NBITS);
+		}
+		if (!m_hashcash_reqs && (random() % 100) == 0 ) {
+			m_hashcash_nbits = std::max(m_hashcash_nbits-1, HASHCASH_MIN_NBITS);
+		}
+		m_hashcash_reqs = 0;
+
 		while (m_tick_residual >= 1000) m_tick_residual -= 1000;
 //		m_peer_pool.release_memory();
 	}

libtorrent/src/torrent.cpp

@@ -432,6 +432,7 @@ namespace libtorrent
 		, m_in_state_updates(false)
 		, m_is_active_download(false)
 		, m_is_active_finished(false)
+		, m_peek_single_piece(-1)
 	{
         if (!p.name.empty()) m_name.reset(new std::string(p.name));
 
@@ -508,6 +509,7 @@ namespace libtorrent
 		set_max_connections(p.max_connections, false);
 		set_upload_limit(p.upload_limit, false);
 		set_download_limit(p.download_limit, false);
+		m_peek_single_piece = p.peek_single_piece;
 
 		if (!m_name && !m_url.empty()) m_name.reset(new std::string(m_url));