From 1c95fa8cd9861496abe8c42c8187a90b13ddf0b8 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Thu, 10 Dec 2015 21:59:18 -0200 Subject: [PATCH] implement new peek / hashcash extension in our libtorrent fork discussion here: https://groups.google.com/d/msg/twister-dev/oDKUr9oOBHg/6rzqqKoUCQAJ --- .../include/libtorrent/add_torrent_params.hpp | 4 + .../include/libtorrent/aux_/session_impl.hpp | 7 + .../include/libtorrent/bt_peer_connection.hpp | 13 ++ .../include/libtorrent/peer_connection.hpp | 24 ++- libtorrent/include/libtorrent/torrent.hpp | 3 + libtorrent/src/bt_peer_connection.cpp | 148 +++++++++++++++++- libtorrent/src/peer_connection.cpp | 131 +++++++++++++++- libtorrent/src/policy.cpp | 8 + libtorrent/src/session_impl.cpp | 13 ++ libtorrent/src/torrent.cpp | 2 + 10 files changed, 350 insertions(+), 3 deletions(-) diff --git a/libtorrent/include/libtorrent/add_torrent_params.hpp b/libtorrent/include/libtorrent/add_torrent_params.hpp index 1a4d938b..6d166d9b 100644 --- a/libtorrent/include/libtorrent/add_torrent_params.hpp +++ b/libtorrent/include/libtorrent/add_torrent_params.hpp @@ -94,6 +94,7 @@ namespace libtorrent , max_connections(-1) , upload_limit(-1) , download_limit(-1) + , peek_single_piece(-1) { } @@ -321,6 +322,9 @@ namespace libtorrent int max_connections; int upload_limit; int download_limit; + + // PEEK single piece with hashcash + int peek_single_piece; }; } diff --git a/libtorrent/include/libtorrent/aux_/session_impl.hpp b/libtorrent/include/libtorrent/aux_/session_impl.hpp index 6a4aa380..8cfb7885 100644 --- a/libtorrent/include/libtorrent/aux_/session_impl.hpp +++ b/libtorrent/include/libtorrent/aux_/session_impl.hpp @@ -105,6 +105,9 @@ POSSIBILITY OF SUCH DAMAGE. #include #endif +#define HASHCASH_MIN_NBITS 16 // 16 bits ~ 10 ms @ i7 3.50GHz +#define HASHCASH_MAX_NBITS 31 + namespace libtorrent { @@ -1143,6 +1146,10 @@ namespace libtorrent // the number of torrents that have apply_ip_filter // set to false. This is typically 0 int m_non_filtered_torrents; + + // hashcash PEEK + int m_hashcash_nbits; + int m_hashcash_reqs; #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING boost::shared_ptr create_log(std::string const& name diff --git a/libtorrent/include/libtorrent/bt_peer_connection.hpp b/libtorrent/include/libtorrent/bt_peer_connection.hpp index 14e1e446..7b18ce4e 100644 --- a/libtorrent/include/libtorrent/bt_peer_connection.hpp +++ b/libtorrent/include/libtorrent/bt_peer_connection.hpp @@ -132,6 +132,9 @@ namespace libtorrent msg_cancel, // DHT extension msg_dht_port, + // hashcash PEEK extension + msg_hashcash_nbits, + msg_hashcash_nonce, // FAST extension msg_suggest_piece = 0xd, msg_have_all, @@ -198,6 +201,10 @@ namespace libtorrent // DHT extension void on_dht_port(int received); + // PEEK extension + void on_hashcash_nbits(int received); + void on_hashcash_nonce(int received); + // FAST extension void on_suggest_piece(int received); void on_have_all(int received); @@ -239,6 +246,10 @@ namespace libtorrent // DHT extension void write_dht_port(int listen_port); + // PEEK extension + void write_hashcash_nbits(int nbits); + void write_hashcash_nonce(const char *nonce, int size); + // FAST extension void write_have_all(); void write_have_none(); @@ -399,6 +410,7 @@ private: bool m_supports_extensions:1; bool m_supports_dht_port:1; bool m_supports_fast:1; + bool m_supports_peek:1; #ifndef TORRENT_DISABLE_ENCRYPTION // this is set to true after the encryption method has been @@ -437,6 +449,7 @@ private: // this is set to true when the client's // bitfield is sent to this peer bool m_sent_bitfield; + bool m_sent_hashcash_nonce; #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS bool m_in_constructor; diff --git a/libtorrent/include/libtorrent/peer_connection.hpp b/libtorrent/include/libtorrent/peer_connection.hpp index c2cf67e8..6dd5e07a 100644 --- a/libtorrent/include/libtorrent/peer_connection.hpp +++ b/libtorrent/include/libtorrent/peer_connection.hpp @@ -487,13 +487,18 @@ namespace libtorrent void incoming_cancel(peer_request const& r); void incoming_dht_port(int listen_port); - + + void incoming_hashcash_nbits(int nbits); + void incoming_hashcash_nonce(const char *ptr, int size); + void incoming_reject_request(peer_request const& r); void incoming_have_all(); void incoming_have_none(); void incoming_allowed_fast(int index); void incoming_suggest(int index); + void recheck_request_blocks(); + void set_has_metadata(bool m) { m_has_metadata = m; } bool has_metadata() const { return m_has_metadata; } @@ -504,9 +509,12 @@ namespace libtorrent void send_interested(); void send_not_interested(); void send_suggest(int piece); + void send_hashcash_nonce(int piece); void snub_peer(); + void sent_hashcash_nbits(int nbits) { m_sent_hashcash_nbits = nbits; } + bool can_request_time_critical() const; void make_time_critical(piece_block const& block); @@ -1069,6 +1077,17 @@ namespace libtorrent // at the remote end. boost::uint8_t m_desired_queue_size; + // the number of bits the remote end requires + // for PEEK hashcash + int m_hashcash_nbits; + + // the number of bits we've sent to remote + int m_sent_hashcash_nbits; + + // the nonce received from the remote end + // for PEEK hashcash + std::vector m_hashcash_nonce; + // if this is true, the disconnection // timestamp is not updated when the connection // is closed. This means the time until we can @@ -1175,6 +1194,9 @@ namespace libtorrent // set to true when we've sent the first round of suggests bool m_sent_suggests:1; + // set to true when we've sent the hashcash nonce for peek piece + bool m_sent_hashcash_nonce:1; + // set to true while we're trying to holepunch bool m_holepunch_mode:1; diff --git a/libtorrent/include/libtorrent/torrent.hpp b/libtorrent/include/libtorrent/torrent.hpp index bf002d22..d9e3a57e 100644 --- a/libtorrent/include/libtorrent/torrent.hpp +++ b/libtorrent/include/libtorrent/torrent.hpp @@ -1406,6 +1406,9 @@ namespace libtorrent // set to false until we've loaded resume data bool m_resume_data_loaded; #endif + public: + // PEEK extension with hashcash + int m_peek_single_piece; }; } diff --git a/libtorrent/src/bt_peer_connection.cpp b/libtorrent/src/bt_peer_connection.cpp index 083a3c23..36946755 100644 --- a/libtorrent/src/bt_peer_connection.cpp +++ b/libtorrent/src/bt_peer_connection.cpp @@ -82,7 +82,9 @@ namespace libtorrent &bt_peer_connection::on_piece, &bt_peer_connection::on_cancel, &bt_peer_connection::on_dht_port, - 0, 0, 0, + &bt_peer_connection::on_hashcash_nbits, + &bt_peer_connection::on_hashcash_nonce, + 0, // FAST extension messages &bt_peer_connection::on_suggest_piece, &bt_peer_connection::on_have_all, @@ -113,12 +115,14 @@ namespace libtorrent #endif , m_supports_dht_port(false) , m_supports_fast(false) + , m_supports_peek(false) #ifndef TORRENT_DISABLE_ENCRYPTION , m_encrypted(false) , m_rc4_encrypted(false) , m_sync_bytes_read(0) #endif , m_sent_bitfield(false) + , m_sent_hashcash_nonce(false) #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS , m_in_constructor(true) , m_sent_handshake(false) @@ -233,6 +237,9 @@ namespace libtorrent if (m_supports_dht_port && m_ses.m_dht) write_dht_port(m_ses.m_external_udp_port); #endif + if (m_supports_peek) { + write_hashcash_nbits(m_ses.m_hashcash_nbits); + } } void bt_peer_connection::write_dht_port(int listen_port) @@ -250,6 +257,45 @@ namespace libtorrent send_buffer(msg, sizeof(msg)); } + void bt_peer_connection::write_hashcash_nbits(int nbits) + { + INVARIANT_CHECK; + + TORRENT_ASSERT(m_sent_handshake && m_sent_bitfield); + +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("==> HASHCASH_NBITS [ %d ]", nbits); +#endif + char msg[] = {0,0,0,3, msg_hashcash_nbits, 0, 0}; + char* ptr = msg + 5; + detail::write_uint16(nbits, ptr); + send_buffer(msg, sizeof(msg)); + sent_hashcash_nbits(nbits); + } + + void bt_peer_connection::write_hashcash_nonce(const char *nonce, int size) + { + INVARIANT_CHECK; + + TORRENT_ASSERT(m_sent_handshake && m_sent_bitfield); + +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("==> HASHCASH_NONCE [ size: %d ]", size); +#endif + const int packet_size = size + 5; + + char* msg = TORRENT_ALLOCA(char, packet_size); + if (msg == 0) return; // out of memory + unsigned char* ptr = (unsigned char*)msg; + + detail::write_int32(packet_size - 4, ptr); + detail::write_uint8(msg_hashcash_nonce, ptr); + memcpy(ptr, nonce, size); + + send_buffer(msg, packet_size); + } + + void bt_peer_connection::write_have_all() { INVARIANT_CHECK; @@ -730,6 +776,9 @@ namespace libtorrent // we support FAST extension *(ptr + 7) |= 0x04; + // we support PEEK/hashcash extension + *(ptr + 7) |= 0x10; + #ifdef TORRENT_VERBOSE_LOGGING std::string bitmask; for (int k = 0; k < 8; ++k) @@ -1272,6 +1321,96 @@ namespace libtorrent } } + void bt_peer_connection::on_hashcash_nbits(int received) + { + INVARIANT_CHECK; + + TORRENT_ASSERT(received > 0); + m_statistics.received_bytes(0, received); + if (packet_size() != 3) + { + disconnect(errors::invalid_message, 2); + return; + } + if (!packet_finished()) return; + + buffer::const_interval recv_buffer = receive_buffer(); + + const char* ptr = recv_buffer.begin + 1; + int nbits = detail::read_uint16(ptr); + incoming_hashcash_nbits(nbits); + + if (!m_sent_hashcash_nonce) { + boost::shared_ptr t = associated_torrent().lock(); + if (!t) return; + if (t->m_peek_single_piece >= 0) { +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("+++ computing hashcash nonce"); +#endif + hasher h; + sha1_hash const& info_hash = t->info_hash(); + sha1_hash max_hash = sha1_hash::max(); + max_hash >>= nbits; + char tmp[8]; + const ptime start_hashcase = time_now_hires(); + + // TODO: move to another thread (this is not the + // proper place for cpu-burning code) + // TODO2: support longer nonces (>32 bits). + for(int nonce = 0; nonce < 0x7fff0000l; nonce++) { + h.reset(); + h.update((const char*)info_hash.begin(), 20); + char* ptr = tmp; + detail::write_int32(t->m_peek_single_piece, ptr); + detail::write_int32(nonce, ptr); + h.update(tmp, sizeof(tmp)); + + sha1_hash finalhash = h.final(); + if (finalhash < max_hash) { + write_hashcash_nonce(tmp+4,sizeof(tmp)-4); + m_sent_hashcash_nonce = true; + recheck_request_blocks(); + break; + } + if( (nonce % 10000) == 0 && + total_milliseconds(time_now_hires() - start_hashcase) > 1000) { +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("+++ hashcash timeout!"); +#endif + break; + } + } +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("+++ computing hashcash done"); +#endif + } + } + } + + void bt_peer_connection::on_hashcash_nonce(int received) + { + INVARIANT_CHECK; + + TORRENT_ASSERT(received > 0); + m_statistics.received_bytes(0, received); + + if (packet_size() > 10) + { + disconnect(errors::invalid_message, 2); + return; + } + + if (!packet_finished()) return; + + buffer::const_interval recv_buffer = receive_buffer(); + + const char* ptr = recv_buffer.begin + 1; + int size = packet_size()-1; + incoming_hashcash_nonce(ptr, size); + } + + + void bt_peer_connection::on_suggest_piece(int received) { INVARIANT_CHECK; @@ -2981,6 +3120,7 @@ namespace libtorrent , extensions.c_str() , (recv_buffer[7] & 0x01) ? "DHT " : "" , (recv_buffer[7] & 0x04) ? "FAST " : "" + , (recv_buffer[7] & 0x10) ? "PEEK " : "" , (recv_buffer[5] & 0x10) ? "extension " : ""); #endif @@ -2995,6 +3135,9 @@ namespace libtorrent if (recv_buffer[7] & 0x04) m_supports_fast = true; + if (recv_buffer[7] & 0x10) + m_supports_peek = true; + // ok, now we have got enough of the handshake. Is this connection // attached to a torrent? if (!t) @@ -3183,6 +3326,9 @@ namespace libtorrent if (m_supports_dht_port && m_ses.m_dht) write_dht_port(m_ses.m_external_udp_port); #endif + if (m_supports_peek) { + write_hashcash_nbits(m_ses.m_hashcash_nbits); + } } TORRENT_ASSERT(!packet_finished()); diff --git a/libtorrent/src/peer_connection.cpp b/libtorrent/src/peer_connection.cpp index 76d8d1cf..1bcaa40f 100644 --- a/libtorrent/src/peer_connection.cpp +++ b/libtorrent/src/peer_connection.cpp @@ -36,6 +36,7 @@ POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include #include // for va_start, va_end #include "libtorrent/peer_connection.hpp" @@ -166,6 +167,8 @@ namespace libtorrent , m_rtt(0) , m_prefer_whole_pieces(0) , m_desired_queue_size(8) + , m_hashcash_nbits(0) + , m_sent_hashcash_nbits(HASHCASH_MAX_NBITS) , m_fast_reconnect(false) , m_outgoing(outgoing) , m_received_listen_port(false) @@ -504,6 +507,10 @@ namespace libtorrent if (!t->ready_for_connections()) return; bool interested = false; + if (t->m_peek_single_piece >= 0) { + interested = t->m_peek_single_piece < m_have_piece.size() && + m_have_piece[t->m_peek_single_piece]; + } else if (!t->is_upload_only()) { piece_picker const& p = t->picker(); @@ -1400,6 +1407,20 @@ namespace libtorrent } } + void peer_connection::recheck_request_blocks() + { + INVARIANT_CHECK; + + boost::shared_ptr t = m_torrent.lock(); + TORRENT_ASSERT(t); + + if (is_disconnecting()) return; + + request_a_block(*t, *this); + send_block_requests(); + } + + // ----------------------------- // -------- INTERESTED --------- // ----------------------------- @@ -2031,10 +2052,40 @@ namespace libtorrent && m_peer_interested && r.length <= t->block_size()) { + bool hashcash_valid = false; + + if (m_choked && m_hashcash_nonce.size() ) { + hasher h; + sha1_hash const& info_hash = t->info_hash(); + sha1_hash max_hash = sha1_hash::max(); + max_hash >>= m_sent_hashcash_nbits; + char tmp[4]; + + h.reset(); + h.update((const char*)info_hash.begin(), 20); + char* ptr = tmp; + detail::write_int32(r.piece, ptr); + h.update(tmp, sizeof(tmp)); + h.update(m_hashcash_nonce.data(), (int)m_hashcash_nonce.size()); + + sha1_hash finalhash = h.final(); + if (finalhash < max_hash) { + hashcash_valid = true; + m_ses.m_hashcash_reqs++; +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("+++ hashcash authorized"); +#endif + } else { +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("+++ hashcash failed"); +#endif + } + } + // if we have choked the client // ignore the request if (m_choked && std::find(m_accept_fast.begin(), m_accept_fast.end() - , r.piece) == m_accept_fast.end()) + , r.piece) == m_accept_fast.end() && !hashcash_valid) { #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING peer_log("*** REJECTING REQUEST [ peer choked and piece not in allowed fast set ]"); @@ -2323,6 +2374,49 @@ namespace libtorrent return; } + if (t->m_peek_single_piece >= 0) { + std::string errmsg; + int hash_ok; + hash_ok = acceptSignedPost((char const*)data.get(), p.length, + t->torrent_file().name(), p.piece, errmsg, NULL); + + if( hash_ok && p.piece == t->m_peek_single_piece ) { + lazy_entry v; + int pos; + libtorrent::error_code ec; + if (lazy_bdecode(data.get(), data.get() + p.length, + v, ec, &pos) == 0 + && v.type() == lazy_entry::dict_t ) { + + // fake a dht encapsulation to reuse dht_reply_data_alert mechanism + entry target; + target["n"] = t->torrent_file().name(); + target["r"] = "post" + boost::lexical_cast(p.piece); + target["t"] = "s"; + entry p; + p["target"] = target; + p["v"] = v; + entry e; + e["p"] = p; + e["sig_p"] = "peek"; + + entry::list_type lst; + lst.push_back(e); + if( t->alerts().should_post() ) { +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("*** post alert of fake dhtget reply (%s,%s,%s)" + , target["n"].string().c_str() + , target["r"].string().c_str() + , target["t"].string().c_str() ); +#endif + t->alerts().post_alert(dht_reply_data_alert(lst)); + } + } + } + disconnect(errors::no_error); + return; + } + // if we're already seeding, don't bother, // just ignore it if (t->is_seed()) @@ -2665,6 +2759,36 @@ namespace libtorrent #endif } + + void peer_connection::incoming_hashcash_nbits(int nbits) + { + INVARIANT_CHECK; + +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("<== HASHCASH BITS [ %d ]", nbits); +#endif + m_hashcash_nbits = nbits; + } + + void peer_connection::incoming_hashcash_nonce(const char *ptr, int size) + { + INVARIANT_CHECK; + +#ifdef TORRENT_VERBOSE_LOGGING + peer_log("<== HASHCASH NONCE [ size: %d ]", size); +#endif + if( m_hashcash_nonce.size() ) { + // just one hashcash nonce per connection allowed + disconnect(errors::invalid_message); + return; + } + + m_hashcash_nonce.clear(); + while(size--) { + m_hashcash_nonce.push_back(*ptr++); + } + } + // ----------------------------- // --------- HAVE ALL ---------- // ----------------------------- @@ -2827,6 +2951,11 @@ namespace libtorrent return; } + // if peek mode and fast piece is not what we want, return + if (t->m_peek_single_piece >=0 + && index != t->m_peek_single_piece) + return; + // if we don't have the metadata, we'll verify // this piece index later m_allowed_fast.push_back(index); diff --git a/libtorrent/src/policy.cpp b/libtorrent/src/policy.cpp index 31478e8a..7a39208e 100644 --- a/libtorrent/src/policy.cpp +++ b/libtorrent/src/policy.cpp @@ -283,6 +283,14 @@ namespace libtorrent bitfield const* bits = &c.get_bitfield(); bitfield fast_mask; + if (t.m_peek_single_piece >= 0) { + // build a bitmask containing just the peek piece + fast_mask.resize(c.get_bitfield().size(), false); + if (t.m_peek_single_piece < c.get_bitfield().size() + && (*bits)[t.m_peek_single_piece]) + fast_mask.set_bit(t.m_peek_single_piece); + bits = &fast_mask; + } else if (c.has_peer_choked()) { // if we are choked we can only pick pieces from the diff --git a/libtorrent/src/session_impl.cpp b/libtorrent/src/session_impl.cpp index 1d3d30e8..3efd5c43 100644 --- a/libtorrent/src/session_impl.cpp +++ b/libtorrent/src/session_impl.cpp @@ -681,6 +681,8 @@ namespace aux { , m_need_auto_manage(false) #if (defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS) && defined BOOST_HAS_PTHREADS , m_network_thread(0) + , m_hashcash_nbits(HASHCASH_MIN_NBITS) + , m_hashcash_reqs(0) #endif { #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS @@ -3563,6 +3565,17 @@ retry: } } + // -------------------------------------------------------------- + // adjust hashcash: too much reqs last second => increase difficulty + // -------------------------------------------------------------- + if (m_hashcash_reqs > 50) { + m_hashcash_nbits = std::min(m_hashcash_nbits+1, HASHCASH_MAX_NBITS); + } + if (!m_hashcash_reqs && (random() % 100) == 0 ) { + m_hashcash_nbits = std::max(m_hashcash_nbits-1, HASHCASH_MIN_NBITS); + } + m_hashcash_reqs = 0; + while (m_tick_residual >= 1000) m_tick_residual -= 1000; // m_peer_pool.release_memory(); } diff --git a/libtorrent/src/torrent.cpp b/libtorrent/src/torrent.cpp index 6cda9725..368c44d6 100644 --- a/libtorrent/src/torrent.cpp +++ b/libtorrent/src/torrent.cpp @@ -432,6 +432,7 @@ namespace libtorrent , m_in_state_updates(false) , m_is_active_download(false) , m_is_active_finished(false) + , m_peek_single_piece(-1) { if (!p.name.empty()) m_name.reset(new std::string(p.name)); @@ -508,6 +509,7 @@ namespace libtorrent set_max_connections(p.max_connections, false); set_upload_limit(p.upload_limit, false); set_download_limit(p.download_limit, false); + m_peek_single_piece = p.peek_single_piece; if (!m_name && !m_url.empty()) m_name.reset(new std::string(m_url));