From 7e1726ba8c08f4259a3e50781f8f77093ab46cd4 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Fri, 18 Oct 2013 12:08:54 -0300 Subject: [PATCH] implement expiring dht items (but it is not enabled yet) --- TODO | 6 +- .../include/libtorrent/kademlia/node.hpp | 1 + libtorrent/src/kademlia/node.cpp | 59 +++++++++++++++++++ src/twister.cpp | 52 ++++++++++++++++ src/twister.h | 3 + 5 files changed, 120 insertions(+), 1 deletion(-) diff --git a/TODO b/TODO index ca59cca8..8760c32b 100644 --- a/TODO +++ b/TODO @@ -47,6 +47,10 @@ according to previous bittorrent research, would be enough to keep data availabl probability). twister also persists keys to disk. As userbase increases, old post storage and unreliable multivalued keys should better expire. Since those posts include the height and time, a policy may me defined. +=> Implemented shouldDhtResourceExpire() which is tested on initialization but not really used yet. + +- Check stored dht values if their signature is still valid before trying to refresh another node. +Key pair might have changed and currently we receive a lot of errors from other nodes. + -- diff --git a/libtorrent/include/libtorrent/kademlia/node.hpp b/libtorrent/include/libtorrent/kademlia/node.hpp index 01434ef6..2898b738 100644 --- a/libtorrent/include/libtorrent/kademlia/node.hpp +++ b/libtorrent/include/libtorrent/kademlia/node.hpp @@ -180,6 +180,7 @@ public: void tick(); bool refresh_storage(); + bool has_expired(dht_storage_item const& item); bool save_storage(entry &save) const; void refresh(node_id const& id, find_data::nodes_callback const& f); void bootstrap(std::vector const& nodes diff --git a/libtorrent/src/kademlia/node.cpp b/libtorrent/src/kademlia/node.cpp index 43ba225d..f6d28f41 100644 --- a/libtorrent/src/kademlia/node.cpp +++ b/libtorrent/src/kademlia/node.cpp @@ -537,6 +537,49 @@ bool node_impl::refresh_storage() { return did_something; } +bool node_impl::has_expired(dht_storage_item const& item) { + if (!verifySignature(item.p, item.sig_user, item.sig_p)) { + // invalid signature counts as expired + printf("node_impl::has_expired verifySignature failed\n"); + return true; + } + + lazy_entry arg_ent; + int pos; + error_code err; + // FIXME: optimize to avoid bdecode (store seq separated, etc) + int ret = lazy_bdecode(item.p.data(), item.p.data() + item.p.size(), arg_ent, err, &pos, 10, 500); + + const static key_desc_t msg_desc[] = { + {"v", lazy_entry::none_t, 0, 0}, + {"seq", lazy_entry::int_t, 0, key_desc_t::optional}, + {"time", lazy_entry::int_t, 0, 0}, + {"height", lazy_entry::int_t, 0, 0}, + {"target", lazy_entry::dict_t, 0, key_desc_t::parse_children}, + {"n", lazy_entry::string_t, 0, 0}, + {"r", lazy_entry::string_t, 0, 0}, + {"t", lazy_entry::string_t, 0, 0}, + }; + enum {mk_v = 0, mk_seq, mk_time, mk_height, + mk_target, mk_n, mk_r, mk_t}; + + // attempt to parse the message + lazy_entry const* msg_keys[8]; + char error_string[200]; + if (!verify_message(&arg_ent, msg_desc, msg_keys, 8, error_string, sizeof(error_string))) + { + printf("node_impl::has_expired verify_message failed\n"); + // parse error (how come?) counts as expired + return true; + } + + bool multi = (msg_keys[mk_t]->string_value() == "m"); + int height = msg_keys[mk_height]->int_value(); + std::string resource = msg_keys[mk_r]->string_value(); + + return shouldDhtResourceExpire(resource, multi, height); +} + bool node_impl::save_storage(entry &save) const { bool did_something = false; @@ -593,6 +636,10 @@ void node_impl::load_storage(entry const* e) { item.p = j->find_key("p")->string(); item.sig_p = j->find_key("sig_p")->string(); item.sig_user = j->find_key("sig_user")->string(); + + // just for printf for now + has_expired(item); + to_add.push_back(item); } m_storage_table.insert(std::make_pair(target, to_add)); @@ -1152,6 +1199,18 @@ void node_impl::incoming_request(msg const& m, entry& e) return; } + /* we can't check username, otherwise we break hashtags etc. + if (multi && !usernameExists(msg_keys[mk_n]->string_value())) { + incoming_error(e, "unknown user for resource"); + return; + } + */ + + if (msg_keys[mk_r]->string_value().size() > 32) { + incoming_error(e, "resource name too big"); + return; + } + if (!multi && (!msg_keys[mk_seq] || msg_keys[mk_seq]->int_value() < 0)) { incoming_error(e, "seq is required for single"); return; diff --git a/src/twister.cpp b/src/twister.cpp index 63be0eb3..f4b64138 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -41,6 +41,7 @@ static map m_dhtgetMap; static CCriticalSection cs_twister; static map m_specialResources; +static map m_noExpireResources; // bool is true if expected number after resource string static map m_userTorrent; static std::string m_preferredSpamLang = "[en]"; @@ -389,6 +390,11 @@ void startSessionTorrent(boost::thread_group& threadGroup) m_specialResources["tracker"] = true; m_specialResources["swarm"] = true; + // these are the resources which shouldn't expire (true when numbering is expected) + m_noExpireResources["avatar"] = false; + m_noExpireResources["profile"] = false; + m_noExpireResources["following"] = true; + threadGroup.create_thread(boost::bind(&ThreadWaitExtIP)); threadGroup.create_thread(boost::bind(&ThreadMaintainDHTNodes)); @@ -713,6 +719,15 @@ bool validatePostNumberForUser(std::string const &username, int k) return true; } +bool usernameExists(std::string const &username) +{ + CTransaction txOut; + uint256 hashBlock; + uint256 userhash = SerializeHash(username); + return GetTransaction(userhash, txOut, hashBlock); +} + + /* "userpost" : { @@ -808,6 +823,43 @@ int getBestHeight() return nBestHeight; } +bool shouldDhtResourceExpire(std::string resource, bool multi, int height) +{ + if ((height + BLOCK_AGE_TO_EXPIRE_DHT_ENTRY) < getBestHeight() ) { + if( multi ) { + printf("shouldDhtResourceExpire: expiring resource multi '%s'\n", resource.c_str()); + return true; + } + + std::string resourceBasic; + for(size_t i = 0; i < resource.size() && isalpha(resource.at(i)); i++) { + resourceBasic.push_back(resource.at(i)); + } + int resourceNumber = -1; + + if( resource.length() > resourceBasic.length() ) { + resourceNumber = atoi( resource.c_str() + resourceBasic.length() ); + } + + if( !m_noExpireResources.count(resourceBasic) ) { + // unknown resource. expire it. + printf("shouldDhtResourceExpire: expiring non-special resource '%s'\n", resource.c_str()); + } else { + if( !m_noExpireResources[resourceBasic] && resourceNumber >= 0 ) { + // this resource admits no number. expire it! + printf("shouldDhtResourceExpire: expiring resource with unexpected numbering '%s'\n", resource.c_str()); + return true; + } + if( m_noExpireResources[resourceBasic] && resourceNumber > 200 ) { + // try keeping a sane number here, otherwise expire it! + printf("shouldDhtResourceExpire: expiring resource with numbering too big '%s'\n", resource.c_str()); + return true; + } + } + } + return false; +} + void receivedSpamMessage(std::string const &message, std::string const &user) { LOCK(cs_twister); diff --git a/src/twister.h b/src/twister.h index 2b858891..ecd82053 100644 --- a/src/twister.h +++ b/src/twister.h @@ -10,6 +10,7 @@ #define USERPOST_FLAG_RT 0x01 #define USERPOST_FLAG_DM 0x02 +#define BLOCK_AGE_TO_EXPIRE_DHT_ENTRY 2000 class twister { @@ -26,9 +27,11 @@ bool verifySignature(std::string const &strMessage, std::string const &strUserna bool acceptSignedPost(char const *data, int data_size, std::string username, int seq, std::string &errmsg, boost::uint32_t *flags); bool validatePostNumberForUser(std::string const &username, int k); +bool usernameExists(std::string const &username); void receivedSpamMessage(std::string const &message, std::string const &user); int getBestHeight(); +bool shouldDhtResourceExpire(std::string resource, bool multi, int height); #endif // TWISTER_H