implement expiring dht items (but it is not enabled yet)

TODO

@@ -47,6 +47,10 @@ according to previous bittorrent research, would be enough to keep data availabl
 probability). twister also persists keys to disk. As userbase increases, old post storage and
 unreliable multivalued keys should better expire. Since those posts include the height and time, a
 policy may me defined.
+=> Implemented shouldDhtResourceExpire() which is tested on initialization but not really used yet.
+
+- Check stored dht values if their signature is still valid before trying to refresh another node.
+Key pair might have changed and currently we receive a lot of errors from other nodes.
+
 
--
 

libtorrent/include/libtorrent/kademlia/node.hpp

@@ -180,6 +180,7 @@ public:
 
 	void tick();
     bool refresh_storage();
+    bool has_expired(dht_storage_item const& item);
     bool save_storage(entry &save) const;
     void refresh(node_id const& id, find_data::nodes_callback const& f);
 	void bootstrap(std::vector<udp::endpoint> const& nodes

libtorrent/src/kademlia/node.cpp

@@ -537,6 +537,49 @@ bool node_impl::refresh_storage() {
     return did_something;
 }
 
+bool node_impl::has_expired(dht_storage_item const& item) {
+    if (!verifySignature(item.p, item.sig_user, item.sig_p)) {
+        // invalid signature counts as expired
+        printf("node_impl::has_expired verifySignature failed\n");
+        return true;
+    }
+
+    lazy_entry arg_ent;
+    int pos;
+    error_code err;
+    // FIXME: optimize to avoid bdecode (store seq separated, etc)
+    int ret = lazy_bdecode(item.p.data(), item.p.data() + item.p.size(), arg_ent, err, &pos, 10, 500);
+
+    const static key_desc_t msg_desc[] = {
+        {"v", lazy_entry::none_t, 0, 0},
+        {"seq", lazy_entry::int_t, 0, key_desc_t::optional},
+        {"time", lazy_entry::int_t, 0, 0},
+        {"height", lazy_entry::int_t, 0, 0},
+        {"target", lazy_entry::dict_t, 0, key_desc_t::parse_children},
+        {"n", lazy_entry::string_t, 0, 0},
+        {"r", lazy_entry::string_t, 0, 0},
+        {"t", lazy_entry::string_t, 0, 0},
+    };
+    enum {mk_v = 0, mk_seq, mk_time, mk_height,
+          mk_target, mk_n, mk_r, mk_t};
+
+    // attempt to parse the message
+    lazy_entry const* msg_keys[8];
+    char error_string[200];
+    if (!verify_message(&arg_ent, msg_desc, msg_keys, 8, error_string, sizeof(error_string)))
+    {
+        printf("node_impl::has_expired verify_message failed\n");
+        // parse error (how come?) counts as expired
+        return true;
+    }
+
+    bool multi = (msg_keys[mk_t]->string_value() == "m");
+    int height = msg_keys[mk_height]->int_value();
+    std::string resource = msg_keys[mk_r]->string_value();
+
+    return shouldDhtResourceExpire(resource, multi, height);
+}
+
 bool node_impl::save_storage(entry &save) const {
     bool did_something = false;
 
@@ -593,6 +636,10 @@ void node_impl::load_storage(entry const* e) {
             item.p = j->find_key("p")->string();
             item.sig_p = j->find_key("sig_p")->string();
             item.sig_user = j->find_key("sig_user")->string();
+
+            // just for printf for now
+            has_expired(item);
+
             to_add.push_back(item);
         }
         m_storage_table.insert(std::make_pair(target, to_add));
@@ -1152,6 +1199,18 @@ void node_impl::incoming_request(msg const& m, entry& e)
 			return;
 		}
 
+		/* we can't check username, otherwise we break hashtags etc.
+		if (multi && !usernameExists(msg_keys[mk_n]->string_value())) {
+			incoming_error(e, "unknown user for resource");
+			return;
+		}
+		*/
+
+		if (msg_keys[mk_r]->string_value().size() > 32) {
+			incoming_error(e, "resource name too big");
+			return;
+		}
+
 		if (!multi && (!msg_keys[mk_seq] || msg_keys[mk_seq]->int_value() < 0)) {
 			incoming_error(e, "seq is required for single");
 			return;

src/twister.cpp

@@ -41,6 +41,7 @@ static map<sha1_hash, alert_manager*> m_dhtgetMap;
 
 static CCriticalSection cs_twister;
 static map<std::string, bool> m_specialResources;
+static map<std::string, bool> m_noExpireResources; // bool is true if expected number after resource string
 static map<std::string, torrent_handle> m_userTorrent;
 
 static std::string m_preferredSpamLang = "[en]";
@@ -389,6 +390,11 @@ void startSessionTorrent(boost::thread_group& threadGroup)
     m_specialResources["tracker"] = true;
     m_specialResources["swarm"] = true;
 
+    // these are the resources which shouldn't expire (true when numbering is expected)
+    m_noExpireResources["avatar"] = false;
+    m_noExpireResources["profile"] = false;
+    m_noExpireResources["following"] = true;
+
 
     threadGroup.create_thread(boost::bind(&ThreadWaitExtIP));
     threadGroup.create_thread(boost::bind(&ThreadMaintainDHTNodes));
@@ -713,6 +719,15 @@ bool validatePostNumberForUser(std::string const &username, int k)
     return true;
 }
 
+bool usernameExists(std::string const &username)
+{
+    CTransaction txOut;
+    uint256 hashBlock;
+    uint256 userhash = SerializeHash(username);
+    return GetTransaction(userhash, txOut, hashBlock);
+}
+
+
 /*
 "userpost" :
 {
@@ -808,6 +823,43 @@ int getBestHeight()
     return nBestHeight;
 }
 
+bool shouldDhtResourceExpire(std::string resource, bool multi, int height)
+{
+    if ((height + BLOCK_AGE_TO_EXPIRE_DHT_ENTRY) < getBestHeight() ) {
+        if( multi ) {
+            printf("shouldDhtResourceExpire: expiring resource multi '%s'\n", resource.c_str());
+            return true;
+        }
+
+        std::string resourceBasic;
+        for(size_t i = 0; i < resource.size() && isalpha(resource.at(i)); i++) {
+            resourceBasic.push_back(resource.at(i));
+        }
+        int resourceNumber = -1;
+
+        if( resource.length() > resourceBasic.length() ) {
+            resourceNumber = atoi( resource.c_str() + resourceBasic.length() );
+        }
+
+        if( !m_noExpireResources.count(resourceBasic) ) {
+            // unknown resource. expire it.
+            printf("shouldDhtResourceExpire: expiring non-special resource '%s'\n", resource.c_str());
+        } else {
+            if( !m_noExpireResources[resourceBasic] && resourceNumber >= 0 ) {
+                // this resource admits no number. expire it!
+                printf("shouldDhtResourceExpire: expiring resource with unexpected numbering '%s'\n", resource.c_str());
+                return true;
+            }
+            if( m_noExpireResources[resourceBasic] && resourceNumber > 200 ) {
+                // try keeping a sane number here, otherwise expire it!
+                printf("shouldDhtResourceExpire: expiring resource with numbering too big '%s'\n", resource.c_str());
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
 void receivedSpamMessage(std::string const &message, std::string const &user)
 {
     LOCK(cs_twister);

src/twister.h

@@ -10,6 +10,7 @@
 #define USERPOST_FLAG_RT 0x01
 #define USERPOST_FLAG_DM 0x02
 
+#define BLOCK_AGE_TO_EXPIRE_DHT_ENTRY 2000
 
 class twister
 {
@@ -26,9 +27,11 @@ bool verifySignature(std::string const &strMessage, std::string const &strUserna
 
 bool acceptSignedPost(char const *data, int data_size, std::string username, int seq, std::string &errmsg, boost::uint32_t *flags);
 bool validatePostNumberForUser(std::string const &username, int k);
+bool usernameExists(std::string const &username);
 
 void receivedSpamMessage(std::string const &message, std::string const &user);
 
 int getBestHeight();
+bool shouldDhtResourceExpire(std::string resource, bool multi, int height);
 
 #endif // TWISTER_H