Merge branch 'master' of /home/miguel/softs/twister

README.md

@@ -41,6 +41,11 @@ Developers of either bitcoin or libtorrent are welcomed and will be granted
 immediate write-access to the repository (a small retribution for
 bastardizing their codebases).
 
+Compiling
+---------
+
+See `INSTALL`.
+
 Testing
 -------
 

TODO

@@ -8,6 +8,9 @@ pseudocode:
    while( h > max_h )
         getTxIndex( "userX_h" ) => block h contains the previous tx
 
+- Until old public key is properly used, disable banning torrent peers due to bad piece hashes.
+note: torrent.cpp line 3286 (function piece_failed), iteration to ban peers is disabled (continue).
+
 - Count UTF8 chars in acceptSignedPost to proper limit the 140 characters.
 
 - Encrypt user_data (which contains all DMs)
@@ -36,6 +39,7 @@ merkle tree inside that block. This resource propagation cannot be sent right af
 registration for obvious reasons (no block yet, other nodes wouldn't accept the signed dht put).
 
 - Discuss and implement the acceptable level of spam per day (priorizing localization).
+DONE (except for the discussion part...)
 
 - Implement the mention forwarding mechanism discussed in the paper so user don't need to do polling
 and can also be sure to receive all mentions.
@@ -47,6 +51,18 @@ according to previous bittorrent research, would be enough to keep data availabl
 probability). twister also persists keys to disk. As userbase increases, old post storage and
 unreliable multivalued keys should better expire. Since those posts include the height and time, a
 policy may me defined.
+=> Implemented shouldDhtResourceExpire() which is tested on initialization but not really used yet.
+
+- Check stored dht values if their signature is still valid before trying to refresh another node.
+Key pair might have changed and currently we receive a lot of errors from other nodes.
+
+- save_file() must truncate file.
+
+- Save lastk field to post so torrent-less navigation through posts is possible. => DONE
+
+- Implement dht-to-torrent gateway, the "swarm" resource (so poster may not need to be member
+of his own torrent)
 
--
+- Estimate number of online followers by quering the "tracker" resource (implement a value within
+this resource to report the number of torrent peers)
 

libtorrent/include/libtorrent/kademlia/node.hpp

@@ -180,6 +180,7 @@ public:
 
 	void tick();
     bool refresh_storage();
+    bool has_expired(dht_storage_item const& item);
     bool save_storage(entry &save) const;
     void refresh(node_id const& id, find_data::nodes_callback const& f);
 	void bootstrap(std::vector<udp::endpoint> const& nodes

libtorrent/src/kademlia/find_data.cpp

@@ -208,7 +208,7 @@ bool find_data::invoke(observer_ptr o)
 
 	// im not going to ask trackers from myself
 	if( o->id() == m_node.nid() )
-		return true;
+		return false;
 
 	entry e;
 	e["z"] = "q";

libtorrent/src/kademlia/node.cpp

@@ -55,6 +55,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/rsa.hpp"
 
 #include "../../src/twister.h"
+//#define ENABLE_DHT_ITEM_EXPIRE
 
 namespace libtorrent { namespace dht
 {
@@ -502,6 +503,12 @@ bool node_impl::refresh_storage() {
         if( lsto.size() == 1 ) {
             dht_storage_item const& item = lsto.front();
 
+#ifdef ENABLE_DHT_ITEM_EXPIRE
+            if( has_expired(item) ) {
+                continue;
+            }
+#endif
+
             lazy_entry p;
             int pos;
             error_code err;
@@ -537,6 +544,49 @@ bool node_impl::refresh_storage() {
     return did_something;
 }
 
+bool node_impl::has_expired(dht_storage_item const& item) {
+    if (!verifySignature(item.p, item.sig_user, item.sig_p)) {
+        // invalid signature counts as expired
+        printf("node_impl::has_expired verifySignature failed\n");
+        return true;
+    }
+
+    lazy_entry arg_ent;
+    int pos;
+    error_code err;
+    // FIXME: optimize to avoid bdecode (store seq separated, etc)
+    int ret = lazy_bdecode(item.p.data(), item.p.data() + item.p.size(), arg_ent, err, &pos, 10, 500);
+
+    const static key_desc_t msg_desc[] = {
+        {"v", lazy_entry::none_t, 0, 0},
+        {"seq", lazy_entry::int_t, 0, key_desc_t::optional},
+        {"time", lazy_entry::int_t, 0, 0},
+        {"height", lazy_entry::int_t, 0, 0},
+        {"target", lazy_entry::dict_t, 0, key_desc_t::parse_children},
+        {"n", lazy_entry::string_t, 0, 0},
+        {"r", lazy_entry::string_t, 0, 0},
+        {"t", lazy_entry::string_t, 0, 0},
+    };
+    enum {mk_v = 0, mk_seq, mk_time, mk_height,
+          mk_target, mk_n, mk_r, mk_t};
+
+    // attempt to parse the message
+    lazy_entry const* msg_keys[8];
+    char error_string[200];
+    if (!verify_message(&arg_ent, msg_desc, msg_keys, 8, error_string, sizeof(error_string)))
+    {
+        printf("node_impl::has_expired verify_message failed\n");
+        // parse error (how come?) counts as expired
+        return true;
+    }
+
+    bool multi = (msg_keys[mk_t]->string_value() == "m");
+    int height = msg_keys[mk_height]->int_value();
+    std::string resource = msg_keys[mk_r]->string_value();
+
+    return shouldDhtResourceExpire(resource, multi, height);
+}
+
 bool node_impl::save_storage(entry &save) const {
     bool did_something = false;
 
@@ -593,7 +643,16 @@ void node_impl::load_storage(entry const* e) {
             item.p = j->find_key("p")->string();
             item.sig_p = j->find_key("sig_p")->string();
             item.sig_user = j->find_key("sig_user")->string();
-            to_add.push_back(item);
+
+            // just for printf for now
+            bool expired = has_expired(item);
+#ifdef ENABLE_DHT_ITEM_EXPIRE
+            if( !expired ) {
+#endif
+                to_add.push_back(item);
+#ifdef ENABLE_DHT_ITEM_EXPIRE
+            }
+#endif
         }
         m_storage_table.insert(std::make_pair(target, to_add));
     }
@@ -1152,6 +1211,18 @@ void node_impl::incoming_request(msg const& m, entry& e)
 			return;
 		}
 
+		/* we can't check username, otherwise we break hashtags etc.
+		if (multi && !usernameExists(msg_keys[mk_n]->string_value())) {
+			incoming_error(e, "unknown user for resource");
+			return;
+		}
+		*/
+
+		if (msg_keys[mk_r]->string_value().size() > 32) {
+			incoming_error(e, "resource name too big");
+			return;
+		}
+
 		if (!multi && (!msg_keys[mk_seq] || msg_keys[mk_seq]->int_value() < 0)) {
 			incoming_error(e, "seq is required for single");
 			return;

libtorrent/src/torrent.cpp

@@ -3283,6 +3283,9 @@ namespace libtorrent
 		for (std::set<void*>::iterator i = peers.begin()
 			, end(peers.end()); i != end; ++i)
 		{
+			// [MF] FIXME FIXME: BANNING BY FAILED HASH DISABLED - READ TODO!
+			continue;
+
 			policy::peer* p = static_cast<policy::peer*>(*i);
 			if (p == 0) continue;
 			TORRENT_ASSERT(p->in_use);
@@ -3337,7 +3340,7 @@ namespace libtorrent
 
 				if (p->connection)
 				{
-#ifdef TORRENT_LOGGING
+#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING
 					debug_log("*** BANNING PEER: \"%s\" Too many corrupt pieces"
 						, print_endpoint(p->ip()).c_str());
 #endif

src/bitcoinrpc.cpp

@@ -852,7 +852,7 @@ void StartRPCThreads()
     }
 
     rpc_worker_group = new boost::thread_group();
-    for (int i = 0; i < GetArg("-rpcthreads", 4); i++)
+    for (int i = 0; i < GetArg("-rpcthreads", 10); i++)
         rpc_worker_group->create_thread(boost::bind(&asio::io_service::run, rpc_io_service));
 }
 
@@ -983,11 +983,23 @@ void ServiceConnection(AcceptedConnection *conn)
             std::vector<char> file_data;
             if( load_file(strURI.c_str(), file_data) == 0 ) {
                 std::string str(file_data.data(), file_data.size());
-                conn->stream() << HTTPReply(HTTP_OK, str, false, "text/html") << std::flush;
+                const char *contentType = "text/html";
+                if( strURI.find(".js") != std::string::npos )
+                    contentType = "text/javascript";
+                if( strURI.find(".css") != std::string::npos )
+                    contentType = "text/css";
+                if( strURI.find(".png") != std::string::npos )
+                    contentType = "image/png";
+                if( strURI.find(".ttf") != std::string::npos )
+                    contentType = "application/x-font-ttf";
+                if( strURI.find(".jpg") != std::string::npos ||
+                    strURI.find(".jpeg") != std::string::npos )
+                    contentType = "image/jpeg";
+                conn->stream() << HTTPReply(HTTP_OK, str, false, contentType) << std::flush;
             } else {
                 conn->stream() << HTTPReply(HTTP_NOT_FOUND, "", false) << std::flush;
             }
-            break;
+            continue;
         }
 
         // Check authorization

src/init.cpp

@@ -726,6 +726,11 @@ bool AppInit2(boost::thread_group& threadGroup)
                     strLoadError = _("Corrupted block database detected");
                     break;
                 }
+
+                if( mapBlockIndex.size() > 1000 && nBestHeight == 0 ) {
+                    strLoadError = _("mapBlockIndex detected but nBestHeight still zero, trying to repair (reindex)");
+                    break;
+                }
             } catch(std::exception &e) {
                 strLoadError = _("Error opening block database");
                 break;
@@ -738,7 +743,7 @@ bool AppInit2(boost::thread_group& threadGroup)
             // first suggest a reindex
             if (!fReset) {
                 /*bool fRet =*/ uiInterface.ThreadSafeMessageBox(
-                    strLoadError + ".\n\n" + _("Do you want to rebuild the block database now?"),
+                    strLoadError + ".\n\n" + _("Do you want to rebuild the block database now? (assuming YES)"),
                     "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT);
                 if (true /* [MF] fRet*/) {
                     fReindex = true;

src/net.cpp

@@ -462,9 +462,9 @@ CNode* ConnectNode(CAddress addrConnect, const char *pszDest)
 
 
     /// debug print
-    printf("trying connection %s lastseen=%.1fhrs\n",
-        pszDest ? pszDest : addrConnect.ToString().c_str(),
-        pszDest ? 0 : (double)(GetAdjustedTime() - addrConnect.nTime)/3600.0);
+//    printf("trying connection %s lastseen=%.1fhrs\n",
+//        pszDest ? pszDest : addrConnect.ToString().c_str(),
+//        pszDest ? 0 : (double)(GetAdjustedTime() - addrConnect.nTime)/3600.0);
 
     // Connect
     SOCKET hSocket;

src/netbase.cpp

@@ -359,7 +359,7 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe
             int nRet = select(hSocket + 1, NULL, &fdset, NULL, &timeout);
             if (nRet == 0)
             {
-                printf("connection timeout\n");
+                //printf("connection timeout\n");
                 closesocket(hSocket);
                 return false;
             }
@@ -393,7 +393,7 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe
         else
 #endif
         {
-            printf("connect() failed: %i\n",WSAGetLastError());
+            //printf("connect() failed: %i\n",WSAGetLastError());
             closesocket(hSocket);
             return false;
         }

src/twister.cpp

@@ -31,6 +31,7 @@ twister::twister()
 #include "libtorrent/aux_/session_impl.hpp"
 
 #define DEBUG_ACCEPT_POST 1
+#define DEBUG_EXPIRE_DHT_ITEM 1
 
 using namespace libtorrent;
 static session *ses = NULL;
@@ -41,13 +42,19 @@ static map<sha1_hash, alert_manager*> m_dhtgetMap;
 
 static CCriticalSection cs_twister;
 static map<std::string, bool> m_specialResources;
+enum ExpireResType { SimpleNoExpire, NumberedNoExpire, PostNoExpireRecent };
+static map<std::string, ExpireResType> m_noExpireResources;
 static map<std::string, torrent_handle> m_userTorrent;
 
 static std::string m_preferredSpamLang = "[en]";
 static std::string m_receivedSpamMsgStr;
 static std::string m_receivedSpamUserStr;
+static int64       m_lastSpamTime = 0;
 static std::map<std::string,UserData> m_users;
 
+#define USER_DATA_FILE "user_data"
+#define GLOBAL_DATA_FILE "global_data"
+
 sha1_hash dhtTargetHash(std::string const &username, std::string const &resource, std::string const &type)
 {
     entry target;
@@ -95,6 +102,47 @@ int lastPostKfromTorrent(std::string const &username)
     return status.last_have;
 }
 
+int saveGlobalData(std::string const& filename)
+{
+    LOCK(cs_twister);
+    entry globalDict;
+
+    globalDict["preferredSpamLang"] = m_preferredSpamLang;
+    globalDict["receivedSpamMsg"]   = m_receivedSpamMsgStr;
+    globalDict["receivedSpamUser"]  = m_receivedSpamUserStr;
+    globalDict["lastSpamTime"]      = m_lastSpamTime;
+
+    std::vector<char> buf;
+    bencode(std::back_inserter(buf), globalDict);
+    return save_file(filename, buf);
+}
+
+int loadGlobalData(std::string const& filename)
+{
+    LOCK(cs_twister);
+    std::vector<char> in;
+    if (load_file(filename, in) == 0) {
+        lazy_entry userDict;
+        error_code ec;
+        if (lazy_bdecode(&in[0], &in[0] + in.size(), userDict, ec) == 0) {
+            if( userDict.type() != lazy_entry::dict_t ) goto data_error;
+
+            m_preferredSpamLang   = userDict.dict_find_string_value("preferredSpamLang");
+            m_receivedSpamMsgStr  = userDict.dict_find_string_value("receivedSpamMsg");
+            m_receivedSpamUserStr = userDict.dict_find_string_value("receivedSpamUser");
+            m_lastSpamTime        = userDict.dict_find_int_value("lastSpamTime");
+
+            return 0;
+        }
+    }
+    return -1;
+
+data_error:
+    printf("loadGlobalData: unexpected bencode type - global_data corrupt!\n");
+    return -2;
+}
+
+
 void ThreadWaitExtIP()
 {
     RenameThread("wait-extip");
@@ -159,6 +207,9 @@ void ThreadWaitExtIP()
     //settings.dht_announce_interval = 60; // test
     //settings.min_announce_interval = 60; // test
     settings.anonymous_mode = false; // (false => send peer_id, avoid connecting to itself)
+    // disable read cache => there is still some bug due to twister piece size changes
+    settings.use_read_cache = false;
+    settings.cache_size = 0;
     ses->set_settings(settings);
 
     printf("libtorrent + dht started\n");
@@ -171,9 +222,12 @@ void ThreadWaitExtIP()
             break;
     }
 
+    boost::filesystem::path globalDataPath = GetDataDir() / GLOBAL_DATA_FILE;
+    loadGlobalData(globalDataPath.string());
+
     {
         LOCK(cs_twister);
-        boost::filesystem::path userDataPath = GetDataDir() / "user_data";
+        boost::filesystem::path userDataPath = GetDataDir() / USER_DATA_FILE;
         loadUserData(userDataPath.string(), m_users);
         printf("loaded user_data for %zd users\n", m_users.size());
 
@@ -389,6 +443,12 @@ void startSessionTorrent(boost::thread_group& threadGroup)
     m_specialResources["tracker"] = true;
     m_specialResources["swarm"] = true;
 
+    // these are the resources which shouldn't expire
+    m_noExpireResources["avatar"] = SimpleNoExpire;
+    m_noExpireResources["profile"] = SimpleNoExpire;
+    m_noExpireResources["following"] = NumberedNoExpire;
+    m_noExpireResources["status"] = SimpleNoExpire;
+    m_noExpireResources["post"] = PostNoExpireRecent;
 
     threadGroup.create_thread(boost::bind(&ThreadWaitExtIP));
     threadGroup.create_thread(boost::bind(&ThreadMaintainDHTNodes));
@@ -442,18 +502,21 @@ void stopSessionTorrent()
             entry session_state;
             ses->save_state(session_state);
 
-	    std::vector<char> out;
-	    bencode(std::back_inserter(out), session_state);
-	    boost::filesystem::path sesStatePath = GetDataDir() / "ses_state";
-	    save_file(sesStatePath.string(), out);
+            std::vector<char> out;
+            bencode(std::back_inserter(out), session_state);
+            boost::filesystem::path sesStatePath = GetDataDir() / "ses_state";
+            save_file(sesStatePath.string(), out);
 
-	    delete ses;
-	    ses = NULL;
+            delete ses;
+            ses = NULL;
     }
 
+    boost::filesystem::path globalDataPath = GetDataDir() / GLOBAL_DATA_FILE;
+    saveGlobalData(globalDataPath.string());
+
     if( m_users.size() ) {
         printf("saving user_data (followers and DMs)...\n");
-        boost::filesystem::path userDataPath = GetDataDir() / "user_data";
+        boost::filesystem::path userDataPath = GetDataDir() / USER_DATA_FILE;
         saveUserData(userDataPath.string(), m_users);
     }
 
@@ -579,9 +642,11 @@ bool processReceivedDM(lazy_entry const* post)
             } else {
                 std::string textOut;
                 if( key.Decrypt(sec, textOut) ) {
+                    /* this printf is good for debug, but bad for security.
                     printf("Received DM for user '%s' text = '%s'\n",
                            item.second.username.c_str(),
                            textOut.c_str());
+                    */
 
                     std::string n = post->dict_find_string_value("n");
 
@@ -713,6 +778,15 @@ bool validatePostNumberForUser(std::string const &username, int k)
     return true;
 }
 
+bool usernameExists(std::string const &username)
+{
+    CTransaction txOut;
+    uint256 hashBlock;
+    uint256 userhash = SerializeHash(username);
+    return GetTransaction(userhash, txOut, hashBlock);
+}
+
+
 /*
 "userpost" :
 {
@@ -808,11 +882,90 @@ int getBestHeight()
     return nBestHeight;
 }
 
+bool shouldDhtResourceExpire(std::string resource, bool multi, int height)
+{
+    if ((height + BLOCK_AGE_TO_EXPIRE_DHT_ENTRY) < getBestHeight() ) {
+        if( multi ) {
+#ifdef DEBUG_EXPIRE_DHT_ITEM
+            printf("shouldDhtResourceExpire: expiring resource multi '%s'\n", resource.c_str());
+#endif
+            return true;
+        }
+
+        // extract basic resource string (without numbering)
+        std::string resourceBasic;
+        for(size_t i = 0; i < resource.size() && isalpha(resource.at(i)); i++) {
+            resourceBasic.push_back(resource.at(i));
+        }
+
+        int resourceNumber = -1;
+        if( resource.length() > resourceBasic.length() ) {
+            // make sure it is a valid number following (all digits)
+            if( resource.at(resourceBasic.length()) == '0' &&
+                resource.size() > resourceBasic.length() + 1 ){
+                // leading zeros not allowed
+            } else {
+                size_t i;
+                for(i = resourceBasic.length(); i < resource.size() &&
+                    isdigit(resource.at(i)); i++) {
+                }
+                if(i == resource.size()) {
+                    resourceNumber = atoi( resource.c_str() + resourceBasic.length() );
+                }
+            }
+        }
+
+        if( !m_noExpireResources.count(resourceBasic) ) {
+            // unknown resource. expire it.
+#ifdef DEBUG_EXPIRE_DHT_ITEM
+            printf("shouldDhtResourceExpire: expiring non-special resource '%s'\n", resource.c_str());
+#endif
+            return true;
+        } else {
+            if( m_noExpireResources[resourceBasic] == SimpleNoExpire &&
+                resource.length() > resourceBasic.length() ) {
+                // this resource admits no number. expire it!
+#ifdef DEBUG_EXPIRE_DHT_ITEM
+                printf("shouldDhtResourceExpire: expiring resource with unexpected numbering '%s'\n", resource.c_str());
+#endif
+                return true;
+            }
+            if( m_noExpireResources[resourceBasic] == NumberedNoExpire &&
+                (resourceNumber < 0 || resourceNumber > 200) ) {
+                // try keeping a sane number here, otherwise expire it!
+#ifdef DEBUG_EXPIRE_DHT_ITEM
+                printf("shouldDhtResourceExpire: expiring numbered resource with no sane number '%s'\n", resource.c_str());
+#endif
+                return true;
+            }
+            if( m_noExpireResources[resourceBasic] == PostNoExpireRecent && resourceNumber < 0 ) {
+#ifdef DEBUG_EXPIRE_DHT_ITEM
+                printf("shouldDhtResourceExpire: expiring post with invalid numbering '%s'\n", resource.c_str());
+#endif
+                return true;
+            }
+            if( m_noExpireResources[resourceBasic] == PostNoExpireRecent &&
+                (height + BLOCK_AGE_TO_EXPIRE_DHT_POSTS) < getBestHeight() ) {
+#ifdef DEBUG_EXPIRE_DHT_ITEM
+                printf("shouldDhtResourceExpire: expiring old post resource '%s'\n", resource.c_str());
+#endif
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
 void receivedSpamMessage(std::string const &message, std::string const &user)
 {
     LOCK(cs_twister);
-    if( !m_receivedSpamMsgStr.length() ||
-         (m_preferredSpamLang.length() && message.find(m_preferredSpamLang) != string::npos) ) {
+    bool hasSingleLangCode = (message.find("[") == message.rfind("["));
+    bool hasPreferredLang  = m_preferredSpamLang.length();
+    bool isSameLang        = hasPreferredLang && hasSingleLangCode &&
+                             message.find(m_preferredSpamLang) != string::npos;
+    bool currentlyEmpty    = !m_receivedSpamMsgStr.length();
+
+    if( currentlyEmpty || (isSameLang && rand() < (RAND_MAX/2)) ) {
         m_receivedSpamMsgStr = message;
         m_receivedSpamUserStr = user;
     }
@@ -904,6 +1057,34 @@ Value dhtget(const Array& params, bool fHelp)
     return ret;
 }
 
+int findLastPublicPostLocalUser( std::string strUsername )
+{
+    int lastk = -1;
+
+    LOCK(cs_twister);
+    if( strUsername.size() && m_userTorrent.count(strUsername) &&
+        m_userTorrent[strUsername].is_valid() ){
+
+        std::vector<std::string> pieces;
+        int max_id = std::numeric_limits<int>::max();
+        int since_id = -1;
+        m_userTorrent[strUsername].get_pieces(pieces, 1, max_id, since_id, USERPOST_FLAG_RT);
+
+        if( pieces.size() ) {
+            string const& piece = pieces.front();
+            lazy_entry v;
+            int pos;
+            error_code ec;
+            if (lazy_bdecode(piece.data(), piece.data()+piece.size(), v, ec, &pos) == 0) {
+                lazy_entry const* post = v.dict_find_dict("userpost");
+                lastk = post->dict_find_int_value("k",-1);
+            }
+        }
+    }
+    return lastk;
+}
+
+
 Value newpostmsg(const Array& params, bool fHelp)
 {
     if (fHelp || (params.size() != 3 && params.size() != 5))
@@ -927,6 +1108,11 @@ Value newpostmsg(const Array& params, bool fHelp)
     }
 
     entry v;
+    // [MF] Warning: findLastPublicPostLocalUser requires that we follow ourselves
+    int lastk = findLastPublicPostLocalUser(strUsername);
+    if( lastk >= 0 )
+        v["userpost"]["lastk"] = lastk;
+
     if( !createSignedUserpost(v, strUsername, k, strMsg,
                          NULL, NULL, NULL,
                          strReplyN, replyK) )
@@ -1046,6 +1232,11 @@ Value newrtmsg(const Array& params, bool fHelp)
     entry const *sig_rt= vrt.find_key("sig_userpost");
 
     entry v;
+    // [MF] Warning: findLastPublicPostLocalUser requires that we follow ourselves
+    int lastk = findLastPublicPostLocalUser(strUsername);
+    if( lastk >= 0 )
+        v["userpost"]["lastk"] = lastk;
+
     if( !createSignedUserpost(v, strUsername, k, "",
                               rt, sig_rt, NULL,
                               std::string(""), 0) )
@@ -1140,24 +1331,26 @@ Value getposts(const Array& params, bool fHelp)
 
     {
         LOCK(cs_twister);
-        if( m_receivedSpamMsgStr.length() ) {
-            // we must agree on an acceptable level here
-            if( rand() < (RAND_MAX/10) ) {
-                entry v;
-                entry &userpost = v["userpost"];
-
-                userpost["n"] = m_receivedSpamUserStr;
-                userpost["k"] = 1;
-                userpost["time"] = GetAdjustedTime();
-                userpost["height"] = getBestHeight();
-
-                userpost["msg"] = m_receivedSpamMsgStr;
-
-                unsigned char vchSig[65];
-                RAND_bytes(vchSig,sizeof(vchSig));
-                v["sig_userpost"] = std::string((const char *)vchSig, sizeof(vchSig));
-                ret.insert(ret.begin(),entryToJson(v));
-            }
+        // we must agree on an acceptable level here
+        // what about one every eight hours? (not cumulative)
+        if( m_receivedSpamMsgStr.length() && GetAdjustedTime() > m_lastSpamTime + (8*3600) ) {
+            m_lastSpamTime = GetAdjustedTime();
+
+            entry v;
+            entry &userpost = v["userpost"];
+
+            userpost["n"] = m_receivedSpamUserStr;
+            userpost["k"] = 1;
+            userpost["time"] = GetAdjustedTime();
+            userpost["height"] = getBestHeight();
+
+            userpost["msg"] = m_receivedSpamMsgStr;
+
+            unsigned char vchSig[65];
+            RAND_bytes(vchSig,sizeof(vchSig));
+            v["sig_userpost"] = std::string((const char *)vchSig, sizeof(vchSig));
+            ret.insert(ret.begin(),entryToJson(v));
+
             m_receivedSpamMsgStr = "";
             m_receivedSpamUserStr = "";
         }

src/twister.h

@@ -10,6 +10,9 @@
 #define USERPOST_FLAG_RT 0x01
 #define USERPOST_FLAG_DM 0x02
 
+#define BLOCK_AGE_TO_EXPIRE_DHT_ENTRY (2016)   // about 2 weeks
+#define BLOCK_AGE_TO_EXPIRE_DHT_POSTS (4320*6) // about 6 months
+
 
 class twister
 {
@@ -26,9 +29,11 @@ bool verifySignature(std::string const &strMessage, std::string const &strUserna
 
 bool acceptSignedPost(char const *data, int data_size, std::string username, int seq, std::string &errmsg, boost::uint32_t *flags);
 bool validatePostNumberForUser(std::string const &username, int k);
+bool usernameExists(std::string const &username);
 
 void receivedSpamMessage(std::string const &message, std::string const &user);
 
 int getBestHeight();
+bool shouldDhtResourceExpire(std::string resource, bool multi, int height);
 
 #endif // TWISTER_H

twister-test.py

@@ -3,7 +3,7 @@
 import os,sys,time
 
 ext_ip  = os.environ['EXTIP']
-twister = "../twister-qt-build-desktop/twisterd"
+twister = "./twisterd"
 
 cmd = sys.argv[1]
 n   = int(sys.argv[2])