From 7e1726ba8c08f4259a3e50781f8f77093ab46cd4 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Fri, 18 Oct 2013 12:08:54 -0300 Subject: [PATCH 01/17] implement expiring dht items (but it is not enabled yet) --- TODO | 6 +- .../include/libtorrent/kademlia/node.hpp | 1 + libtorrent/src/kademlia/node.cpp | 59 +++++++++++++++++++ src/twister.cpp | 52 ++++++++++++++++ src/twister.h | 3 + 5 files changed, 120 insertions(+), 1 deletion(-) diff --git a/TODO b/TODO index ca59cca8..8760c32b 100644 --- a/TODO +++ b/TODO @@ -47,6 +47,10 @@ according to previous bittorrent research, would be enough to keep data availabl probability). twister also persists keys to disk. As userbase increases, old post storage and unreliable multivalued keys should better expire. Since those posts include the height and time, a policy may me defined. +=> Implemented shouldDhtResourceExpire() which is tested on initialization but not really used yet. + +- Check stored dht values if their signature is still valid before trying to refresh another node. +Key pair might have changed and currently we receive a lot of errors from other nodes. + -- diff --git a/libtorrent/include/libtorrent/kademlia/node.hpp b/libtorrent/include/libtorrent/kademlia/node.hpp index 01434ef6..2898b738 100644 --- a/libtorrent/include/libtorrent/kademlia/node.hpp +++ b/libtorrent/include/libtorrent/kademlia/node.hpp @@ -180,6 +180,7 @@ public: void tick(); bool refresh_storage(); + bool has_expired(dht_storage_item const& item); bool save_storage(entry &save) const; void refresh(node_id const& id, find_data::nodes_callback const& f); void bootstrap(std::vector const& nodes diff --git a/libtorrent/src/kademlia/node.cpp b/libtorrent/src/kademlia/node.cpp index 43ba225d..f6d28f41 100644 --- a/libtorrent/src/kademlia/node.cpp +++ b/libtorrent/src/kademlia/node.cpp @@ -537,6 +537,49 @@ bool node_impl::refresh_storage() { return did_something; } +bool node_impl::has_expired(dht_storage_item const& item) { + if (!verifySignature(item.p, item.sig_user, item.sig_p)) { + // invalid signature counts as expired + printf("node_impl::has_expired verifySignature failed\n"); + return true; + } + + lazy_entry arg_ent; + int pos; + error_code err; + // FIXME: optimize to avoid bdecode (store seq separated, etc) + int ret = lazy_bdecode(item.p.data(), item.p.data() + item.p.size(), arg_ent, err, &pos, 10, 500); + + const static key_desc_t msg_desc[] = { + {"v", lazy_entry::none_t, 0, 0}, + {"seq", lazy_entry::int_t, 0, key_desc_t::optional}, + {"time", lazy_entry::int_t, 0, 0}, + {"height", lazy_entry::int_t, 0, 0}, + {"target", lazy_entry::dict_t, 0, key_desc_t::parse_children}, + {"n", lazy_entry::string_t, 0, 0}, + {"r", lazy_entry::string_t, 0, 0}, + {"t", lazy_entry::string_t, 0, 0}, + }; + enum {mk_v = 0, mk_seq, mk_time, mk_height, + mk_target, mk_n, mk_r, mk_t}; + + // attempt to parse the message + lazy_entry const* msg_keys[8]; + char error_string[200]; + if (!verify_message(&arg_ent, msg_desc, msg_keys, 8, error_string, sizeof(error_string))) + { + printf("node_impl::has_expired verify_message failed\n"); + // parse error (how come?) counts as expired + return true; + } + + bool multi = (msg_keys[mk_t]->string_value() == "m"); + int height = msg_keys[mk_height]->int_value(); + std::string resource = msg_keys[mk_r]->string_value(); + + return shouldDhtResourceExpire(resource, multi, height); +} + bool node_impl::save_storage(entry &save) const { bool did_something = false; @@ -593,6 +636,10 @@ void node_impl::load_storage(entry const* e) { item.p = j->find_key("p")->string(); item.sig_p = j->find_key("sig_p")->string(); item.sig_user = j->find_key("sig_user")->string(); + + // just for printf for now + has_expired(item); + to_add.push_back(item); } m_storage_table.insert(std::make_pair(target, to_add)); @@ -1152,6 +1199,18 @@ void node_impl::incoming_request(msg const& m, entry& e) return; } + /* we can't check username, otherwise we break hashtags etc. + if (multi && !usernameExists(msg_keys[mk_n]->string_value())) { + incoming_error(e, "unknown user for resource"); + return; + } + */ + + if (msg_keys[mk_r]->string_value().size() > 32) { + incoming_error(e, "resource name too big"); + return; + } + if (!multi && (!msg_keys[mk_seq] || msg_keys[mk_seq]->int_value() < 0)) { incoming_error(e, "seq is required for single"); return; diff --git a/src/twister.cpp b/src/twister.cpp index 63be0eb3..f4b64138 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -41,6 +41,7 @@ static map m_dhtgetMap; static CCriticalSection cs_twister; static map m_specialResources; +static map m_noExpireResources; // bool is true if expected number after resource string static map m_userTorrent; static std::string m_preferredSpamLang = "[en]"; @@ -389,6 +390,11 @@ void startSessionTorrent(boost::thread_group& threadGroup) m_specialResources["tracker"] = true; m_specialResources["swarm"] = true; + // these are the resources which shouldn't expire (true when numbering is expected) + m_noExpireResources["avatar"] = false; + m_noExpireResources["profile"] = false; + m_noExpireResources["following"] = true; + threadGroup.create_thread(boost::bind(&ThreadWaitExtIP)); threadGroup.create_thread(boost::bind(&ThreadMaintainDHTNodes)); @@ -713,6 +719,15 @@ bool validatePostNumberForUser(std::string const &username, int k) return true; } +bool usernameExists(std::string const &username) +{ + CTransaction txOut; + uint256 hashBlock; + uint256 userhash = SerializeHash(username); + return GetTransaction(userhash, txOut, hashBlock); +} + + /* "userpost" : { @@ -808,6 +823,43 @@ int getBestHeight() return nBestHeight; } +bool shouldDhtResourceExpire(std::string resource, bool multi, int height) +{ + if ((height + BLOCK_AGE_TO_EXPIRE_DHT_ENTRY) < getBestHeight() ) { + if( multi ) { + printf("shouldDhtResourceExpire: expiring resource multi '%s'\n", resource.c_str()); + return true; + } + + std::string resourceBasic; + for(size_t i = 0; i < resource.size() && isalpha(resource.at(i)); i++) { + resourceBasic.push_back(resource.at(i)); + } + int resourceNumber = -1; + + if( resource.length() > resourceBasic.length() ) { + resourceNumber = atoi( resource.c_str() + resourceBasic.length() ); + } + + if( !m_noExpireResources.count(resourceBasic) ) { + // unknown resource. expire it. + printf("shouldDhtResourceExpire: expiring non-special resource '%s'\n", resource.c_str()); + } else { + if( !m_noExpireResources[resourceBasic] && resourceNumber >= 0 ) { + // this resource admits no number. expire it! + printf("shouldDhtResourceExpire: expiring resource with unexpected numbering '%s'\n", resource.c_str()); + return true; + } + if( m_noExpireResources[resourceBasic] && resourceNumber > 200 ) { + // try keeping a sane number here, otherwise expire it! + printf("shouldDhtResourceExpire: expiring resource with numbering too big '%s'\n", resource.c_str()); + return true; + } + } + } + return false; +} + void receivedSpamMessage(std::string const &message, std::string const &user) { LOCK(cs_twister); diff --git a/src/twister.h b/src/twister.h index 2b858891..ecd82053 100644 --- a/src/twister.h +++ b/src/twister.h @@ -10,6 +10,7 @@ #define USERPOST_FLAG_RT 0x01 #define USERPOST_FLAG_DM 0x02 +#define BLOCK_AGE_TO_EXPIRE_DHT_ENTRY 2000 class twister { @@ -26,9 +27,11 @@ bool verifySignature(std::string const &strMessage, std::string const &strUserna bool acceptSignedPost(char const *data, int data_size, std::string username, int seq, std::string &errmsg, boost::uint32_t *flags); bool validatePostNumberForUser(std::string const &username, int k); +bool usernameExists(std::string const &username); void receivedSpamMessage(std::string const &message, std::string const &user); int getBestHeight(); +bool shouldDhtResourceExpire(std::string resource, bool multi, int height); #endif // TWISTER_H From a19c4f231f5b51276e24f306e830bdba047e67bf Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Sun, 20 Oct 2013 14:02:19 -0200 Subject: [PATCH 02/17] return false when packet not sent to ourselves --- libtorrent/src/kademlia/find_data.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libtorrent/src/kademlia/find_data.cpp b/libtorrent/src/kademlia/find_data.cpp index a0609398..2bcc08d1 100644 --- a/libtorrent/src/kademlia/find_data.cpp +++ b/libtorrent/src/kademlia/find_data.cpp @@ -208,7 +208,7 @@ bool find_data::invoke(observer_ptr o) // im not going to ask trackers from myself if( o->id() == m_node.nid() ) - return true; + return false; entry e; e["z"] = "q"; From d9c98f3a731e6a00f10fb3fbe88b45ec2a385e63 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Sun, 20 Oct 2013 14:41:55 -0200 Subject: [PATCH 03/17] add some mime types to the lame web server --- src/bitcoinrpc.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/bitcoinrpc.cpp b/src/bitcoinrpc.cpp index 14660750..272b6ca8 100644 --- a/src/bitcoinrpc.cpp +++ b/src/bitcoinrpc.cpp @@ -979,11 +979,23 @@ void ServiceConnection(AcceptedConnection *conn) std::vector file_data; if( load_file(strURI.c_str(), file_data) == 0 ) { std::string str(file_data.data(), file_data.size()); - conn->stream() << HTTPReply(HTTP_OK, str, false, "text/html") << std::flush; + const char *contentType = "text/html"; + if( strURI.find(".js") != std::string::npos ) + contentType = "text/javascript"; + if( strURI.find(".css") != std::string::npos ) + contentType = "text/css"; + if( strURI.find(".png") != std::string::npos ) + contentType = "image/png"; + if( strURI.find(".ttf") != std::string::npos ) + contentType = "application/x-font-ttf"; + if( strURI.find(".jpg") != std::string::npos || + strURI.find(".jpeg") != std::string::npos ) + contentType = "image/jpeg"; + conn->stream() << HTTPReply(HTTP_OK, str, false, contentType) << std::flush; } else { conn->stream() << HTTPReply(HTTP_NOT_FOUND, "", false) << std::flush; } - break; + continue; } // Check authorization From 6cb53646050d797a68569596690d6edb02ffdd37 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Tue, 22 Oct 2013 18:31:54 -0200 Subject: [PATCH 04/17] disable printf of dm's --- src/twister.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/twister.cpp b/src/twister.cpp index f4b64138..9c4eb700 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -585,9 +585,11 @@ bool processReceivedDM(lazy_entry const* post) } else { std::string textOut; if( key.Decrypt(sec, textOut) ) { + /* this printf is good for debug, but bad for security. printf("Received DM for user '%s' text = '%s'\n", item.second.username.c_str(), textOut.c_str()); + */ std::string n = post->dict_find_string_value("n"); From 0150dd2f7b7556bd39d6293720f204ba6dad2266 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Tue, 22 Oct 2013 18:32:04 -0200 Subject: [PATCH 05/17] support to disable loading/refreshing of expired dht keys --- libtorrent/src/kademlia/node.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/libtorrent/src/kademlia/node.cpp b/libtorrent/src/kademlia/node.cpp index f6d28f41..4675c92f 100644 --- a/libtorrent/src/kademlia/node.cpp +++ b/libtorrent/src/kademlia/node.cpp @@ -55,6 +55,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/rsa.hpp" #include "../../src/twister.h" +//#define ENABLE_DHT_ITEM_EXPIRE namespace libtorrent { namespace dht { @@ -502,6 +503,12 @@ bool node_impl::refresh_storage() { if( lsto.size() == 1 ) { dht_storage_item const& item = lsto.front(); +#ifdef ENABLE_DHT_ITEM_EXPIRE + if( has_expired(item) ) { + continue; + } +#endif + lazy_entry p; int pos; error_code err; @@ -638,9 +645,14 @@ void node_impl::load_storage(entry const* e) { item.sig_user = j->find_key("sig_user")->string(); // just for printf for now - has_expired(item); - - to_add.push_back(item); + bool expired = has_expired(item); +#ifdef ENABLE_DHT_ITEM_EXPIRE + if( !expired ) { +#endif + to_add.push_back(item); +#ifdef ENABLE_DHT_ITEM_EXPIRE + } +#endif } m_storage_table.insert(std::make_pair(target, to_add)); } From 21faab36b76f7bbab8c5c0771de7847c9540ffa9 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Tue, 22 Oct 2013 19:47:50 -0200 Subject: [PATCH 06/17] inform the user that database will be rebuilt. without UI, the old message was confusing. --- src/init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/init.cpp b/src/init.cpp index 01af14e6..96302dc9 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -738,7 +738,7 @@ bool AppInit2(boost::thread_group& threadGroup) // first suggest a reindex if (!fReset) { /*bool fRet =*/ uiInterface.ThreadSafeMessageBox( - strLoadError + ".\n\n" + _("Do you want to rebuild the block database now?"), + strLoadError + ".\n\n" + _("Do you want to rebuild the block database now? (assuming YES)"), "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (true /* [MF] fRet*/) { fReindex = true; From 552b7b542d8c7c774f048bff85fd7af0a555814f Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Tue, 22 Oct 2013 19:51:18 -0200 Subject: [PATCH 07/17] persist some global config. now we implement a spam policy max 1 every 8 hours (not cumulative). --- src/twister.cpp | 88 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 11 deletions(-) diff --git a/src/twister.cpp b/src/twister.cpp index 9c4eb700..99d9ac09 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -31,6 +31,7 @@ twister::twister() #include "libtorrent/aux_/session_impl.hpp" #define DEBUG_ACCEPT_POST 1 +#define DEBUG_EXPIRE_DHT_ITEM 1 using namespace libtorrent; static session *ses = NULL; @@ -47,8 +48,12 @@ static map m_userTorrent; static std::string m_preferredSpamLang = "[en]"; static std::string m_receivedSpamMsgStr; static std::string m_receivedSpamUserStr; +static int64 m_lastSpamTime = 0; static std::map m_users; +#define USER_DATA_FILE "user_data" +#define GLOBAL_DATA_FILE "global_data" + sha1_hash dhtTargetHash(std::string const &username, std::string const &resource, std::string const &type) { entry target; @@ -96,6 +101,47 @@ int lastPostKfromTorrent(std::string const &username) return status.last_have; } +int saveGlobalData(std::string const& filename) +{ + LOCK(cs_twister); + entry globalDict; + + globalDict["preferredSpamLang"] = m_preferredSpamLang; + globalDict["receivedSpamMsg"] = m_receivedSpamMsgStr; + globalDict["receivedSpamUser"] = m_receivedSpamUserStr; + globalDict["lastSpamTime"] = m_lastSpamTime; + + std::vector buf; + bencode(std::back_inserter(buf), globalDict); + return save_file(filename, buf); +} + +int loadGlobalData(std::string const& filename) +{ + LOCK(cs_twister); + std::vector in; + if (load_file(filename, in) == 0) { + lazy_entry userDict; + error_code ec; + if (lazy_bdecode(&in[0], &in[0] + in.size(), userDict, ec) == 0) { + if( userDict.type() != lazy_entry::dict_t ) goto data_error; + + m_preferredSpamLang = userDict.dict_find_string_value("preferredSpamLang"); + m_receivedSpamMsgStr = userDict.dict_find_string_value("receivedSpamMsg"); + m_receivedSpamUserStr = userDict.dict_find_string_value("receivedSpamUser"); + m_lastSpamTime = userDict.dict_find_int_value("lastSpamTime"); + + return 0; + } + } + return -1; + +data_error: + printf("loadGlobalData: unexpected bencode type - global_data corrupt!\n"); + return -2; +} + + void ThreadWaitExtIP() { RenameThread("wait-extip"); @@ -172,9 +218,12 @@ void ThreadWaitExtIP() break; } + boost::filesystem::path globalDataPath = GetDataDir() / GLOBAL_DATA_FILE; + loadGlobalData(globalDataPath.string()); + { LOCK(cs_twister); - boost::filesystem::path userDataPath = GetDataDir() / "user_data"; + boost::filesystem::path userDataPath = GetDataDir() / USER_DATA_FILE; loadUserData(userDataPath.string(), m_users); printf("loaded user_data for %zd users\n", m_users.size()); @@ -448,18 +497,21 @@ void stopSessionTorrent() entry session_state; ses->save_state(session_state); - std::vector out; - bencode(std::back_inserter(out), session_state); - boost::filesystem::path sesStatePath = GetDataDir() / "ses_state"; - save_file(sesStatePath.string(), out); + std::vector out; + bencode(std::back_inserter(out), session_state); + boost::filesystem::path sesStatePath = GetDataDir() / "ses_state"; + save_file(sesStatePath.string(), out); - delete ses; - ses = NULL; + delete ses; + ses = NULL; } + boost::filesystem::path globalDataPath = GetDataDir() / GLOBAL_DATA_FILE; + saveGlobalData(globalDataPath.string()); + if( m_users.size() ) { printf("saving user_data (followers and DMs)...\n"); - boost::filesystem::path userDataPath = GetDataDir() / "user_data"; + boost::filesystem::path userDataPath = GetDataDir() / USER_DATA_FILE; saveUserData(userDataPath.string(), m_users); } @@ -829,7 +881,9 @@ bool shouldDhtResourceExpire(std::string resource, bool multi, int height) { if ((height + BLOCK_AGE_TO_EXPIRE_DHT_ENTRY) < getBestHeight() ) { if( multi ) { +#ifdef DEBUG_EXPIRE_DHT_ITEM printf("shouldDhtResourceExpire: expiring resource multi '%s'\n", resource.c_str()); +#endif return true; } @@ -849,12 +903,16 @@ bool shouldDhtResourceExpire(std::string resource, bool multi, int height) } else { if( !m_noExpireResources[resourceBasic] && resourceNumber >= 0 ) { // this resource admits no number. expire it! +#ifdef DEBUG_EXPIRE_DHT_ITEM printf("shouldDhtResourceExpire: expiring resource with unexpected numbering '%s'\n", resource.c_str()); +#endif return true; } if( m_noExpireResources[resourceBasic] && resourceNumber > 200 ) { // try keeping a sane number here, otherwise expire it! +#ifdef DEBUG_EXPIRE_DHT_ITEM printf("shouldDhtResourceExpire: expiring resource with numbering too big '%s'\n", resource.c_str()); +#endif return true; } } @@ -865,8 +923,13 @@ bool shouldDhtResourceExpire(std::string resource, bool multi, int height) void receivedSpamMessage(std::string const &message, std::string const &user) { LOCK(cs_twister); - if( !m_receivedSpamMsgStr.length() || - (m_preferredSpamLang.length() && message.find(m_preferredSpamLang) != string::npos) ) { + bool hasSingleLangCode = (message.find("[") == message.rfind("[")); + bool hasPreferredLang = m_preferredSpamLang.length(); + bool isSameLang = hasPreferredLang && hasSingleLangCode && + message.find(m_preferredSpamLang) != string::npos; + bool currentlyEmpty = !m_receivedSpamMsgStr.length(); + + if( currentlyEmpty || (isSameLang && rand() < (RAND_MAX/2)) ) { m_receivedSpamMsgStr = message; m_receivedSpamUserStr = user; } @@ -1196,7 +1259,10 @@ Value getposts(const Array& params, bool fHelp) LOCK(cs_twister); if( m_receivedSpamMsgStr.length() ) { // we must agree on an acceptable level here - if( rand() < (RAND_MAX/10) ) { + // what about one every eight hours? (not cumulative) + if( GetAdjustedTime() > m_lastSpamTime + (8*3600) ) { + m_lastSpamTime = GetAdjustedTime(); + entry v; entry &userpost = v["userpost"]; From 55b3f628728b3457d82e1910efcd62a97ac638c6 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Tue, 22 Oct 2013 19:55:23 -0200 Subject: [PATCH 08/17] fix previous commit: don't clear spammsg if it is not consumed. --- src/twister.cpp | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/twister.cpp b/src/twister.cpp index 99d9ac09..03fa8d56 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -1257,27 +1257,26 @@ Value getposts(const Array& params, bool fHelp) { LOCK(cs_twister); - if( m_receivedSpamMsgStr.length() ) { - // we must agree on an acceptable level here - // what about one every eight hours? (not cumulative) - if( GetAdjustedTime() > m_lastSpamTime + (8*3600) ) { - m_lastSpamTime = GetAdjustedTime(); - - entry v; - entry &userpost = v["userpost"]; - - userpost["n"] = m_receivedSpamUserStr; - userpost["k"] = 1; - userpost["time"] = GetAdjustedTime(); - userpost["height"] = getBestHeight(); - - userpost["msg"] = m_receivedSpamMsgStr; - - unsigned char vchSig[65]; - RAND_bytes(vchSig,sizeof(vchSig)); - v["sig_userpost"] = std::string((const char *)vchSig, sizeof(vchSig)); - ret.insert(ret.begin(),entryToJson(v)); - } + // we must agree on an acceptable level here + // what about one every eight hours? (not cumulative) + if( m_receivedSpamMsgStr.length() && GetAdjustedTime() > m_lastSpamTime + (8*3600) ) { + m_lastSpamTime = GetAdjustedTime(); + + entry v; + entry &userpost = v["userpost"]; + + userpost["n"] = m_receivedSpamUserStr; + userpost["k"] = 1; + userpost["time"] = GetAdjustedTime(); + userpost["height"] = getBestHeight(); + + userpost["msg"] = m_receivedSpamMsgStr; + + unsigned char vchSig[65]; + RAND_bytes(vchSig,sizeof(vchSig)); + v["sig_userpost"] = std::string((const char *)vchSig, sizeof(vchSig)); + ret.insert(ret.begin(),entryToJson(v)); + m_receivedSpamMsgStr = ""; m_receivedSpamUserStr = ""; } From 39e8ed8fb716f3a1c96212c356526d518530ae31 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Thu, 24 Oct 2013 23:32:20 -0200 Subject: [PATCH 09/17] increase number of rpc workers --- src/bitcoinrpc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bitcoinrpc.cpp b/src/bitcoinrpc.cpp index 272b6ca8..0e19356c 100644 --- a/src/bitcoinrpc.cpp +++ b/src/bitcoinrpc.cpp @@ -848,7 +848,7 @@ void StartRPCThreads() } rpc_worker_group = new boost::thread_group(); - for (int i = 0; i < GetArg("-rpcthreads", 4); i++) + for (int i = 0; i < GetArg("-rpcthreads", 10); i++) rpc_worker_group->create_thread(boost::bind(&asio::io_service::run, rpc_io_service)); } From d14dca10e13305e46c99ab996ef046eaa9a25f0e Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Fri, 25 Oct 2013 23:44:08 -0200 Subject: [PATCH 10/17] disable libtorrent read cache cause i've messed up something --- src/twister.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/twister.cpp b/src/twister.cpp index 03fa8d56..9ca6f429 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -206,6 +206,9 @@ void ThreadWaitExtIP() //settings.dht_announce_interval = 60; // test //settings.min_announce_interval = 60; // test settings.anonymous_mode = false; // (false => send peer_id, avoid connecting to itself) + // disable read cache => there is still some bug due to twister piece size changes + settings.use_read_cache = false; + settings.cache_size = 0; ses->set_settings(settings); printf("libtorrent + dht started\n"); From c0ec1f4afe673d47e917791b5111a2814e6f2313 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Sat, 26 Oct 2013 00:43:32 -0200 Subject: [PATCH 11/17] Until old public key is properly used, disable banning torrent peers due to bad piece hashes. +note: torrent.cpp line 3286 (function piece_failed), iteration to ban peers is disabled (continue) --- TODO | 5 +++++ libtorrent/src/torrent.cpp | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/TODO b/TODO index 8760c32b..5b79b76a 100644 --- a/TODO +++ b/TODO @@ -8,6 +8,9 @@ pseudocode: while( h > max_h ) getTxIndex( "userX_h" ) => block h contains the previous tx +- Until old public key is properly used, disable banning torrent peers due to bad piece hashes. +note: torrent.cpp line 3286 (function piece_failed), iteration to ban peers is disabled (continue). + - Count UTF8 chars in acceptSignedPost to proper limit the 140 characters. - Encrypt user_data (which contains all DMs) @@ -52,5 +55,7 @@ policy may me defined. - Check stored dht values if their signature is still valid before trying to refresh another node. Key pair might have changed and currently we receive a lot of errors from other nodes. +- save_file() must truncate file. +- diff --git a/libtorrent/src/torrent.cpp b/libtorrent/src/torrent.cpp index f6499a5e..dde7b2aa 100644 --- a/libtorrent/src/torrent.cpp +++ b/libtorrent/src/torrent.cpp @@ -3283,6 +3283,9 @@ namespace libtorrent for (std::set::iterator i = peers.begin() , end(peers.end()); i != end; ++i) { + // [MF] FIXME FIXME: BANNING BY FAILED HASH DISABLED - READ TODO! + continue; + policy::peer* p = static_cast(*i); if (p == 0) continue; TORRENT_ASSERT(p->in_use); @@ -3337,7 +3340,7 @@ namespace libtorrent if (p->connection) { -#ifdef TORRENT_LOGGING +#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING debug_log("*** BANNING PEER: \"%s\" Too many corrupt pieces" , print_endpoint(p->ip()).c_str()); #endif From 66fb23c4b87d7c3ba3b9d8e1464179697d262da0 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Sat, 26 Oct 2013 13:08:37 -0200 Subject: [PATCH 12/17] try to fix automatically another block db inconsistency problem --- src/init.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/init.cpp b/src/init.cpp index 96302dc9..dccbda3b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -726,6 +726,11 @@ bool AppInit2(boost::thread_group& threadGroup) strLoadError = _("Corrupted block database detected"); break; } + + if( mapBlockIndex.size() > 1000 && nBestHeight == 0 ) { + strLoadError = _("mapBlockIndex detected but nBestHeight still zero, trying to repair (reindex)"); + break; + } } catch(std::exception &e) { strLoadError = _("Error opening block database"); break; From 2c6318caae947ffdc16442561458b6d4be1a365d Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Sat, 26 Oct 2013 13:09:20 -0200 Subject: [PATCH 13/17] less noisy in debug --- src/net.cpp | 6 +++--- src/netbase.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 1bb14446..968d5dd4 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -462,9 +462,9 @@ CNode* ConnectNode(CAddress addrConnect, const char *pszDest) /// debug print - printf("trying connection %s lastseen=%.1fhrs\n", - pszDest ? pszDest : addrConnect.ToString().c_str(), - pszDest ? 0 : (double)(GetAdjustedTime() - addrConnect.nTime)/3600.0); +// printf("trying connection %s lastseen=%.1fhrs\n", +// pszDest ? pszDest : addrConnect.ToString().c_str(), +// pszDest ? 0 : (double)(GetAdjustedTime() - addrConnect.nTime)/3600.0); // Connect SOCKET hSocket; diff --git a/src/netbase.cpp b/src/netbase.cpp index 08d13374..8708316c 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -359,7 +359,7 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe int nRet = select(hSocket + 1, NULL, &fdset, NULL, &timeout); if (nRet == 0) { - printf("connection timeout\n"); + //printf("connection timeout\n"); closesocket(hSocket); return false; } @@ -393,7 +393,7 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe else #endif { - printf("connect() failed: %i\n",WSAGetLastError()); + //printf("connect() failed: %i\n",WSAGetLastError()); closesocket(hSocket); return false; } From 604600234105b369aeb042b03690bb2368760987 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Sat, 26 Oct 2013 13:10:04 -0200 Subject: [PATCH 14/17] some docs, todos --- README.md | 5 +++++ TODO | 9 ++++++++- twister-test.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6549519b..80d8665f 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,11 @@ Developers of either bitcoin or libtorrent are welcomed and will be granted immediate write-access to the repository (a small retribution for bastardizing their codebases). +Compiling +--------- + +See `INSTALL`. + Testing ------- diff --git a/TODO b/TODO index 5b79b76a..e730e042 100644 --- a/TODO +++ b/TODO @@ -39,6 +39,7 @@ merkle tree inside that block. This resource propagation cannot be sent right af registration for obvious reasons (no block yet, other nodes wouldn't accept the signed dht put). - Discuss and implement the acceptable level of spam per day (priorizing localization). +DONE (except for the discussion part...) - Implement the mention forwarding mechanism discussed in the paper so user don't need to do polling and can also be sure to receive all mentions. @@ -57,5 +58,11 @@ Key pair might have changed and currently we receive a lot of errors from other - save_file() must truncate file. -- +- Save lastk field to post so torrent-less navigation through posts is possible. + +- Implement dht-to-torrent gateway, the "swarm" resource (so poster may not need to be member +of his own torrent) + +- Estimate number of online followers by quering the "tracker" resource (implement a value within +this resource to report the number of torrent peers) diff --git a/twister-test.py b/twister-test.py index 3663b14c..377fc834 100755 --- a/twister-test.py +++ b/twister-test.py @@ -3,7 +3,7 @@ import os,sys,time ext_ip = os.environ['EXTIP'] -twister = "../twister-qt-build-desktop/twisterd" +twister = "./twisterd" cmd = sys.argv[1] n = int(sys.argv[2]) From 3e64454b7d85c5f2834358880e18b520017c373a Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Sat, 26 Oct 2013 13:42:50 -0200 Subject: [PATCH 15/17] Save lastk field to post so torrent-less navigation through posts is possible. => DONE --- TODO | 2 +- src/twister.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/TODO b/TODO index e730e042..2a912581 100644 --- a/TODO +++ b/TODO @@ -58,7 +58,7 @@ Key pair might have changed and currently we receive a lot of errors from other - save_file() must truncate file. -- Save lastk field to post so torrent-less navigation through posts is possible. +- Save lastk field to post so torrent-less navigation through posts is possible. => DONE - Implement dht-to-torrent gateway, the "swarm" resource (so poster may not need to be member of his own torrent) diff --git a/src/twister.cpp b/src/twister.cpp index 9ca6f429..c4b6b598 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -1024,6 +1024,34 @@ Value dhtget(const Array& params, bool fHelp) return ret; } +int findLastPublicPostLocalUser( std::string strUsername ) +{ + int lastk = -1; + + LOCK(cs_twister); + if( strUsername.size() && m_userTorrent.count(strUsername) && + m_userTorrent[strUsername].is_valid() ){ + + std::vector pieces; + int max_id = std::numeric_limits::max(); + int since_id = -1; + m_userTorrent[strUsername].get_pieces(pieces, 1, max_id, since_id, USERPOST_FLAG_RT); + + if( pieces.size() ) { + string const& piece = pieces.front(); + lazy_entry v; + int pos; + error_code ec; + if (lazy_bdecode(piece.data(), piece.data()+piece.size(), v, ec, &pos) == 0) { + lazy_entry const* post = v.dict_find_dict("userpost"); + lastk = post->dict_find_int_value("k",-1); + } + } + } + return lastk; +} + + Value newpostmsg(const Array& params, bool fHelp) { if (fHelp || (params.size() != 3 && params.size() != 5)) @@ -1047,6 +1075,11 @@ Value newpostmsg(const Array& params, bool fHelp) } entry v; + // [MF] Warning: findLastPublicPostLocalUser requires that we follow ourselves + int lastk = findLastPublicPostLocalUser(strUsername); + if( lastk >= 0 ) + v["userpost"]["lastk"] = lastk; + if( !createSignedUserpost(v, strUsername, k, strMsg, NULL, NULL, NULL, strReplyN, replyK) ) @@ -1166,6 +1199,11 @@ Value newrtmsg(const Array& params, bool fHelp) entry const *sig_rt= vrt.find_key("sig_userpost"); entry v; + // [MF] Warning: findLastPublicPostLocalUser requires that we follow ourselves + int lastk = findLastPublicPostLocalUser(strUsername); + if( lastk >= 0 ) + v["userpost"]["lastk"] = lastk; + if( !createSignedUserpost(v, strUsername, k, "", rt, sig_rt, NULL, std::string(""), 0) ) From 1d78f7ec27ccea375b4d528c71b66aa4451bf22d Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Sat, 26 Oct 2013 16:00:16 -0200 Subject: [PATCH 16/17] improve dht resource expiration policy (still not enabled) --- src/twister.cpp | 65 +++++++++++++++++++++++++++++++++++++++++-------- src/twister.h | 4 ++- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/src/twister.cpp b/src/twister.cpp index c4b6b598..f841d5d8 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -42,7 +42,8 @@ static map m_dhtgetMap; static CCriticalSection cs_twister; static map m_specialResources; -static map m_noExpireResources; // bool is true if expected number after resource string +enum ExpireResType { SimpleNoExpire, NumberedNoExpire, PostNoExpireRecent }; +static map m_noExpireResources; static map m_userTorrent; static std::string m_preferredSpamLang = "[en]"; @@ -442,10 +443,22 @@ void startSessionTorrent(boost::thread_group& threadGroup) m_specialResources["tracker"] = true; m_specialResources["swarm"] = true; - // these are the resources which shouldn't expire (true when numbering is expected) - m_noExpireResources["avatar"] = false; - m_noExpireResources["profile"] = false; - m_noExpireResources["following"] = true; + // these are the resources which shouldn't expire + m_noExpireResources["avatar"] = SimpleNoExpire; + m_noExpireResources["profile"] = SimpleNoExpire; + m_noExpireResources["following"] = NumberedNoExpire; + m_noExpireResources["status"] = SimpleNoExpire; + m_noExpireResources["post"] = PostNoExpireRecent; + + + shouldDhtResourceExpire("post", false, 0); + shouldDhtResourceExpire("post0", false, 0); + shouldDhtResourceExpire("post01", false, 0); + shouldDhtResourceExpire("post1 ", false, 0); + shouldDhtResourceExpire("post1233", false, 0); + shouldDhtResourceExpire("avatar", false, 0); + shouldDhtResourceExpire("following", false, 0); + shouldDhtResourceExpire("following300", false, 0); threadGroup.create_thread(boost::bind(&ThreadWaitExtIP)); @@ -890,31 +903,63 @@ bool shouldDhtResourceExpire(std::string resource, bool multi, int height) return true; } + // extract basic resource string (without numbering) std::string resourceBasic; for(size_t i = 0; i < resource.size() && isalpha(resource.at(i)); i++) { resourceBasic.push_back(resource.at(i)); } - int resourceNumber = -1; + int resourceNumber = -1; if( resource.length() > resourceBasic.length() ) { - resourceNumber = atoi( resource.c_str() + resourceBasic.length() ); + // make sure it is a valid number following (all digits) + if( resource.at(resourceBasic.length()) == '0' && + resource.size() > resourceBasic.length() + 1 ){ + // leading zeros not allowed + } else { + size_t i; + for(i = resourceBasic.length(); i < resource.size() && + isdigit(resource.at(i)); i++) { + } + if(i == resource.size()) { + resourceNumber = atoi( resource.c_str() + resourceBasic.length() ); + printf("shouldDhtResourceExpire: %s = %s + %d\n", resource.c_str(), resourceBasic.c_str(), resourceNumber); + } + } } if( !m_noExpireResources.count(resourceBasic) ) { // unknown resource. expire it. +#ifdef DEBUG_EXPIRE_DHT_ITEM printf("shouldDhtResourceExpire: expiring non-special resource '%s'\n", resource.c_str()); +#endif + return true; } else { - if( !m_noExpireResources[resourceBasic] && resourceNumber >= 0 ) { + if( m_noExpireResources[resourceBasic] == SimpleNoExpire && + resource.length() > resourceBasic.length() ) { // this resource admits no number. expire it! #ifdef DEBUG_EXPIRE_DHT_ITEM printf("shouldDhtResourceExpire: expiring resource with unexpected numbering '%s'\n", resource.c_str()); #endif return true; } - if( m_noExpireResources[resourceBasic] && resourceNumber > 200 ) { + if( m_noExpireResources[resourceBasic] == NumberedNoExpire && + (resourceNumber < 0 || resourceNumber > 200) ) { // try keeping a sane number here, otherwise expire it! #ifdef DEBUG_EXPIRE_DHT_ITEM - printf("shouldDhtResourceExpire: expiring resource with numbering too big '%s'\n", resource.c_str()); + printf("shouldDhtResourceExpire: expiring numbered resource with no sane number '%s'\n", resource.c_str()); +#endif + return true; + } + if( m_noExpireResources[resourceBasic] == PostNoExpireRecent && resourceNumber < 0 ) { +#ifdef DEBUG_EXPIRE_DHT_ITEM + printf("shouldDhtResourceExpire: expiring post with invalid numbering '%s'\n", resource.c_str()); +#endif + return true; + } + if( m_noExpireResources[resourceBasic] == PostNoExpireRecent && + (height + BLOCK_AGE_TO_EXPIRE_DHT_POSTS) < getBestHeight() ) { +#ifdef DEBUG_EXPIRE_DHT_ITEM + printf("shouldDhtResourceExpire: expiring old post resource '%s'\n", resource.c_str()); #endif return true; } diff --git a/src/twister.h b/src/twister.h index ecd82053..38233776 100644 --- a/src/twister.h +++ b/src/twister.h @@ -10,7 +10,9 @@ #define USERPOST_FLAG_RT 0x01 #define USERPOST_FLAG_DM 0x02 -#define BLOCK_AGE_TO_EXPIRE_DHT_ENTRY 2000 +#define BLOCK_AGE_TO_EXPIRE_DHT_ENTRY (2016) // about 2 weeks +#define BLOCK_AGE_TO_EXPIRE_DHT_POSTS (4320*6) // about 6 months + class twister { From a7f7b45976163171c94cf57c86e2ab9b9506b773 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Sat, 26 Oct 2013 16:16:07 -0200 Subject: [PATCH 17/17] remove test code --- src/twister.cpp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/twister.cpp b/src/twister.cpp index f841d5d8..e7742069 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -450,17 +450,6 @@ void startSessionTorrent(boost::thread_group& threadGroup) m_noExpireResources["status"] = SimpleNoExpire; m_noExpireResources["post"] = PostNoExpireRecent; - - shouldDhtResourceExpire("post", false, 0); - shouldDhtResourceExpire("post0", false, 0); - shouldDhtResourceExpire("post01", false, 0); - shouldDhtResourceExpire("post1 ", false, 0); - shouldDhtResourceExpire("post1233", false, 0); - shouldDhtResourceExpire("avatar", false, 0); - shouldDhtResourceExpire("following", false, 0); - shouldDhtResourceExpire("following300", false, 0); - - threadGroup.create_thread(boost::bind(&ThreadWaitExtIP)); threadGroup.create_thread(boost::bind(&ThreadMaintainDHTNodes)); threadGroup.create_thread(boost::bind(&ThreadSessionAlerts)); @@ -922,7 +911,6 @@ bool shouldDhtResourceExpire(std::string resource, bool multi, int height) } if(i == resource.size()) { resourceNumber = atoi( resource.c_str() + resourceBasic.length() ); - printf("shouldDhtResourceExpire: %s = %s + %d\n", resource.c_str(), resourceBasic.c_str(), resourceNumber); } } }