From ab48764e386972bcc5a04a1fed74c21d5441575d Mon Sep 17 00:00:00 2001 From: Denis Ryabov Date: Wed, 27 Aug 2014 01:03:37 +0400 Subject: [PATCH] add search method --- .../include/libtorrent/aux_/session_impl.hpp | 1 + libtorrent/include/libtorrent/session.hpp | 1 + libtorrent/src/session.cpp | 9 + libtorrent/src/session_impl.cpp | 6 + src/bitcoinrpc.cpp | 2 + src/bitcoinrpc.h | 1 + src/twister.cpp | 268 ++++++++++++++++++ 7 files changed, 288 insertions(+) diff --git a/libtorrent/include/libtorrent/aux_/session_impl.hpp b/libtorrent/include/libtorrent/aux_/session_impl.hpp index 1724749b..2aa48989 100644 --- a/libtorrent/include/libtorrent/aux_/session_impl.hpp +++ b/libtorrent/include/libtorrent/aux_/session_impl.hpp @@ -311,6 +311,7 @@ namespace libtorrent entry const &p, std::string const &sig_p, std::string const &sig_user, bool local); void dht_getData(std::string const &username, std::string const &resource, bool multi); + entry dht_getLocalData() const; #ifndef TORRENT_NO_DEPRECATE diff --git a/libtorrent/include/libtorrent/session.hpp b/libtorrent/include/libtorrent/session.hpp index 5186bbc2..2b934af4 100644 --- a/libtorrent/include/libtorrent/session.hpp +++ b/libtorrent/include/libtorrent/session.hpp @@ -442,6 +442,7 @@ namespace libtorrent entry const &p, std::string const &sig_p, std::string const &sig_user, bool local); void dht_getData(std::string const &username, std::string const &resource, bool multi); + entry dht_getLocalData() const; #ifndef TORRENT_NO_DEPRECATE // deprecated in 0.15 diff --git a/libtorrent/src/session.cpp b/libtorrent/src/session.cpp index 145019aa..c9374565 100644 --- a/libtorrent/src/session.cpp +++ b/libtorrent/src/session.cpp @@ -869,6 +869,15 @@ namespace libtorrent #endif } + entry session::dht_getLocalData() const + { +#ifndef TORRENT_DISABLE_DHT + return m_impl->dht_getLocalData(); +#else + return entry(); +#endif + } + bool session::is_dht_running() const { #ifndef TORRENT_DISABLE_DHT diff --git a/libtorrent/src/session_impl.cpp b/libtorrent/src/session_impl.cpp index 7a7cc281..7c412258 100644 --- a/libtorrent/src/session_impl.cpp +++ b/libtorrent/src/session_impl.cpp @@ -5821,6 +5821,12 @@ retry: boost::bind( getDataDone_fun, this, username, resource, multi, _1, _2)); } + entry session_impl::dht_getLocalData() const + { + entry state = m_dht->state(); + return state["storage_table"]; + } + void session_impl::on_dht_router_name_lookup(error_code const& e , tcp::resolver::iterator host) { diff --git a/src/bitcoinrpc.cpp b/src/bitcoinrpc.cpp index 8addc2ba..5f7a2225 100644 --- a/src/bitcoinrpc.cpp +++ b/src/bitcoinrpc.cpp @@ -264,6 +264,7 @@ static const CRPCCommand vRPCCommands[] = { "gettrendinghashtags", &gettrendinghashtags, false, true, true }, { "getspamposts", &getspamposts, false, true, false }, { "torrentstatus", &torrentstatus, false, true, false }, + { "search", &search, false, true, false }, }; CRPCTable::CRPCTable() @@ -1314,6 +1315,7 @@ Array RPCConvertValues(const std::string &strMethod, const std::vector 0) ConvertTo(params[0]); if (strMethod == "getspamposts" && n > 1) ConvertTo(params[1]); if (strMethod == "getspamposts" && n > 2) ConvertTo(params[2]); + if (strMethod == "search" && n > 2) ConvertTo(params[2]); return params; } diff --git a/src/bitcoinrpc.h b/src/bitcoinrpc.h index b0822e06..ed767cf6 100644 --- a/src/bitcoinrpc.h +++ b/src/bitcoinrpc.h @@ -217,5 +217,6 @@ extern json_spirit::Value recheckusertorrent(const json_spirit::Array& params, b extern json_spirit::Value gettrendinghashtags(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getspamposts(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value torrentstatus(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value search(const json_spirit::Array& params, bool fHelp); #endif diff --git a/src/twister.cpp b/src/twister.cpp index 24a19f21..0d9b40a4 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -2346,3 +2346,271 @@ Value torrentstatus(const Array& params, bool fHelp) return result; } + +Value search(const Array& params, bool fHelp) +{ + if (fHelp || params.size() < 3 || params.size() > 4) + throw runtime_error( + "search []\n" + "search text in known data\n" + " is data area: messages, directmsgs, profiles, users, hashtags\n" + " is a phrase to search\n" + "up to entries are returned\n" + " in messages scope is optional and allows to search in username's messages only\n" + " in directmsgs scope is required and sets whose conversation to search"); + + string scope = params[0].get_str(); + string keyword = params[1].get_str(); + int count = params[2].get_int(); + string username = params.size()==4 ? params[3].get_str() : string(); + + if( keyword.size() == 0 ) { + throw runtime_error("Empty parameter"); + } + + Array ret; + + if( scope == "messages" ) { + // search public messages + std::map< pair, pair > posts; + + // search public messages in torrents + { + LOCK(cs_twister); + + std::map users; + if( username.size() == 0 ) { + users = m_userTorrent; + } else { + if( m_userTorrent.count(username) ) + users[username] = m_userTorrent[username]; + } + + BOOST_FOREACH(const PAIRTYPE(std::string,torrent_handle)& item, users) { + std::vector pieces; + item.second.get_pieces(pieces, std::numeric_limits::max(), std::numeric_limits::max(), -1, ~USERPOST_FLAG_DM); + + BOOST_FOREACH(string const& piece, pieces) { + if( piece.find(keyword) != string::npos ) { + lazy_entry v; + int pos; + libtorrent::error_code ec; + if (lazy_bdecode(piece.data(), piece.data()+piece.size(), v, ec, &pos) == 0) { + lazy_entry const* post = v.dict_find_dict("userpost"); + if( post ) { + lazy_entry const* rt = post->dict_find_dict("rt"); + lazy_entry const* p = rt ? rt : post; + string msg = p->dict_find_string_value("msg"); + if( msg.find(keyword) != string::npos ) { + string n = p->dict_find_string_value("n"); + int k = p->dict_find_int_value("k"); + int64 time = p->dict_find_int_value("time",-1); + + entry vEntry; + vEntry = *p; + hexcapePost(vEntry); + + posts[pair(n,k)] = pair(time,vEntry); + } + } + } + } + } + } + } + + // search messages in dht + boost::shared_ptr ses(m_ses); + if( ses ) + { + entry data = ses->dht_getLocalData(); + + for (entry::dictionary_type::const_iterator i = data.dict().begin(); i != data.dict().end(); ++i) { + if ( i->second.type() != entry::list_t ) + continue; + for (entry::list_type::const_iterator j = i->second.list().begin(); j != i->second.list().end(); ++j) { + string str_p = j->find_key("p")->string(); + if( str_p.find(keyword) != string::npos ) { + lazy_entry p; + int pos; + libtorrent::error_code err; + int ret = lazy_bdecode(str_p.data(), str_p.data() + str_p.size(), p, err, &pos); + + lazy_entry const* v = p.dict_find_dict("v"); + if( v ) { + lazy_entry const* post = v->dict_find_dict("userpost"); + if( post ) { + // post, mention, status + lazy_entry const* rt = post->dict_find_dict("rt"); + lazy_entry const* p = rt ? rt : post; + string msg = p->dict_find_string_value("msg"); + if( msg.find(keyword) != string::npos ) { + string n = p->dict_find_string_value("n"); + + if( username.size() == 0 || n == username ) { + int k = p->dict_find_int_value("k"); + + pair post_id(n,k); + if( posts.count(post_id) == 0 ) { + int64 time = p->dict_find_int_value("time",-1); + + entry vEntry; + vEntry = *p; + hexcapePost(vEntry); + + posts[post_id] = pair(time,vEntry); + } + } + } + } + } + } + } + } + } + + std::multimap postsByTime; + BOOST_FOREACH(const PAIRTYPE(PAIRTYPE(std::string,int),PAIRTYPE(int64,entry))& item, posts) { + postsByTime.insert(item.second); + } + + std::multimap::reverse_iterator rit; + for (rit=postsByTime.rbegin(); rit!=postsByTime.rend() && (int)ret.size() < count; ++rit) { + ret.push_back( entryToJson(rit->second) ); + } + + } else if( scope == "directmsgs" ) { + // search direct messages + if( m_users.count(username) ){ + std::multimap postsByTime; + + { + LOCK(cs_twister); + + BOOST_FOREACH(const PAIRTYPE(std::string,std::vector)& list, m_users[username].m_directmsg) { + string remoteUser = list.first; + BOOST_FOREACH(const StoredDirectMsg& item, list.second) { + if( item.m_text.find(keyword) != string::npos ) { + int64 time = item.m_utcTime; + entry vEntry; + vEntry["remoteUser"] = remoteUser; + vEntry["text"] = item.m_text; + vEntry["time"] = time; + vEntry["fromMe"] = item.m_fromMe; + hexcapePost(vEntry); + postsByTime.insert( pair(time, vEntry) ); + } + } + } + } + + std::multimap::reverse_iterator rit; + for (rit=postsByTime.rbegin(); rit!=postsByTime.rend() && (int)ret.size() < count; ++rit) { + ret.push_back( entryToJson(rit->second) ); + } + } + + } else if( scope == "profiles" ) { + // search dht profiles + boost::shared_ptr ses(m_ses); + if( ses ) + { + entry data = ses->dht_getLocalData(); + std::map users; + + for (entry::dictionary_type::const_iterator i = data.dict().begin(); i != data.dict().end(); ++i) { + if ( i->second.type() != entry::list_t ) + continue; + for (entry::list_type::const_iterator j = i->second.list().begin(); j != i->second.list().end(); ++j) { + string str_p = j->find_key("p")->string(); + if( str_p.find(keyword) != string::npos ) { + lazy_entry p; + int pos; + libtorrent::error_code err; + int ret = lazy_bdecode(str_p.data(), str_p.data() + str_p.size(), p, err, &pos); + + lazy_entry const* target = p.dict_find_dict("target"); + if( target ) { + string resource = target->dict_find_string_value("r"); + if( resource == "profile" ) { + lazy_entry const* v = p.dict_find_dict("v"); + if( v ) { + string bio = v->dict_find_string_value("bio"); + string fullname = v->dict_find_string_value("fullname"); + string location = v->dict_find_string_value("location"); + string url = v->dict_find_string_value("url"); + + if( bio.find(keyword) != string::npos || + fullname.find(keyword) != string::npos || + location.find(keyword) != string::npos || + url.find(keyword) != string::npos ) { + + string n = target->dict_find_string_value("n"); + entry vEntry; + vEntry = *v; + users.insert(pair(n,vEntry)); + } + } + } + } + } + } + } + + std::map::iterator it; + for (it=users.begin(); it!=users.end() && (int)ret.size() < count; ++it) { + entry user; + user["username"] = it->first; + user["profile"] = it->second; + ret.push_back( entryToJson(user) ); + } + } + + } else if( scope == "users" ) { + // search users (blockchain) + // @todo: there should be a faster way + std::multimap usernamesByLength; + + string allowed = "abcdefghijklmnopqrstuvwxyz0123456789_"; + for( int i = 0; i < allowed.size(); ++i ) { + set usernames; + string prefix(1, allowed[i]); + pblocktree->GetNamesFromPartial(prefix, usernames, std::numeric_limits::max()); + + BOOST_FOREACH(string username, usernames) { + if( username.find(keyword) != string::npos ) { + usernamesByLength.insert( pair(username.size(), username) ); + } + } + } + + std::multimap::iterator it; + for (it=usernamesByLength.begin(); it!=usernamesByLength.end() && (int)ret.size() < count; ++it) { + ret.push_back( entryToJson(it->second) ); + } + + } else if( scope == "hashtags" ) { + // search hashtags + std::multimap hashtagsByLength; + + { + LOCK(cs_seenHashtags); + + BOOST_FOREACH(const PAIRTYPE(std::string,double)& item, m_seenHashtags) { + if (item.first.find(keyword) != std::string::npos) { + hashtagsByLength.insert( pair(item.first.size(), item.first) ); + } + } + } + + std::multimap::iterator it; + for (it=hashtagsByLength.begin(); it!=hashtagsByLength.end() && (int)ret.size() < count; ++it) { + ret.push_back( entryToJson(it->second) ); + } + + } else { + throw runtime_error("Unknown value"); + } + + return ret; +}