From 8287b2b1e1b758eaa3b2929267c6c24dbb1f39c8 Mon Sep 17 00:00:00 2001 From: Miguel Freitas Date: Mon, 6 Jan 2014 15:10:32 -0200 Subject: [PATCH] use a single database for all swarms of the same libtorrent session. upgrade resume file to version 2 and enforce on loading (to force existing clients to download all pieces again). fixes #24. --- .../include/libtorrent/aux_/session_impl.hpp | 3 +- libtorrent/include/libtorrent/session.hpp | 10 +- libtorrent/include/libtorrent/storage.hpp | 14 +- .../include/libtorrent/storage_defs.hpp | 7 +- libtorrent/src/create_torrent.cpp | 4 + libtorrent/src/session.cpp | 4 +- libtorrent/src/session_impl.cpp | 3 +- libtorrent/src/storage.cpp | 173 ++---------------- libtorrent/src/torrent.cpp | 17 +- src/twister.cpp | 10 +- 10 files changed, 58 insertions(+), 187 deletions(-) diff --git a/libtorrent/include/libtorrent/aux_/session_impl.hpp b/libtorrent/include/libtorrent/aux_/session_impl.hpp index 2589bc10..e65abb32 100644 --- a/libtorrent/include/libtorrent/aux_/session_impl.hpp +++ b/libtorrent/include/libtorrent/aux_/session_impl.hpp @@ -211,7 +211,7 @@ namespace libtorrent typedef std::set > connection_map; typedef std::map > torrent_map; - session_impl( + session_impl(CLevelDB &swarmDb, std::pair listen_port_range , fingerprint const& cl_fprint , char const* listen_interface @@ -656,6 +656,7 @@ namespace libtorrent // since they will still have references to it // when they are destructed. file_pool m_files; + CLevelDB &m_swarmDb; // this is where all active sockets are stored. // the selector can sleep while there's no activity on diff --git a/libtorrent/include/libtorrent/session.hpp b/libtorrent/include/libtorrent/session.hpp index 77b9969b..73b8551d 100644 --- a/libtorrent/include/libtorrent/session.hpp +++ b/libtorrent/include/libtorrent/session.hpp @@ -135,7 +135,7 @@ namespace libtorrent // them to start, pass 0 as the flags parameter. // // The ``alert_mask`` is the same mask that you would send to `set_alert_mask()`_. - session(fingerprint const& print = fingerprint("LT" + session(CLevelDB &swarmDb, fingerprint const& print = fingerprint("LT" , LIBTORRENT_VERSION_MAJOR, LIBTORRENT_VERSION_MINOR, 0, 0) , int flags = start_default_features | add_default_plugins , boost::uint32_t alert_mask = alert::error_notification @@ -143,13 +143,13 @@ namespace libtorrent TORRENT_LOGPATH_ARG_DEFAULT) { TORRENT_CFG(); - init(listen_range, "0.0.0.0", print, alert_mask, ext_ip); + init(swarmDb, listen_range, "0.0.0.0", print, alert_mask, ext_ip); #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING set_log_path(logpath); #endif start(flags); } - session(fingerprint const& print + session(CLevelDB &swarmDb, fingerprint const& print , std::pair listen_port_range , char const* listen_interface = "0.0.0.0" , int flags = start_default_features | add_default_plugins @@ -160,7 +160,7 @@ namespace libtorrent TORRENT_CFG(); TORRENT_ASSERT(listen_port_range.first > 0); TORRENT_ASSERT(listen_port_range.first < listen_port_range.second); - init(listen_port_range, listen_interface, print, alert_mask, ext_ip); + init(swarmDb, listen_port_range, listen_interface, print, alert_mask, ext_ip); #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING set_log_path(logpath); #endif @@ -859,7 +859,7 @@ namespace libtorrent private: - void init(std::pair listen_range, char const* listen_interface + void init(CLevelDB &swarmDb, std::pair listen_range, char const* listen_interface , fingerprint const& id, boost::uint32_t alert_mask, char const* ext_ip); void set_log_path(std::string const& p); void start(int flags); diff --git a/libtorrent/include/libtorrent/storage.hpp b/libtorrent/include/libtorrent/storage.hpp index 7c3a8d65..a5f503f2 100644 --- a/libtorrent/include/libtorrent/storage.hpp +++ b/libtorrent/include/libtorrent/storage.hpp @@ -178,11 +178,11 @@ namespace libtorrent session_settings* m_settings; }; - class TORRENT_EXPORT default_storage : public storage_interface, boost::noncopyable, CLevelDB + class TORRENT_EXPORT default_storage : public storage_interface, boost::noncopyable { public: default_storage(file_storage const& fs, file_storage const* mapped, std::string const& path - , file_pool& fp, std::vector const& file_prio); + , CLevelDB &db, std::vector const& file_prio); ~default_storage(); #ifndef TORRENT_NO_DEPRECATE @@ -233,11 +233,11 @@ namespace libtorrent , error_code& ec) const; std::vector m_file_priority; - std::string m_save_path; - // the file pool is typically stored in + std::string m_db_path; + // the leveldb is typically stored in // the session, to make all storage - // instances use the same pool - file_pool& m_pool; + // instances use the same database + CLevelDB& m_db; int m_page_size; bool m_allocate_files; @@ -310,7 +310,7 @@ namespace libtorrent boost::shared_ptr const& torrent , boost::intrusive_ptr info , std::string const& path - , file_pool& fp + , CLevelDB &db , disk_io_thread& io , storage_constructor_type sc , storage_mode_t sm diff --git a/libtorrent/include/libtorrent/storage_defs.hpp b/libtorrent/include/libtorrent/storage_defs.hpp index d981c9d3..1b7982ab 100644 --- a/libtorrent/include/libtorrent/storage_defs.hpp +++ b/libtorrent/include/libtorrent/storage_defs.hpp @@ -34,6 +34,7 @@ POSSIBILITY OF SUCH DAMAGE. #define TORRENT_STORAGE_DEFS_HPP_INCLUDE #include "libtorrent/config.hpp" +#include "leveldb.h" #include #include @@ -62,14 +63,14 @@ namespace libtorrent }; typedef boost::function const&)> storage_constructor_type; + , std::string const&, CLevelDB&, std::vector const&)> storage_constructor_type; TORRENT_EXPORT storage_interface* default_storage_constructor( - file_storage const&, file_storage const* mapped, std::string const&, file_pool& + file_storage const&, file_storage const* mapped, std::string const&, CLevelDB& , std::vector const&); TORRENT_EXPORT storage_interface* disabled_storage_constructor( - file_storage const&, file_storage const* mapped, std::string const&, file_pool& + file_storage const&, file_storage const* mapped, std::string const&, CLevelDB& , std::vector const&); } diff --git a/libtorrent/src/create_torrent.cpp b/libtorrent/src/create_torrent.cpp index 5fa8f056..c2bdd8d1 100644 --- a/libtorrent/src/create_torrent.cpp +++ b/libtorrent/src/create_torrent.cpp @@ -160,6 +160,7 @@ namespace libtorrent void set_piece_hashes(create_torrent& t, std::wstring const& p , boost::function const& f, error_code& ec) { + /* file_pool fp; std::string utf8; wchar_utf8(p, utf8); @@ -187,12 +188,14 @@ namespace libtorrent t.set_hash(i, h.final()); f(i); } + */ } #endif void set_piece_hashes(create_torrent& t, std::string const& p , boost::function f, error_code& ec) { + /* file_pool fp; #if TORRENT_USE_UNC_PATHS std::string path = canonicalize_path(p); @@ -254,6 +257,7 @@ namespace libtorrent t.set_hash(i, h.final()); f(i); } + */ } create_torrent::~create_torrent() {} diff --git a/libtorrent/src/session.cpp b/libtorrent/src/session.cpp index 196d452a..53edf214 100644 --- a/libtorrent/src/session.cpp +++ b/libtorrent/src/session.cpp @@ -403,7 +403,7 @@ namespace libtorrent { throw; } #endif - void session::init(std::pair listen_range, char const* listen_interface + void session::init(CLevelDB &swarmDb, std::pair listen_range, char const* listen_interface , fingerprint const& id, boost::uint32_t alert_mask, char const* ext_ip) { #if defined _MSC_VER && defined TORRENT_DEBUG @@ -413,7 +413,7 @@ namespace libtorrent ::_set_se_translator(straight_to_debugger); #endif - m_impl.reset(new session_impl(listen_range, id, listen_interface, alert_mask, ext_ip)); + m_impl.reset(new session_impl(swarmDb, listen_range, id, listen_interface, alert_mask, ext_ip)); #ifdef TORRENT_MEMDEBUG start_malloc_debug(); diff --git a/libtorrent/src/session_impl.cpp b/libtorrent/src/session_impl.cpp index b3feb0fc..d47c5b2b 100644 --- a/libtorrent/src/session_impl.cpp +++ b/libtorrent/src/session_impl.cpp @@ -594,7 +594,7 @@ namespace aux { } #endif - session_impl::session_impl( + session_impl::session_impl(CLevelDB &swarmDb, std::pair listen_port_range , fingerprint const& cl_fprint , char const* listen_interface @@ -609,6 +609,7 @@ namespace aux { , m_send_buffers(send_buffer_size) #endif , m_files(40) + , m_swarmDb(swarmDb) , m_io_service() #ifdef TORRENT_USE_OPENSSL , m_ssl_ctx(m_io_service, asio::ssl::context::sslv23) diff --git a/libtorrent/src/storage.cpp b/libtorrent/src/storage.cpp index 9d900ef2..f2864a10 100644 --- a/libtorrent/src/storage.cpp +++ b/libtorrent/src/storage.cpp @@ -242,22 +242,20 @@ namespace libtorrent } default_storage::default_storage(file_storage const& fs, file_storage const* mapped, std::string const& path - , file_pool& fp, std::vector const& file_prio) - // [MF] FIXME CLevelDB use path directly, cacheSize = 256K. - : CLevelDB(boost::filesystem::path(path), 256*1024, false, false) - , m_files(fs) + , CLevelDB &db, std::vector const& file_prio) + : m_files(fs) , m_file_priority(file_prio) - , m_pool(fp) + , m_db_path(path) + , m_db(db) , m_page_size(page_size()) , m_allocate_files(false) { if (mapped) m_mapped_files.reset(new file_storage(*mapped)); TORRENT_ASSERT(m_files.begin() != m_files.end()); - m_save_path = complete(path); } - default_storage::~default_storage() { m_pool.release(this); } + default_storage::~default_storage() { } bool default_storage::initialize(bool allocate_files) { @@ -265,8 +263,6 @@ namespace libtorrent error_code ec; std::vector().swap(m_file_priority); - // close files that were opened in write mode - m_pool.release(this); return error() ? true : false; } @@ -287,7 +283,6 @@ namespace libtorrent bool default_storage::release_files() { - m_pool.release(this); return false; } @@ -302,41 +297,6 @@ namespace libtorrent bool default_storage::delete_files() { - // make sure we don't have the files open - m_pool.release(this); - - // delete the files from disk - std::set directories; - typedef std::set::iterator iter_t; - for (file_storage::iterator i = files().begin() - , end(files().end()); i != end; ++i) - { - std::string fp = files().file_path(*i); - bool complete = is_complete(fp); - std::string p = complete ? fp : combine_path(m_save_path, fp); - if (!complete) - { - std::string bp = parent_path(fp); - std::pair ret; - ret.second = true; - while (ret.second && !bp.empty()) - { - ret = directories.insert(combine_path(m_save_path, bp)); - bp = parent_path(bp); - } - } - } - - // remove the directories. Reverse order to delete - // subdirectories first - - for (std::set::reverse_iterator i = directories.rbegin() - , end(directories.rend()); i != end; ++i) - { - delete_one_file(*i); - } - - if (error()) return true; return false; } @@ -398,99 +358,6 @@ namespace libtorrent int default_storage::move_storage(std::string const& sp, int flags) { int ret = piece_manager::no_error; - std::string save_path = complete(sp); - - // check to see if any of the files exist - error_code ec; - file_storage const& f = files(); - - file_status s; - if (flags == fail_if_exist) - { - stat_file(combine_path(save_path, f.name()), &s, ec); - if (ec != boost::system::errc::no_such_file_or_directory) - { - // the directory exists, check all the files - for (file_storage::iterator i = f.begin() - , end(f.end()); i != end; ++i) - { - // files moved out to absolute paths are ignored - if (is_complete(f.file_path(*i))) continue; - - std::string new_path = f.file_path(*i, save_path); - stat_file(new_path, &s, ec); - if (ec != boost::system::errc::no_such_file_or_directory) - return piece_manager::file_exist; - } - } - } - - // collect all directories in to_move. This is because we - // try to move entire directories by default (instead of - // files independently). - std::set to_move; - for (file_storage::iterator i = f.begin() - , end(f.end()); i != end; ++i) - { - // files moved out to absolute paths are not moved - if (is_complete(f.file_path(*i))) continue; - - std::string split = split_path(f.file_path(*i)); - to_move.insert(to_move.begin(), split); - } - - ec.clear(); - stat_file(save_path, &s, ec); - if (ec == boost::system::errc::no_such_file_or_directory) - { - ec.clear(); - create_directories(save_path, ec); - } - - if (ec) - { - set_error(save_path, ec); - return piece_manager::fatal_disk_error; - } - - m_pool.release(this); - - for (std::set::const_iterator i = to_move.begin() - , end(to_move.end()); i != end; ++i) - { - std::string old_path = combine_path(m_save_path, *i); - std::string new_path = combine_path(save_path, *i); - - rename(old_path, new_path, ec); - if (ec) - { - if (flags == dont_replace && ec == boost::system::errc::file_exists) - { - if (ret == piece_manager::no_error) ret = piece_manager::need_full_check; - continue; - } - - if (ec != boost::system::errc::no_such_file_or_directory) - { - error_code ec; - recursive_copy(old_path, new_path, ec); - if (ec) - { - set_error(old_path, ec); - ret = piece_manager::fatal_disk_error; - } - else - { - remove_all(old_path, ec); - } - break; - } - } - } - - if (ret == piece_manager::no_error || ret == piece_manager::need_full_check) - m_save_path = save_path; - return ret; } @@ -543,13 +410,14 @@ namespace libtorrent int tries = 2; while( tries-- ) { try { - if( Write(std::make_pair('p', slot), postStr) ) { + std::pair pathSlot = std::make_pair(m_db_path, slot); + if( m_db.Write(std::make_pair('p', pathSlot), postStr) ) { return postStr.size(); } else { return -1; } } catch( leveldb_error &e ) { - RepairDB(); + m_db.RepairDB(); } } @@ -583,7 +451,8 @@ namespace libtorrent while( tries-- ) { try { std::string postStr; - if( Read(std::make_pair('p', slot), postStr) ) { + std::pair pathSlot = std::make_pair(m_db_path, slot); + if( m_db.Read(std::make_pair('p', pathSlot), postStr) ) { TORRENT_ASSERT(bufs[0].iov_len >= postStr.size()); memcpy(bufs[0].iov_base, postStr.data(), postStr.size()); return postStr.size(); @@ -591,7 +460,7 @@ namespace libtorrent return 0; } } catch( leveldb_error &e ) { - RepairDB(); + m_db.RepairDB(); } } } @@ -619,24 +488,14 @@ namespace libtorrent boost::intrusive_ptr default_storage::open_file(file_storage::iterator fe, int mode , error_code& ec) const { - int cache_setting = m_settings ? settings().disk_io_write_mode : 0; - if (cache_setting == session_settings::disable_os_cache - || (cache_setting == session_settings::disable_os_cache_for_aligned_files - && ((fe->offset + files().file_base(*fe)) & (m_page_size-1)) == 0)) - mode |= file::no_buffer; - bool lock_files = m_settings ? settings().lock_files : false; - if (lock_files) mode |= file::lock_file; - if (!m_allocate_files) mode |= file::sparse; - if (m_settings && settings().no_atime_storage) mode |= file::no_atime; - - return m_pool.open_file(const_cast(this), m_save_path, fe, files(), mode, ec); + return new file(); } storage_interface* default_storage_constructor(file_storage const& fs - , file_storage const* mapped, std::string const& path, file_pool& fp + , file_storage const* mapped, std::string const& path, CLevelDB &db , std::vector const& file_prio) { - return new default_storage(fs, mapped, path, fp, file_prio); + return new default_storage(fs, mapped, path, db, file_prio); } int disabled_storage::readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs, int flags) @@ -698,7 +557,7 @@ namespace libtorrent boost::shared_ptr const& torrent , boost::intrusive_ptr info , std::string const& save_path - , file_pool& fp + , CLevelDB &db , disk_io_thread& io , storage_constructor_type sc , storage_mode_t sm @@ -706,7 +565,7 @@ namespace libtorrent : m_info(info) , m_files(m_info->files()) , m_storage(sc(m_info->orig_files(), &m_info->files() != &m_info->orig_files() - ? &m_info->files() : 0, combine_path(save_path,to_hex(m_info->info_hash().to_string())), fp, file_prio)) + ? &m_info->files() : 0, to_hex(m_info->info_hash().to_string()), db, file_prio)) , m_storage_mode(sm) , m_save_path(complete(save_path)) , m_state(state_none) diff --git a/libtorrent/src/torrent.cpp b/libtorrent/src/torrent.cpp index 345519d4..1e9e9dec 100644 --- a/libtorrent/src/torrent.cpp +++ b/libtorrent/src/torrent.cpp @@ -1540,7 +1540,7 @@ namespace libtorrent // the shared_from_this() will create an intentional // cycle of ownership, se the hpp file for description. m_owning_storage = new piece_manager(shared_from_this(), m_torrent_file - , m_save_path, m_ses.m_files, m_ses.m_disk_thread, m_storage_constructor + , m_save_path, m_ses.m_swarmDb, m_ses.m_disk_thread, m_storage_constructor , (storage_mode_t)m_storage_mode, m_file_priority); m_storage = m_owning_storage.get(); @@ -1600,7 +1600,10 @@ namespace libtorrent int ev = 0; if (m_resume_entry.dict_find_string_value("file-format") != "libtorrent resume file") ev = errors::invalid_file_tag; - + + if (!ev && m_resume_entry.dict_find_int_value("file-version") < 2 ) + ev = errors::invalid_file_tag; + std::string info_hash = m_resume_entry.dict_find_string_value("info-hash"); if (!ev && info_hash.empty()) ev = errors::missing_info_hash; @@ -4945,9 +4948,8 @@ namespace libtorrent void torrent::read_resume_data(lazy_entry const& rd) { - // [MF] - int num_pieces = rd.dict_find_int_value("num_pieces"); - increase_num_pieces(num_pieces); + int num_pieces = rd.dict_find_int_value("num_pieces"); + increase_num_pieces(num_pieces); m_total_uploaded = rd.dict_find_int_value("total_uploaded"); m_total_downloaded = rd.dict_find_int_value("total_downloaded"); @@ -5125,11 +5127,10 @@ namespace libtorrent { using namespace libtorrent::detail; // for write_*_endpoint() ret["file-format"] = "libtorrent resume file"; - ret["file-version"] = 1; + ret["file-version"] = 2; ret["libtorrent-version"] = LIBTORRENT_VERSION; - // [MF] - ret["num_pieces"] = m_torrent_file->num_pieces(); + ret["num_pieces"] = m_torrent_file->num_pieces(); ret["total_uploaded"] = m_total_uploaded; ret["total_downloaded"] = m_total_downloaded; diff --git a/src/twister.cpp b/src/twister.cpp index 8a6d9d66..35ced31e 100644 --- a/src/twister.cpp +++ b/src/twister.cpp @@ -48,6 +48,7 @@ static map m_specialResources; enum ExpireResType { SimpleNoExpire, NumberedNoExpire, PostNoExpireRecent }; static map m_noExpireResources; static map m_userTorrent; +static boost::scoped_ptr m_swarmDb; static CCriticalSection cs_spamMsg; static std::string m_preferredSpamLang = "[en]"; @@ -87,10 +88,8 @@ torrent_handle startTorrentUser(std::string const &username) tparams.name = username; boost::filesystem::path torrentPath = GetDataDir() / "swarm"; tparams.save_path= torrentPath.string(); - libtorrent::error_code ec; create_directory(tparams.save_path, ec); - std::string filename = combine_path(tparams.save_path, to_hex(ih.to_string()) + ".resume"); load_file(filename.c_str(), tparams.resume_data); @@ -208,12 +207,17 @@ void ThreadWaitExtIP() } libtorrent::error_code ec; + + boost::filesystem::path swarmDbPath = GetDataDir() / "swarm" / "db"; + create_directories(swarmDbPath.string(), ec); + m_swarmDb.reset(new CLevelDB(swarmDbPath.string(), 256*1024, false, false)); + int listen_port = GetListenPort() + LIBTORRENT_PORT_OFFSET; std::string bind_to_interface = ""; printf("Creating new libtorrent session ext_ip=%s port=%d\n", ipStr.c_str(), listen_port); - ses = new session(fingerprint("TW", LIBTORRENT_VERSION_MAJOR, LIBTORRENT_VERSION_MINOR, 0, 0) + ses = new session(*m_swarmDb, fingerprint("TW", LIBTORRENT_VERSION_MAJOR, LIBTORRENT_VERSION_MINOR, 0, 0) , session::add_default_plugins , alert::dht_notification , ipStr.size() ? ipStr.c_str() : NULL