From 9f281c2d255d5ea5b1e44f8b719ba005dd863797 Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Wed, 1 Apr 2020 14:25:00 +0800 Subject: [PATCH] Avoid holding entire file in memory Previously we need a file buffer that is as large as the file size and this could be a problem when user has less free memory available or having very large data. Now with the help of `FileOutputIterator`, we can have a much smaller, fixed size immediate file buffer and also the code looks nice with `lt::bencode()`. --- src/base/CMakeLists.txt | 2 + src/base/base.pri | 2 + src/base/bittorrent/torrentcreatorthread.cpp | 22 +++--- src/base/bittorrent/torrentinfo.cpp | 11 +-- src/base/utils/io.cpp | 73 ++++++++++++++++++++ src/base/utils/io.h | 63 +++++++++++++++++ 6 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 src/base/utils/io.cpp create mode 100644 src/base/utils/io.h diff --git a/src/base/CMakeLists.txt b/src/base/CMakeLists.txt index 4467c9bf8..ab1d97686 100644 --- a/src/base/CMakeLists.txt +++ b/src/base/CMakeLists.txt @@ -58,6 +58,7 @@ utils/bytearray.h utils/foreignapps.h utils/fs.h utils/gzip.h +utils/io.h utils/misc.h utils/net.h utils/password.h @@ -133,6 +134,7 @@ utils/bytearray.cpp utils/foreignapps.cpp utils/fs.cpp utils/gzip.cpp +utils/io.cpp utils/misc.cpp utils/net.cpp utils/password.cpp diff --git a/src/base/base.pri b/src/base/base.pri index 1847d33f4..e33255930 100644 --- a/src/base/base.pri +++ b/src/base/base.pri @@ -73,6 +73,7 @@ HEADERS += \ $$PWD/utils/foreignapps.h \ $$PWD/utils/fs.h \ $$PWD/utils/gzip.h \ + $$PWD/utils/io.h \ $$PWD/utils/misc.h \ $$PWD/utils/net.h \ $$PWD/utils/password.h \ @@ -143,6 +144,7 @@ SOURCES += \ $$PWD/utils/foreignapps.cpp \ $$PWD/utils/fs.cpp \ $$PWD/utils/gzip.cpp \ + $$PWD/utils/io.cpp \ $$PWD/utils/misc.cpp \ $$PWD/utils/net.cpp \ $$PWD/utils/password.cpp \ diff --git a/src/base/bittorrent/torrentcreatorthread.cpp b/src/base/bittorrent/torrentcreatorthread.cpp index a48173d7c..ee3c5427e 100644 --- a/src/base/bittorrent/torrentcreatorthread.cpp +++ b/src/base/bittorrent/torrentcreatorthread.cpp @@ -41,8 +41,10 @@ #include #include +#include "base/exceptions.h" #include "base/global.h" #include "base/utils/fs.h" +#include "base/utils/io.h" #include "base/utils/string.h" #include "private/ltunderlyingtype.h" @@ -182,19 +184,19 @@ void TorrentCreatorThread::run() if (isInterruptionRequested()) return; // create the torrent - std::ofstream outfile( -#ifdef _MSC_VER - Utils::Fs::toNativePath(m_params.savePath).toStdWString().c_str() -#else - Utils::Fs::toNativePath(m_params.savePath).toUtf8().constData() -#endif - , (std::ios_base::out | std::ios_base::binary | std::ios_base::trunc)); - if (outfile.fail()) - throw std::runtime_error(tr("create new torrent file failed").toStdString()); + QFile outfile {m_params.savePath}; + if (!outfile.open(QIODevice::WriteOnly)) { + throw RuntimeError {tr("Create new torrent file failed. Reason: %1") + .arg(outfile.errorString())}; + } if (isInterruptionRequested()) return; - lt::bencode(std::ostream_iterator(outfile), entry); + lt::bencode(Utils::IO::FileDeviceOutputIterator {outfile}, entry); + if (outfile.error() != QFileDevice::NoError) { + throw RuntimeError {tr("Create new torrent file failed. Reason: %1") + .arg(outfile.errorString())}; + } outfile.close(); emit updateProgress(100); diff --git a/src/base/bittorrent/torrentinfo.cpp b/src/base/bittorrent/torrentinfo.cpp index 4838a80a5..299dad867 100644 --- a/src/base/bittorrent/torrentinfo.cpp +++ b/src/base/bittorrent/torrentinfo.cpp @@ -47,6 +47,7 @@ #include "base/exceptions.h" #include "base/global.h" #include "base/utils/fs.h" +#include "base/utils/io.h" #include "base/utils/misc.h" #include "infohash.h" #include "trackerentry.h" @@ -166,12 +167,12 @@ void TorrentInfo::saveToFile(const QString &path) const #endif const lt::entry torrentEntry = torrentCreator.generate(); - QByteArray out; - out.reserve(1024 * 1024); // most torrent file sizes are under 1 MB - lt::bencode(std::back_inserter(out), torrentEntry); + QFile torrentFile {path}; + if (!torrentFile.open(QIODevice::WriteOnly)) + throw RuntimeError {torrentFile.errorString()}; - QFile torrentFile{path}; - if (!torrentFile.open(QIODevice::WriteOnly) || (torrentFile.write(out) != out.size())) + lt::bencode(Utils::IO::FileDeviceOutputIterator {torrentFile}, torrentEntry); + if (torrentFile.error() != QFileDevice::NoError) throw RuntimeError {torrentFile.errorString()}; } diff --git a/src/base/utils/io.cpp b/src/base/utils/io.cpp new file mode 100644 index 000000000..499b55a1e --- /dev/null +++ b/src/base/utils/io.cpp @@ -0,0 +1,73 @@ +/* + * Bittorrent Client using Qt and libtorrent. + * Copyright (C) 2020 Mike Tzou (Chocobo1) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * In addition, as a special exception, the copyright holders give permission to + * link this program with the OpenSSL project's "OpenSSL" library (or with + * modified versions of it that use the same license as the "OpenSSL" library), + * and distribute the linked executables. You must obey the GNU General Public + * License in all respects for all of the code used other than "OpenSSL". If you + * modify file(s), you may extend this exception to your version of the file(s), + * but you are not obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + */ + +#include "io.h" + +#include +#include + +Utils::IO::FileDeviceOutputIterator::FileDeviceOutputIterator(QFileDevice &device, const int bufferSize) + : m_device {&device} + , m_buffer {std::make_shared()} + , m_bufferSize {bufferSize} +{ + m_buffer->reserve(bufferSize); +} + +Utils::IO::FileDeviceOutputIterator::~FileDeviceOutputIterator() +{ + if (m_device->error() == QFileDevice::NoError) + m_device->write(*m_buffer); + m_buffer->clear(); +} + +Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator=(const char c) +{ + m_buffer->append(c); + if (m_buffer->size() >= m_bufferSize) { + if (m_device->error() == QFileDevice::NoError) + m_device->write(*m_buffer); + m_buffer->clear(); + } + return *this; +} + +Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator*() +{ + return *this; +} + +Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator++() +{ + return *this; +} + +Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator++(int) +{ + return *this; +} diff --git a/src/base/utils/io.h b/src/base/utils/io.h new file mode 100644 index 000000000..0c1959e63 --- /dev/null +++ b/src/base/utils/io.h @@ -0,0 +1,63 @@ +/* + * Bittorrent Client using Qt and libtorrent. + * Copyright (C) 2020 Mike Tzou (Chocobo1) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * In addition, as a special exception, the copyright holders give permission to + * link this program with the OpenSSL project's "OpenSSL" library (or with + * modified versions of it that use the same license as the "OpenSSL" library), + * and distribute the linked executables. You must obey the GNU General Public + * License in all respects for all of the code used other than "OpenSSL". If you + * modify file(s), you may extend this exception to your version of the file(s), + * but you are not obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + */ + +#pragma once + +#include +#include + +class QByteArray; +class QFileDevice; + +namespace Utils +{ + namespace IO + { + // A wrapper class that satisfy LegacyOutputIterator requirement + class FileDeviceOutputIterator + : public std::iterator + { + public: + explicit FileDeviceOutputIterator(QFileDevice &device, const int bufferSize = (4 * 1024)); + FileDeviceOutputIterator(const FileDeviceOutputIterator &other) = default; + ~FileDeviceOutputIterator(); + + // mimic std::ostream_iterator behavior + FileDeviceOutputIterator &operator=(char c); + // TODO: make these `constexpr` in C++17 + FileDeviceOutputIterator &operator*(); + FileDeviceOutputIterator &operator++(); + FileDeviceOutputIterator &operator++(int); + + private: + QFileDevice *m_device; + std::shared_ptr m_buffer; + int m_bufferSize; + }; + } +}