Merge pull request #12364 from Chocobo1/fileOutIter

Avoid holding entire file in memory

src/base/CMakeLists.txt

@@ -58,6 +58,7 @@ utils/bytearray.h
 utils/foreignapps.h
 utils/fs.h
 utils/gzip.h
+utils/io.h
 utils/misc.h
 utils/net.h
 utils/password.h
@@ -133,6 +134,7 @@ utils/bytearray.cpp
 utils/foreignapps.cpp
 utils/fs.cpp
 utils/gzip.cpp
+utils/io.cpp
 utils/misc.cpp
 utils/net.cpp
 utils/password.cpp

src/base/base.pri

@@ -73,6 +73,7 @@ HEADERS += \
     $$PWD/utils/foreignapps.h \
     $$PWD/utils/fs.h \
     $$PWD/utils/gzip.h \
+    $$PWD/utils/io.h \
     $$PWD/utils/misc.h \
     $$PWD/utils/net.h \
     $$PWD/utils/password.h \
@@ -143,6 +144,7 @@ SOURCES += \
     $$PWD/utils/foreignapps.cpp \
     $$PWD/utils/fs.cpp \
     $$PWD/utils/gzip.cpp \
+    $$PWD/utils/io.cpp \
     $$PWD/utils/misc.cpp \
     $$PWD/utils/net.cpp \
     $$PWD/utils/password.cpp \

src/base/bittorrent/torrentcreatorthread.cpp

@@ -41,8 +41,10 @@
 #include <QFileInfo>
 #include <QHash>
 
+#include "base/exceptions.h"
 #include "base/global.h"
 #include "base/utils/fs.h"
+#include "base/utils/io.h"
 #include "base/utils/string.h"
 #include "private/ltunderlyingtype.h"
 
@@ -182,19 +184,19 @@ void TorrentCreatorThread::run()
         if (isInterruptionRequested()) return;
 
         // create the torrent
-        std::ofstream outfile(
-#ifdef _MSC_VER
-            Utils::Fs::toNativePath(m_params.savePath).toStdWString().c_str()
-#else
-            Utils::Fs::toNativePath(m_params.savePath).toUtf8().constData()
-#endif
-            , (std::ios_base::out | std::ios_base::binary | std::ios_base::trunc));
-        if (outfile.fail())
-            throw std::runtime_error(tr("create new torrent file failed").toStdString());
+        QFile outfile {m_params.savePath};
+        if (!outfile.open(QIODevice::WriteOnly)) {
+            throw RuntimeError {tr("Create new torrent file failed. Reason: %1")
+                .arg(outfile.errorString())};
+        }
 
         if (isInterruptionRequested()) return;
 
-        lt::bencode(std::ostream_iterator<char>(outfile), entry);
+        lt::bencode(Utils::IO::FileDeviceOutputIterator {outfile}, entry);
+        if (outfile.error() != QFileDevice::NoError) {
+            throw RuntimeError {tr("Create new torrent file failed. Reason: %1")
+                .arg(outfile.errorString())};
+        }
         outfile.close();
 
         emit updateProgress(100);

src/base/bittorrent/torrentinfo.cpp

@@ -47,6 +47,7 @@
 #include "base/exceptions.h"
 #include "base/global.h"
 #include "base/utils/fs.h"
+#include "base/utils/io.h"
 #include "base/utils/misc.h"
 #include "infohash.h"
 #include "trackerentry.h"
@@ -166,12 +167,12 @@ void TorrentInfo::saveToFile(const QString &path) const
 #endif
     const lt::entry torrentEntry = torrentCreator.generate();
 
-    QByteArray out;
-    out.reserve(1024 * 1024);  // most torrent file sizes are under 1 MB
-    lt::bencode(std::back_inserter(out), torrentEntry);
+    QFile torrentFile {path};
+    if (!torrentFile.open(QIODevice::WriteOnly))
+        throw RuntimeError {torrentFile.errorString()};
 
-    QFile torrentFile{path};
-    if (!torrentFile.open(QIODevice::WriteOnly) || (torrentFile.write(out) != out.size()))
+    lt::bencode(Utils::IO::FileDeviceOutputIterator {torrentFile}, torrentEntry);
+    if (torrentFile.error() != QFileDevice::NoError)
         throw RuntimeError {torrentFile.errorString()};
 }
 

src/base/utils/io.cpp

@@ -0,0 +1,73 @@
+/*
+ * Bittorrent Client using Qt and libtorrent.
+ * Copyright (C) 2020  Mike Tzou (Chocobo1)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * In addition, as a special exception, the copyright holders give permission to
+ * link this program with the OpenSSL project's "OpenSSL" library (or with
+ * modified versions of it that use the same license as the "OpenSSL" library),
+ * and distribute the linked executables. You must obey the GNU General Public
+ * License in all respects for all of the code used other than "OpenSSL".  If you
+ * modify file(s), you may extend this exception to your version of the file(s),
+ * but you are not obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ */
+
+#include "io.h"
+
+#include <QByteArray>
+#include <QFileDevice>
+
+Utils::IO::FileDeviceOutputIterator::FileDeviceOutputIterator(QFileDevice &device, const int bufferSize)
+    : m_device {&device}
+    , m_buffer {std::make_shared<QByteArray>()}
+    , m_bufferSize {bufferSize}
+{
+    m_buffer->reserve(bufferSize);
+}
+
+Utils::IO::FileDeviceOutputIterator::~FileDeviceOutputIterator()
+{
+    if (m_device->error() == QFileDevice::NoError)
+        m_device->write(*m_buffer);
+    m_buffer->clear();
+}
+
+Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator=(const char c)
+{
+    m_buffer->append(c);
+    if (m_buffer->size() >= m_bufferSize) {
+        if (m_device->error() == QFileDevice::NoError)
+            m_device->write(*m_buffer);
+        m_buffer->clear();
+    }
+    return *this;
+}
+
+Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator*()
+{
+    return *this;
+}
+
+Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator++()
+{
+    return *this;
+}
+
+Utils::IO::FileDeviceOutputIterator &Utils::IO::FileDeviceOutputIterator::operator++(int)
+{
+    return *this;
+}

src/base/utils/io.h

@@ -0,0 +1,63 @@
+/*
+ * Bittorrent Client using Qt and libtorrent.
+ * Copyright (C) 2020  Mike Tzou (Chocobo1)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * In addition, as a special exception, the copyright holders give permission to
+ * link this program with the OpenSSL project's "OpenSSL" library (or with
+ * modified versions of it that use the same license as the "OpenSSL" library),
+ * and distribute the linked executables. You must obey the GNU General Public
+ * License in all respects for all of the code used other than "OpenSSL".  If you
+ * modify file(s), you may extend this exception to your version of the file(s),
+ * but you are not obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ */
+
+#pragma once
+
+#include <iterator>
+#include <memory>
+
+class QByteArray;
+class QFileDevice;
+
+namespace Utils
+{
+    namespace IO
+    {
+        // A wrapper class that satisfy LegacyOutputIterator requirement
+        class FileDeviceOutputIterator
+            : public std::iterator<std::output_iterator_tag, void, void, void, void>
+        {
+        public:
+            explicit FileDeviceOutputIterator(QFileDevice &device, const int bufferSize = (4 * 1024));
+            FileDeviceOutputIterator(const FileDeviceOutputIterator &other) = default;
+            ~FileDeviceOutputIterator();
+
+            // mimic std::ostream_iterator behavior
+            FileDeviceOutputIterator &operator=(char c);
+            // TODO: make these `constexpr` in C++17
+            FileDeviceOutputIterator &operator*();
+            FileDeviceOutputIterator &operator++();
+            FileDeviceOutputIterator &operator++(int);
+
+        private:
+            QFileDevice *m_device;
+            std::shared_ptr<QByteArray> m_buffer;
+            int m_bufferSize;
+        };
+    }
+}