Implement "Sequential downloading" feature

Closes #6835.

src/base/net/downloadhandler.cpp

@@ -1,6 +1,6 @@
 /*
  * Bittorrent Client using Qt and libtorrent.
- * Copyright (C) 2015  Vladimir Golovnev <glassez@yandex.ru>
+ * Copyright (C) 2015, 2018  Vladimir Golovnev <glassez@yandex.ru>
  * Copyright (C) 2006  Christophe Dumez <chris@qbittorrent.org>
  *
  * This program is free software; you can redistribute it and/or
@@ -29,6 +29,7 @@
 
 #include "downloadhandler.h"
 
+#include <QCoreApplication>
 #include <QDebug>
 #include <QNetworkAccessManager>
 #include <QNetworkCookie>
@@ -43,32 +44,46 @@
 #include "base/utils/misc.h"
 #include "downloadmanager.h"
 
-static QString errorCodeToString(QNetworkReply::NetworkError status);
+namespace
+{
+    QString tr(const char *message);
+    QString errorCodeToString(QNetworkReply::NetworkError status);
+}
 
-using namespace Net;
+Net::DownloadHandler::DownloadHandler(QNetworkReply *reply, DownloadManager *manager, const DownloadRequest &downloadRequest)
-
-DownloadHandler::DownloadHandler(QNetworkReply *reply, DownloadManager *manager, const DownloadRequest &downloadRequest)
     : QObject(manager)
     , m_reply(reply)
     , m_manager(manager)
     , m_downloadRequest(downloadRequest)
 {
-    init();
+    if (reply)
+        assignNetworkReply(reply);
 }
 
-DownloadHandler::~DownloadHandler()
+Net::DownloadHandler::~DownloadHandler()
 {
     if (m_reply)
         delete m_reply;
 }
 
+void Net::DownloadHandler::assignNetworkReply(QNetworkReply *reply)
+{
+    Q_ASSERT(reply);
+
+    m_reply = reply;
+    m_reply->setParent(this);
+    if (m_downloadRequest.limit() > 0)
+        connect(m_reply, &QNetworkReply::downloadProgress, this, &Net::DownloadHandler::checkDownloadSize);
+    connect(m_reply, &QNetworkReply::finished, this, &Net::DownloadHandler::processFinishedDownload);
+}
+
 // Returns original url
-QString DownloadHandler::url() const
+QString Net::DownloadHandler::url() const
 {
     return m_downloadRequest.url();
 }
 
-void DownloadHandler::processFinishedDownload()
+void Net::DownloadHandler::processFinishedDownload()
 {
     QString url = m_reply->url().toString();
     qDebug("Download finished: %s", qUtf8Printable(url));
@@ -110,7 +125,7 @@ void DownloadHandler::processFinishedDownload()
     }
 }
 
-void DownloadHandler::checkDownloadSize(qint64 bytesReceived, qint64 bytesTotal)
+void Net::DownloadHandler::checkDownloadSize(qint64 bytesReceived, qint64 bytesTotal)
 {
     QString msg = tr("The file size is %1. It exceeds the download limit of %2.");
 
@@ -130,15 +145,7 @@ void DownloadHandler::checkDownloadSize(qint64 bytesReceived, qint64 bytesTotal)
     }
 }
 
-void DownloadHandler::init()
+bool Net::DownloadHandler::saveToFile(const QByteArray &replyData, QString &filePath)
-{
-    m_reply->setParent(this);
-    if (m_downloadRequest.limit() > 0)
-        connect(m_reply, &QNetworkReply::downloadProgress, this, &Net::DownloadHandler::checkDownloadSize);
-    connect(m_reply, &QNetworkReply::finished, this, &Net::DownloadHandler::processFinishedDownload);
-}
-
-bool DownloadHandler::saveToFile(const QByteArray &replyData, QString &filePath)
 {
     QTemporaryFile *tmpfile = new QTemporaryFile(Utils::Fs::tempPath() + "XXXXXX");
     if (!tmpfile->open()) {
@@ -165,14 +172,14 @@ bool DownloadHandler::saveToFile(const QByteArray &replyData, QString &filePath)
     return false;
 }
 
-void DownloadHandler::handleRedirection(QUrl newUrl)
+void Net::DownloadHandler::handleRedirection(QUrl newUrl)
 {
     // Resolve relative urls
     if (newUrl.isRelative())
         newUrl = m_reply->url().resolved(newUrl);
 
     const QString newUrlString = newUrl.toString();
-    qDebug("Redirecting from %s to %s", qUtf8Printable(m_reply->url().toString()), qUtf8Printable(newUrlString));
+    qDebug("Redirecting from %s to %s...", qUtf8Printable(m_reply->url().toString()), qUtf8Printable(newUrlString));
 
     // Redirect to magnet workaround
     if (newUrlString.startsWith("magnet:", Qt::CaseInsensitive)) {
@@ -186,61 +193,83 @@ void DownloadHandler::handleRedirection(QUrl newUrl)
         this->deleteLater();
     }
     else {
-        DownloadHandler *tmp = m_manager->download(DownloadRequest(m_downloadRequest).url(newUrlString));
+        DownloadHandler *redirected = m_manager->download(DownloadRequest(m_downloadRequest).url(newUrlString));
-        m_reply->deleteLater();
+        connect(redirected, &DownloadHandler::destroyed, this, &DownloadHandler::deleteLater);
-        m_reply = tmp->m_reply;
+        connect(redirected, &DownloadHandler::downloadFailed, this, [this](const QString &, const QString &reason)
-        init();
+        {
-        tmp->m_reply = nullptr;
+            emit downloadFailed(url(), reason);
-        delete tmp;
+        });
+        connect(redirected, &DownloadHandler::redirectedToMagnet, this, [this](const QString &, const QString &magnetUri)
+        {
+            emit redirectedToMagnet(url(), magnetUri);
+        });
+        connect(redirected, static_cast<void (DownloadHandler::*)(const QString &, const QString &)>(&DownloadHandler::downloadFinished)
+                , this, [this](const QString &, const QString &fileName)
+        {
+            emit downloadFinished(url(), fileName);
+        });
+        connect(redirected, static_cast<void (DownloadHandler::*)(const QString &, const QByteArray &)>(&DownloadHandler::downloadFinished)
+                , this, [this](const QString &, const QByteArray &data)
+        {
+            emit downloadFinished(url(), data);
+        });
     }
 }
 
+namespace
+{
+    QString tr(const char *message)
+    {
+        return QCoreApplication::translate("DownloadHandler", message);
+    }
+
     QString errorCodeToString(QNetworkReply::NetworkError status)
     {
         switch (status) {
         case QNetworkReply::HostNotFoundError:
-        return QObject::tr("The remote host name was not found (invalid hostname)");
+            return tr("The remote host name was not found (invalid hostname)");
         case QNetworkReply::OperationCanceledError:
-        return QObject::tr("The operation was canceled");
+            return tr("The operation was canceled");
         case QNetworkReply::RemoteHostClosedError:
-        return QObject::tr("The remote server closed the connection prematurely, before the entire reply was received and processed");
+            return tr("The remote server closed the connection prematurely, before the entire reply was received and processed");
         case QNetworkReply::TimeoutError:
-        return QObject::tr("The connection to the remote server timed out");
+            return tr("The connection to the remote server timed out");
         case QNetworkReply::SslHandshakeFailedError:
-        return QObject::tr("SSL/TLS handshake failed");
+            return tr("SSL/TLS handshake failed");
         case QNetworkReply::ConnectionRefusedError:
-        return QObject::tr("The remote server refused the connection");
+            return tr("The remote server refused the connection");
         case QNetworkReply::ProxyConnectionRefusedError:
-        return QObject::tr("The connection to the proxy server was refused");
+            return tr("The connection to the proxy server was refused");
         case QNetworkReply::ProxyConnectionClosedError:
-        return QObject::tr("The proxy server closed the connection prematurely");
+            return tr("The proxy server closed the connection prematurely");
         case QNetworkReply::ProxyNotFoundError:
-        return QObject::tr("The proxy host name was not found");
+            return tr("The proxy host name was not found");
         case QNetworkReply::ProxyTimeoutError:
-        return QObject::tr("The connection to the proxy timed out or the proxy did not reply in time to the request sent");
+            return tr("The connection to the proxy timed out or the proxy did not reply in time to the request sent");
         case QNetworkReply::ProxyAuthenticationRequiredError:
-        return QObject::tr("The proxy requires authentication in order to honor the request but did not accept any credentials offered");
+            return tr("The proxy requires authentication in order to honor the request but did not accept any credentials offered");
         case QNetworkReply::ContentAccessDenied:
-        return QObject::tr("The access to the remote content was denied (401)");
+            return tr("The access to the remote content was denied (401)");
         case QNetworkReply::ContentOperationNotPermittedError:
-        return QObject::tr("The operation requested on the remote content is not permitted");
+            return tr("The operation requested on the remote content is not permitted");
         case QNetworkReply::ContentNotFoundError:
-        return QObject::tr("The remote content was not found at the server (404)");
+            return tr("The remote content was not found at the server (404)");
         case QNetworkReply::AuthenticationRequiredError:
-        return QObject::tr("The remote server requires authentication to serve the content but the credentials provided were not accepted");
+            return tr("The remote server requires authentication to serve the content but the credentials provided were not accepted");
         case QNetworkReply::ProtocolUnknownError:
-        return QObject::tr("The Network Access API cannot honor the request because the protocol is not known");
+            return tr("The Network Access API cannot honor the request because the protocol is not known");
         case QNetworkReply::ProtocolInvalidOperationError:
-        return QObject::tr("The requested operation is invalid for this protocol");
+            return tr("The requested operation is invalid for this protocol");
         case QNetworkReply::UnknownNetworkError:
-        return QObject::tr("An unknown network-related error was detected");
+            return tr("An unknown network-related error was detected");
         case QNetworkReply::UnknownProxyError:
-        return QObject::tr("An unknown proxy-related error was detected");
+            return tr("An unknown proxy-related error was detected");
         case QNetworkReply::UnknownContentError:
-        return QObject::tr("An unknown error related to the remote content was detected");
+            return tr("An unknown error related to the remote content was detected");
         case QNetworkReply::ProtocolFailure:
-        return QObject::tr("A breakdown in protocol was detected");
+            return tr("A breakdown in protocol was detected");
         default:
-        return QObject::tr("Unknown error");
+            return tr("Unknown error");
+        }
     }
 }

src/base/net/downloadhandler.h

@@ -1,6 +1,6 @@
 /*
  * Bittorrent Client using Qt and libtorrent.
- * Copyright (C) 2015  Vladimir Golovnev <glassez@yandex.ru>
+ * Copyright (C) 2015, 2018  Vladimir Golovnev <glassez@yandex.ru>
  * Copyright (C) 2006  Christophe Dumez <chris@qbittorrent.org>
  *
  * This program is free software; you can redistribute it and/or
@@ -33,7 +33,6 @@
 #include <QObject>
 #include "downloadmanager.h"
 
-class QNetworkAccessManager;
 class QNetworkReply;
 class QUrl;
 
@@ -44,10 +43,14 @@ namespace Net
     class DownloadHandler : public QObject
     {
         Q_OBJECT
+        Q_DISABLE_COPY(DownloadHandler)
+
+        friend class DownloadManager;
+
+        DownloadHandler(QNetworkReply *reply, DownloadManager *manager, const DownloadRequest &downloadRequest);
 
     public:
-        DownloadHandler(QNetworkReply *reply, DownloadManager *manager, const DownloadRequest &downloadRequest);
+        ~DownloadHandler() override;
-        ~DownloadHandler();
 
         QString url() const;
 
@@ -62,7 +65,7 @@ namespace Net
         void checkDownloadSize(qint64 bytesReceived, qint64 bytesTotal);
 
     private:
-        void init();
+        void assignNetworkReply(QNetworkReply *reply);
         bool saveToFile(const QByteArray &replyData, QString &filePath);
         void handleRedirection(QUrl newUrl);
 

src/base/net/downloadmanager.cpp

@@ -1,6 +1,6 @@
 /*
  * Bittorrent Client using Qt and libtorrent.
- * Copyright (C) 2015  Vladimir Golovnev <glassez@yandex.ru>
+ * Copyright (C) 2015, 2018  Vladimir Golovnev <glassez@yandex.ru>
  * Copyright (C) 2006  Christophe Dumez <chris@qbittorrent.org>
  *
  * This program is free software; you can redistribute it and/or
@@ -103,52 +103,10 @@ namespace
             return QNetworkCookieJar::setCookiesFromUrl(cookies, url);
         }
     };
-}
 
-using namespace Net;
+    QNetworkRequest createNetworkRequest(const Net::DownloadRequest &downloadRequest)
-
-DownloadManager *DownloadManager::m_instance = nullptr;
-
-DownloadManager::DownloadManager(QObject *parent)
-    : QObject(parent)
     {
-#ifndef QT_NO_OPENSSL
+        QNetworkRequest request {downloadRequest.url()};
-    connect(&m_networkManager, &QNetworkAccessManager::sslErrors, this, &Net::DownloadManager::ignoreSslErrors);
-#endif
-    m_networkManager.setCookieJar(new NetworkCookieJar(this));
-}
-
-void DownloadManager::initInstance()
-{
-    if (!m_instance)
-        m_instance = new DownloadManager;
-}
-
-void DownloadManager::freeInstance()
-{
-    if (m_instance) {
-        delete m_instance;
-        m_instance = nullptr;
-    }
-}
-
-DownloadManager *DownloadManager::instance()
-{
-    return m_instance;
-}
-
-DownloadHandler *DownloadManager::downloadUrl(const QString &url, bool saveToFile, qint64 limit, bool handleRedirectToMagnet, const QString &userAgent)
-{
-    return download(DownloadRequest(url).saveToFile(saveToFile).limit(limit).handleRedirectToMagnet(handleRedirectToMagnet).userAgent(userAgent));
-}
-
-DownloadHandler *DownloadManager::download(const DownloadRequest &downloadRequest)
-{
-    // Update proxy settings
-    applyProxySettings();
-
-    // Process download request
-    QNetworkRequest request(downloadRequest.url());
 
         if (downloadRequest.userAgent().isEmpty())
             request.setRawHeader("User-Agent", DEFAULT_USER_AGENT);
@@ -157,39 +115,105 @@ DownloadHandler *DownloadManager::download(const DownloadRequest &downloadReques
 
         // Spoof HTTP Referer to allow adding torrent link from Torcache/KickAssTorrents
         request.setRawHeader("Referer", request.url().toEncoded().data());
-
+        // Accept gzip
-    qDebug("Downloading %s...", request.url().toEncoded().data());
-    // accept gzip
         request.setRawHeader("Accept-Encoding", "gzip");
-    return new DownloadHandler(m_networkManager.get(request), this, downloadRequest);
+
+        return request;
+    }
 }
 
-QList<QNetworkCookie> DownloadManager::cookiesForUrl(const QUrl &url) const
+Net::DownloadManager *Net::DownloadManager::m_instance = nullptr;
+
+Net::DownloadManager::DownloadManager(QObject *parent)
+    : QObject(parent)
+{
+#ifndef QT_NO_OPENSSL
+    connect(&m_networkManager, &QNetworkAccessManager::sslErrors, this, &Net::DownloadManager::ignoreSslErrors);
+#endif
+    connect(&m_networkManager, &QNetworkAccessManager::finished, this, &DownloadManager::handleReplyFinished);
+    connect(ProxyConfigurationManager::instance(), &ProxyConfigurationManager::proxyConfigurationChanged
+            , this, &DownloadManager::applyProxySettings);
+    m_networkManager.setCookieJar(new NetworkCookieJar(this));
+}
+
+void Net::DownloadManager::initInstance()
+{
+    if (!m_instance)
+        m_instance = new DownloadManager;
+}
+
+void Net::DownloadManager::freeInstance()
+{
+    if (m_instance) {
+        delete m_instance;
+        m_instance = nullptr;
+    }
+}
+
+Net::DownloadManager *Net::DownloadManager::instance()
+{
+    return m_instance;
+}
+
+Net::DownloadHandler *Net::DownloadManager::downloadUrl(const QString &url, bool saveToFile, qint64 limit, bool handleRedirectToMagnet, const QString &userAgent)
+{
+    return download(DownloadRequest(url).saveToFile(saveToFile).limit(limit).handleRedirectToMagnet(handleRedirectToMagnet).userAgent(userAgent));
+}
+
+Net::DownloadHandler *Net::DownloadManager::download(const DownloadRequest &downloadRequest)
+{
+    // Process download request
+    const QNetworkRequest request = createNetworkRequest(downloadRequest);
+    const ServiceID id = ServiceID::fromURL(request.url());
+    const bool isSequentialService = m_sequentialServices.contains(id);
+    if (!isSequentialService || !m_busyServices.contains(id)) {
+        qDebug("Downloading %s...", qUtf8Printable(downloadRequest.url()));
+        if (isSequentialService)
+            m_busyServices.insert(id);
+        return new DownloadHandler {
+            m_networkManager.get(request), this, downloadRequest};
+    }
+
+    auto *downloadHandler = new DownloadHandler {nullptr, this, downloadRequest};
+    connect(downloadHandler, &DownloadHandler::destroyed, this, [this, id, downloadHandler]()
+    {
+        m_waitingJobs[id].removeOne(downloadHandler);
+    });
+    m_waitingJobs[id].enqueue(downloadHandler);
+    return downloadHandler;
+}
+
+void Net::DownloadManager::registerSequentialService(const Net::ServiceID &serviceID)
+{
+    m_sequentialServices.insert(serviceID);
+}
+
+QList<QNetworkCookie> Net::DownloadManager::cookiesForUrl(const QUrl &url) const
 {
     return m_networkManager.cookieJar()->cookiesForUrl(url);
 }
 
-bool DownloadManager::setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url)
+bool Net::DownloadManager::setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url)
 {
     return m_networkManager.cookieJar()->setCookiesFromUrl(cookieList, url);
 }
 
-QList<QNetworkCookie> DownloadManager::allCookies() const
+QList<QNetworkCookie> Net::DownloadManager::allCookies() const
 {
     return static_cast<NetworkCookieJar *>(m_networkManager.cookieJar())->allCookies();
 }
 
-void DownloadManager::setAllCookies(const QList<QNetworkCookie> &cookieList)
+void Net::DownloadManager::setAllCookies(const QList<QNetworkCookie> &cookieList)
 {
     static_cast<NetworkCookieJar *>(m_networkManager.cookieJar())->setAllCookies(cookieList);
 }
 
-bool DownloadManager::deleteCookie(const QNetworkCookie &cookie)
+bool Net::DownloadManager::deleteCookie(const QNetworkCookie &cookie)
 {
     return static_cast<NetworkCookieJar *>(m_networkManager.cookieJar())->deleteCookie(cookie);
 }
 
-void DownloadManager::applyProxySettings()
+void Net::DownloadManager::applyProxySettings()
 {
     auto proxyManager = ProxyConfigurationManager::instance();
     ProxyConfiguration proxyConfig = proxyManager->proxyConfiguration();
@@ -210,7 +234,7 @@ void DownloadManager::applyProxySettings()
         }
         // Authentication?
         if (proxyManager->isAuthenticationRequired()) {
-            qDebug("Proxy requires authentication, authenticating");
+            qDebug("Proxy requires authentication, authenticating...");
             proxy.setUser(proxyConfig.username);
             proxy.setPassword(proxyConfig.password);
         }
@@ -222,8 +246,23 @@ void DownloadManager::applyProxySettings()
     m_networkManager.setProxy(proxy);
 }
 
+void Net::DownloadManager::handleReplyFinished(QNetworkReply *reply)
+{
+    const ServiceID id = ServiceID::fromURL(reply->url());
+    auto waitingJobsIter = m_waitingJobs.find(id);
+    if ((waitingJobsIter == m_waitingJobs.end()) || waitingJobsIter.value().isEmpty()) {
+        m_busyServices.remove(id);
+        return;
+    }
+
+    DownloadHandler *handler = waitingJobsIter.value().dequeue();
+    qDebug("Downloading %s...", qUtf8Printable(handler->m_downloadRequest.url()));
+    handler->assignNetworkReply(m_networkManager.get(createNetworkRequest(handler->m_downloadRequest)));
+    handler->disconnect(this);
+}
+
 #ifndef QT_NO_OPENSSL
-void DownloadManager::ignoreSslErrors(QNetworkReply *reply, const QList<QSslError> &errors)
+void Net::DownloadManager::ignoreSslErrors(QNetworkReply *reply, const QList<QSslError> &errors)
 {
     Q_UNUSED(errors)
     // Ignore all SSL errors
@@ -290,3 +329,18 @@ Net::DownloadRequest &Net::DownloadRequest::handleRedirectToMagnet(bool value)
     m_handleRedirectToMagnet = value;
     return *this;
 }
+
+Net::ServiceID Net::ServiceID::fromURL(const QUrl &url)
+{
+    return {url.host(), url.port(80)};
+}
+
+uint Net::qHash(const ServiceID &serviceID, uint seed)
+{
+    return ::qHash(serviceID.hostName, seed) ^ serviceID.port;
+}
+
+bool Net::operator==(const ServiceID &lhs, const ServiceID &rhs)
+{
+    return ((lhs.hostName == rhs.hostName) && (lhs.port == rhs.port));
+}

src/base/net/downloadmanager.h

@@ -1,6 +1,6 @@
 /*
  * Bittorrent Client using Qt and libtorrent.
- * Copyright (C) 2015  Vladimir Golovnev <glassez@yandex.ru>
+ * Copyright (C) 2015, 2018  Vladimir Golovnev <glassez@yandex.ru>
  * Copyright (C) 2006  Christophe Dumez <chris@qbittorrent.org>
  *
  * This program is free software; you can redistribute it and/or
@@ -30,8 +30,12 @@
 #ifndef NET_DOWNLOADMANAGER_H
 #define NET_DOWNLOADMANAGER_H
 
-#include <QObject>
+#include <QHash>
 #include <QNetworkAccessManager>
+#include <QNetworkRequest>
+#include <QObject>
+#include <QQueue>
+#include <QSet>
 
 class QNetworkReply;
 class QNetworkCookie;
@@ -71,6 +75,14 @@ namespace Net
         bool m_handleRedirectToMagnet = false;
     };
 
+    struct ServiceID
+    {
+        QString hostName;
+        int port;
+
+        static ServiceID fromURL(const QUrl &url);
+    };
+
     class DownloadManager : public QObject
     {
         Q_OBJECT
@@ -83,6 +95,9 @@ namespace Net
 
         DownloadHandler *downloadUrl(const QString &url, bool saveToFile = false, qint64 limit = 0, bool handleRedirectToMagnet = false, const QString &userAgent = "");
         DownloadHandler *download(const DownloadRequest &downloadRequest);
+
+        void registerSequentialService(const ServiceID &serviceID);
+
         QList<QNetworkCookie> cookiesForUrl(const QUrl &url) const;
         bool setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url);
         QList<QNetworkCookie> allCookies() const;
@@ -98,10 +113,18 @@ namespace Net
         explicit DownloadManager(QObject *parent = nullptr);
 
         void applyProxySettings();
+        void handleReplyFinished(QNetworkReply *reply);
 
         static DownloadManager *m_instance;
         QNetworkAccessManager m_networkManager;
+
+        QSet<ServiceID> m_sequentialServices;
+        QSet<ServiceID> m_busyServices;
+        QHash<ServiceID, QQueue<DownloadHandler *>> m_waitingJobs;
     };
+
+    uint qHash(const ServiceID &serviceID, uint seed);
+    bool operator==(const ServiceID &lhs, const ServiceID &rhs);
 }
 
 #endif // NET_DOWNLOADMANAGER_H

src/base/rss/rss_feed.cpp

@@ -88,6 +88,8 @@ Feed::Feed(const QUuid &uid, const QString &url, const QString &path, Session *s
     else
         connect(m_session, &Session::processingStateChanged, this, &Feed::handleSessionProcessingEnabledChanged);
 
+    Net::DownloadManager::instance()->registerSequentialService(Net::ServiceID::fromURL(m_url));
+
     load();
 }