d4708/qBittorrent
1
0
Fork 0
mirror of https://github.com/d47081/qBittorrent.git synced 2025-01-11 07:18:08 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #6864 from glassez/rss

Browse Source 
Improve RSS Article handling
This commit is contained in:
Vladimir Golovnev 2017-06-02 19:47:08 +03:00 committed by GitHub
commit c503583046
5 changed files with 152 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
src/base/rss/private/rss_parser.cpp
View File

@ -39,6 +39,8 @@
#include <QVariant> #include <QVariant>
#include <QXmlStreamReader> #include <QXmlStreamReader>
#include "../rss_article.h"
namespace namespace
{ {
const char shortDay[][4] = { const char shortDay[][4] = {
@ -280,36 +282,41 @@ void Parser::parseRssArticle(QXmlStreamReader &xml)
while (!xml.atEnd()) { while (!xml.atEnd()) {
xml.readNext(); xml.readNext();
const QString name(xml.name().toString());
if(xml.isEndElement() && xml.name() == "item") if (xml.isEndElement() && (name == QLatin1String("item")))
break; break;
if (xml.isStartElement()) { if (xml.isStartElement()) {
if (xml.name() == "title") { const QString text(xml.readElementText().trimmed());
article["title"] = xml.readElementText().trimmed();
if (name == QLatin1String("title")) {
article[Article::KeyTitle] = text;
} }
else if (xml.name() == "enclosure") { else if (name == QLatin1String("enclosure")) {
if (xml.attributes().value("type") == "application/x-bittorrent") if (xml.attributes().value("type") == QLatin1String("application/x-bittorrent"))
article["torrent_url"] = xml.attributes().value("url").toString(); article[Article::KeyTorrentURL] = xml.attributes().value(QLatin1String("url")).toString();
} }
else if (xml.name() == "link") { else if (name == QLatin1String("link")) {
QString link = xml.readElementText().trimmed(); if (text.startsWith(QLatin1String("magnet:"), Qt::CaseInsensitive))
if (link.startsWith("magnet:", Qt::CaseInsensitive)) article[Article::KeyTorrentURL] = text; // magnet link instead of a news URL
article["torrent_url"] = link; // magnet link instead of a news URL
else else
article["news_link"] = link; article[Article::KeyLink] = text;
} }
else if (xml.name() == "description") { else if (name == QLatin1String("description")) {
article["description"] = xml.readElementText().trimmed(); article[Article::KeyDescription] = text;
} }
else if (xml.name() == "pubDate") { else if (name == QLatin1String("pubDate")) {
article["date"] = parseDate(xml.readElementText().trimmed()); article[Article::KeyDate] = parseDate(text);
} }
else if (xml.name() == "author") { else if (name == QLatin1String("author")) {
article["author"] = xml.readElementText().trimmed(); article[Article::KeyAuthor] = text;
} }
else if (xml.name() == "guid") { else if (name == QLatin1String("guid")) {
article["id"] = xml.readElementText().trimmed(); article[Article::KeyId] = text;
}
else {
article[name] = text;
} }
} }
} }
@ -353,33 +360,36 @@ void Parser::parseAtomArticle(QXmlStreamReader &xml)
while (!xml.atEnd()) { while (!xml.atEnd()) {
xml.readNext(); xml.readNext();
const QString name(xml.name().toString());
if (xml.isEndElement() && (xml.name() == "entry")) if (xml.isEndElement() && (name == QLatin1String("entry")))
break; break;
if (xml.isStartElement()) { if (xml.isStartElement()) {
if (xml.name() == "title") { const QString text(xml.readElementText().trimmed());
article["title"] = xml.readElementText().trimmed();
}
else if (xml.name() == "link") {
QString link = (xml.attributes().isEmpty()
? xml.readElementText().trimmed()
: xml.attributes().value("href").toString());
if (link.startsWith("magnet:", Qt::CaseInsensitive)) if (name == QLatin1String("title")) {
article["torrent_url"] = link; // magnet link instead of a news URL article[Article::KeyTitle] = text;
}
else if (name == QLatin1String("link")) {
QString link = (xml.attributes().isEmpty()
? text
: xml.attributes().value(QLatin1String("href")).toString());
if (link.startsWith(QLatin1String("magnet:"), Qt::CaseInsensitive))
article[Article::KeyTorrentURL] = link; // magnet link instead of a news URL
else else
// Atom feeds can have relative links, work around this and // Atom feeds can have relative links, work around this and
// take the stress of figuring article full URI from UI // take the stress of figuring article full URI from UI
// Assemble full URI // Assemble full URI
article["news_link"] = (m_baseUrl.isEmpty() ? link : m_baseUrl + link); article[Article::KeyLink] = (m_baseUrl.isEmpty() ? link : m_baseUrl + link);
} }
else if ((xml.name() == "summary") || (xml.name() == "content")){ else if ((name == QLatin1String("summary")) || (name == QLatin1String("content"))){
if (doubleContent) { // Duplicate content -> ignore if (doubleContent) { // Duplicate content -> ignore
xml.readNext(); xml.readNext();
while ((xml.name() != "summary") && (xml.name() != "content")) while ((xml.name() != QLatin1String("summary")) && (xml.name() != QLatin1String("content")))
xml.readNext(); xml.readNext();
continue; continue;
@ -389,25 +399,28 @@ void Parser::parseAtomArticle(QXmlStreamReader &xml)
// Actually works great for non-broken content too // Actually works great for non-broken content too
QString feedText = xml.readElementText(QXmlStreamReader::IncludeChildElements); QString feedText = xml.readElementText(QXmlStreamReader::IncludeChildElements);
if (!feedText.isEmpty()) if (!feedText.isEmpty())
article["description"] = feedText.trimmed(); article[Article::KeyDescription] = feedText.trimmed();
doubleContent = true; doubleContent = true;
} }
else if (xml.name() == "updated") { else if (name == QLatin1String("updated")) {
// ATOM uses standard compliant date, don't do fancy stuff // ATOM uses standard compliant date, don't do fancy stuff
QDateTime articleDate = QDateTime::fromString(xml.readElementText().trimmed(), Qt::ISODate); QDateTime articleDate = QDateTime::fromString(text, Qt::ISODate);
article["date"] = (articleDate.isValid() ? articleDate : QDateTime::currentDateTime()); article[Article::KeyDate] = (articleDate.isValid() ? articleDate : QDateTime::currentDateTime());
} }
else if (xml.name() == "author") { else if (name == QLatin1String("author")) {
xml.readNext(); xml.readNext();
while (xml.name() != "author") { while (xml.name() != QLatin1String("author")) {
if (xml.name() == "name") if (xml.name() == QLatin1String("name"))
article["author"] = xml.readElementText().trimmed(); article[Article::KeyAuthor] = xml.readElementText().trimmed();
xml.readNext(); xml.readNext();
} }
} }
else if (xml.name() == "id") { else if (name == QLatin1String("id")) {
article["id"] = xml.readElementText().trimmed(); article[Article::KeyId] = text;
}
else {
article[name] = text;
} }
} }
} }

121
src/base/rss/rss_article.cpp
View File

@ -30,38 +30,53 @@
#include "rss_article.h" #include "rss_article.h"
#include <stdexcept>
#include <QJsonObject> #include <QJsonObject>
#include <QVariant> #include <QVariant>
#include "rss_feed.h" #include "rss_feed.h"
const QString Str_Id(QStringLiteral("id"));
const QString Str_Date(QStringLiteral("date"));
const QString Str_Title(QStringLiteral("title"));
const QString Str_Author(QStringLiteral("author"));
const QString Str_Description(QStringLiteral("description"));
const QString Str_TorrentURL(QStringLiteral("torrentURL"));
const QString Str_Torrent_Url(QStringLiteral("torrent_url"));
const QString Str_Link(QStringLiteral("link"));
const QString Str_News_Link(QStringLiteral("news_link"));
const QString Str_IsRead(QStringLiteral("isRead"));
const QString Str_Read(QStringLiteral("read"));
using namespace RSS; using namespace RSS;
Article::Article(Feed *feed, QString guid, QDateTime date, QString title, QString author const QString Article::KeyId(QStringLiteral("id"));
, QString description, QString torrentUrl, QString link, bool isRead) const QString Article::KeyDate(QStringLiteral("date"));
const QString Article::KeyTitle(QStringLiteral("title"));
const QString Article::KeyAuthor(QStringLiteral("author"));
const QString Article::KeyDescription(QStringLiteral("description"));
const QString Article::KeyTorrentURL(QStringLiteral("torrentURL"));
const QString Article::KeyLink(QStringLiteral("link"));
const QString Article::KeyIsRead(QStringLiteral("isRead"));
Article::Article(Feed *feed, const QVariantHash &varHash)
: QObject(feed) : QObject(feed)
, m_feed(feed) , m_feed(feed)
, m_guid(guid) , m_guid(varHash.value(KeyId).toString())
, m_date(date) , m_date(varHash.value(KeyDate).toDateTime())
, m_title(title) , m_title(varHash.value(KeyTitle).toString())
, m_author(author) , m_author(varHash.value(KeyAuthor).toString())
, m_description(description) , m_description(varHash.value(KeyDescription).toString())
, m_torrentURL(torrentUrl) , m_torrentURL(varHash.value(KeyTorrentURL).toString())
, m_link(link) , m_link(varHash.value(KeyLink).toString())
, m_isRead(isRead) , m_isRead(varHash.value(KeyIsRead, false).toBool())
, m_data(varHash)
{ {
// If item does not have a guid, fall back to some other identifier
if (m_guid.isEmpty())
m_guid = varHash.value(KeyTorrentURL).toString();
if (m_guid.isEmpty())
m_guid = varHash.value(KeyTitle).toString();
if (m_guid.isEmpty())
throw std::runtime_error("Bad RSS Article data");
m_data[KeyId] = m_guid;
}
Article::Article(Feed *feed, const QJsonObject &jsonObj)
: Article(feed, jsonObj.toVariantHash())
{
// JSON object store DateTime as string so we need to convert it
m_date = QDateTime::fromString(jsonObj.value(KeyDate).toString(), Qt::RFC2822Date);
m_data[KeyDate] = m_date;
} }
QString Article::guid() const QString Article::guid() const
@ -104,26 +119,27 @@ bool Article::isRead() const
return m_isRead; return m_isRead;
} }
QVariantHash Article::data() const
{
return m_data;
}
void Article::markAsRead() void Article::markAsRead()
{ {
if (!m_isRead) { if (!m_isRead) {
m_isRead = true; m_isRead = true;
m_data[KeyIsRead] = m_isRead;
emit read(this); emit read(this);
} }
} }
QJsonObject Article::toJsonObject() const QJsonObject Article::toJsonObject() const
{ {
return { auto jsonObj = QJsonObject::fromVariantHash(m_data);
{Str_Id, m_guid}, // JSON object doesn't support DateTime so we need to convert it
{Str_Date, m_date.toString(Qt::RFC2822Date)}, jsonObj[KeyDate] = m_date.toString(Qt::RFC2822Date);
{Str_Title, m_title},
{Str_Author, m_author}, return jsonObj;
{Str_Description, m_description},
{Str_TorrentURL, m_torrentURL},
{Str_Link, m_link},
{Str_IsRead, m_isRead}
};
} }
bool Article::articleDateRecentThan(Article *article, const QDateTime &date) bool Article::articleDateRecentThan(Article *article, const QDateTime &date)
@ -131,47 +147,6 @@ bool Article::articleDateRecentThan(Article *article, const QDateTime &date)
return article->date() > date; return article->date() > date;
} }
Article *Article::fromJsonObject(Feed *feed, const QJsonObject &jsonObj)
{
QString guid = jsonObj.value(Str_Id).toString();
// If item does not have a guid, fall back to some other identifier
if (guid.isEmpty())
guid = jsonObj.value(Str_Torrent_Url).toString();
if (guid.isEmpty())
guid = jsonObj.value(Str_Title).toString();
if (guid.isEmpty()) return nullptr;
return new Article(
feed, guid
, QDateTime::fromString(jsonObj.value(Str_Date).toString(), Qt::RFC2822Date)
, jsonObj.value(Str_Title).toString()
, jsonObj.value(Str_Author).toString()
, jsonObj.value(Str_Description).toString()
, jsonObj.value(Str_TorrentURL).toString()
, jsonObj.value(Str_Link).toString()
, jsonObj.value(Str_IsRead).toBool(false));
}
Article *Article::fromVariantHash(Feed *feed, const QVariantHash &varHash)
{
QString guid = varHash[Str_Id].toString();
// If item does not have a guid, fall back to some other identifier
if (guid.isEmpty())
guid = varHash.value(Str_Torrent_Url).toString();
if (guid.isEmpty())
guid = varHash.value(Str_Title).toString();
if (guid.isEmpty()) return nullptr;
return new Article(feed, guid
, varHash.value(Str_Date).toDateTime()
, varHash.value(Str_Title).toString()
, varHash.value(Str_Author).toString()
, varHash.value(Str_Description).toString()
, varHash.value(Str_Torrent_Url).toString()
, varHash.value(Str_News_Link).toString()
, varHash.value(Str_Read, false).toBool());
}
Feed *Article::feed() const Feed *Article::feed() const
{ {
return m_feed; return m_feed;

18
src/base/rss/rss_article.h
View File

@ -33,6 +33,7 @@
#include <QDateTime> #include <QDateTime>
#include <QObject> #include <QObject>
#include <QString> #include <QString>
#include <QVariantHash>
namespace RSS namespace RSS
{ {
@ -45,12 +46,19 @@ namespace RSS
friend class Feed; friend class Feed;
Article(Feed *feed, QString guid, QDateTime date, QString title, QString author Article(Feed *feed, const QVariantHash &varHash);
, QString description, QString torrentUrl, QString link, bool isRead = false); Article(Feed *feed, const QJsonObject &jsonObj);
static Article *fromJsonObject(Feed *feed, const QJsonObject &jsonObj);
static Article *fromVariantHash(Feed *feed, const QVariantHash &varHash);
public: public:
static const QString KeyId;
static const QString KeyDate;
static const QString KeyTitle;
static const QString KeyAuthor;
static const QString KeyDescription;
static const QString KeyTorrentURL;
static const QString KeyLink;
static const QString KeyIsRead;
Feed *feed() const; Feed *feed() const;
QString guid() const; QString guid() const;
QDateTime date() const; QDateTime date() const;
@ -60,6 +68,7 @@ namespace RSS
QString torrentUrl() const; QString torrentUrl() const;
QString link() const; QString link() const;
bool isRead() const; bool isRead() const;
QVariantHash data() const;
void markAsRead(); void markAsRead();
@ -80,5 +89,6 @@ namespace RSS
QString m_torrentURL; QString m_torrentURL;
QString m_link; QString m_link;
bool m_isRead = false; bool m_isRead = false;
QVariantHash m_data;
}; };
} }

33
src/base/rss/rss_autodownloader.cpp
View File

@ -36,6 +36,7 @@
#include <QSaveFile> #include <QSaveFile>
#include <QThread> #include <QThread>
#include <QTimer> #include <QTimer>
#include <QVariant>
#include "../bittorrent/magneturi.h" #include "../bittorrent/magneturi.h"
#include "../bittorrent/session.h" #include "../bittorrent/session.h"
@ -54,10 +55,7 @@
struct ProcessingJob struct ProcessingJob
{ {
QString feedURL; QString feedURL;
QString articleGUID; QVariantHash articleData;
QString articleTitle;
QDateTime articleDate;
QString torrentURL;
}; };
const QString ConfFolderName(QStringLiteral("rss")); const QString ConfFolderName(QStringLiteral("rss"));
@ -191,8 +189,8 @@ void AutoDownloader::handleTorrentDownloadFinished(const QString &url)
auto job = m_waitingJobs.take(url); auto job = m_waitingJobs.take(url);
if (!job) return; if (!job) return;
if (auto feed = Session::instance()->feedByURL(job->feedURL)) if (Feed *feed = Session::instance()->feedByURL(job->feedURL))
if (auto article = feed->articleByGUID(job->articleGUID)) if (Article *article = feed->articleByGUID(job->articleData.value(Article::KeyId).toString()))
article->markAsRead(); article->markAsRead();
} }
@ -220,10 +218,7 @@ void AutoDownloader::addJobForArticle(Article *article)
QSharedPointer<ProcessingJob> job(new ProcessingJob); QSharedPointer<ProcessingJob> job(new ProcessingJob);
job->feedURL = article->feed()->url(); job->feedURL = article->feed()->url();
job->articleGUID = article->guid(); job->articleData = article->data();
job->articleTitle = article->title();
job->articleDate = article->date();
job->torrentURL = torrentURL;
m_processingQueue.append(job); m_processingQueue.append(job);
if (!m_processingTimer->isActive()) if (!m_processingTimer->isActive())
m_processingTimer->start(); m_processingTimer->start();
@ -234,17 +229,18 @@ void AutoDownloader::processJob(const QSharedPointer<ProcessingJob> &job)
for (AutoDownloadRule &rule: m_rules) { for (AutoDownloadRule &rule: m_rules) {
if (!rule.isEnabled()) continue; if (!rule.isEnabled()) continue;
if (!rule.feedURLs().contains(job->feedURL)) continue; if (!rule.feedURLs().contains(job->feedURL)) continue;
if (!rule.matches(job->articleTitle)) continue; if (!rule.matches(job->articleData.value(Article::KeyTitle).toString())) continue;
auto articleDate = job->articleData.value(Article::KeyDate).toDateTime();
// if rule is in ignoring state do nothing with matched torrent // if rule is in ignoring state do nothing with matched torrent
if (rule.ignoreDays() > 0) { if (rule.ignoreDays() > 0) {
if (rule.lastMatch().isValid()) { if (rule.lastMatch().isValid()) {
if (job->articleDate < rule.lastMatch().addDays(rule.ignoreDays())) if (articleDate < rule.lastMatch().addDays(rule.ignoreDays()))
return; return;
} }
} }
rule.setLastMatch(job->articleDate); rule.setLastMatch(articleDate);
m_dirty = true; m_dirty = true;
storeDeferred(); storeDeferred();
@ -252,18 +248,19 @@ void AutoDownloader::processJob(const QSharedPointer<ProcessingJob> &job)
params.savePath = rule.savePath(); params.savePath = rule.savePath();
params.category = rule.assignedCategory(); params.category = rule.assignedCategory();
params.addPaused = rule.addPaused(); params.addPaused = rule.addPaused();
BitTorrent::Session::instance()->addTorrent(job->torrentURL, params); auto torrentURL = job->articleData.value(Article::KeyTorrentURL).toString();
BitTorrent::Session::instance()->addTorrent(torrentURL, params);
if (BitTorrent::MagnetUri(job->torrentURL).isValid()) { if (BitTorrent::MagnetUri(torrentURL).isValid()) {
if (auto feed = Session::instance()->feedByURL(job->feedURL)) { if (Feed *feed = Session::instance()->feedByURL(job->feedURL)) {
if (auto article = feed->articleByGUID(job->articleGUID)) if (Article *article = feed->articleByGUID(job->articleData.value(Article::KeyId).toString()))
article->markAsRead(); article->markAsRead();
} }
} }
else { else {
// waiting for torrent file downloading // waiting for torrent file downloading
// normalize URL string via QUrl since DownloadManager do it // normalize URL string via QUrl since DownloadManager do it
m_waitingJobs.insert(QUrl(job->torrentURL).toString(), job); m_waitingJobs.insert(QUrl(torrentURL).toString(), job);
} }
return; return;

24
src/base/rss/rss_feed.cpp
View File

@ -207,13 +207,14 @@ void Feed::handleParsingFinished(const RSS::Private::ParsingResult &result)
m_lastBuildDate = result.lastBuildDate; m_lastBuildDate = result.lastBuildDate;
foreach (const QVariantHash &varHash, result.articles) { foreach (const QVariantHash &varHash, result.articles) {
auto article = Article::fromVariantHash(this, varHash); try {
if (article) { auto article = new Article(this, varHash);
if (!addArticle(article)) if (!addArticle(article))
delete article; delete article;
else else
m_dirty = true; m_dirty = true;
} }
catch (const std::runtime_error&) {}
} }
store(); store();
@ -272,10 +273,13 @@ void Feed::loadArticles(const QByteArray &data)
continue; continue;
} }
auto article = Article::fromJsonObject(this, jsonVal.toObject()); try {
if (article && !addArticle(article)) auto article = new Article(this, jsonVal.toObject());
if (!addArticle(article))
delete article; delete article;
} }
catch (const std::runtime_error&) {}
}
} }
void Feed::loadArticlesLegacy() void Feed::loadArticlesLegacy()
@ -284,10 +288,18 @@ void Feed::loadArticlesLegacy()
QVariantHash allOldItems = qBTRSSFeeds->value("old_items").toHash(); QVariantHash allOldItems = qBTRSSFeeds->value("old_items").toHash();
foreach (const QVariant &var, allOldItems.value(m_url).toList()) { foreach (const QVariant &var, allOldItems.value(m_url).toList()) {
auto article = Article::fromVariantHash(this, var.toHash()); auto hash = var.toHash();
if (article && !addArticle(article)) // update legacy keys
hash[Article::KeyLink] = hash.take(QLatin1String("news_link"));
hash[Article::KeyTorrentURL] = hash.take(QLatin1String("torrent_url"));
hash[Article::KeyIsRead] = hash.take(QLatin1String("read"));
try {
auto article = new Article(this, hash);
if (!addArticle(article))
delete article; delete article;
} }
catch (const std::runtime_error&) {}
}
} }
void Feed::store() void Feed::store()