diff --git a/src/base/rss/private/rss_parser.cpp b/src/base/rss/private/rss_parser.cpp index 54bfdc125..d16b20f26 100644 --- a/src/base/rss/private/rss_parser.cpp +++ b/src/base/rss/private/rss_parser.cpp @@ -583,6 +583,7 @@ void Parser::parse_impl(const QByteArray &feedData) emit finished(m_result); m_result.articles.clear(); // clear articles only + m_articleIDs.clear(); } void Parser::parseRssArticle(QXmlStreamReader &xml) @@ -635,7 +636,7 @@ void Parser::parseRssArticle(QXmlStreamReader &xml) if (article[Article::KeyTorrentURL].toString().isEmpty()) article[Article::KeyTorrentURL] = altTorrentUrl; - m_result.articles.prepend(article); + addArticle(article); } void Parser::parseRSSChannel(QXmlStreamReader &xml) @@ -730,7 +731,7 @@ void Parser::parseAtomArticle(QXmlStreamReader &xml) } } - m_result.articles.prepend(article); + addArticle(article); } void Parser::parseAtomChannel(QXmlStreamReader &xml) @@ -760,3 +761,34 @@ void Parser::parseAtomChannel(QXmlStreamReader &xml) } } } + +void Parser::addArticle(QVariantHash article) +{ + QVariant &torrentURL = article[Article::KeyTorrentURL]; + if (torrentURL.toString().isEmpty()) + torrentURL = article[Article::KeyLink]; + + // If item does not have an ID, fall back to some other identifier. + QVariant &localId = article[Article::KeyId]; + if (localId.toString().isEmpty()) + localId = article.value(Article::KeyTorrentURL); + if (localId.toString().isEmpty()) + localId = article.value(Article::KeyTitle); + + if (localId.toString().isEmpty()) { + // The article could not be uniquely identified + // since it has no appropriate data. + // Just ignore it. + return; + } + + if (m_articleIDs.contains(localId.toString())) { + // The article could not be uniquely identified + // since the Feed has duplicate identifiers. + // Just ignore it. + return; + } + + m_articleIDs.insert(localId.toString()); + m_result.articles.prepend(article); +} diff --git a/src/base/rss/private/rss_parser.h b/src/base/rss/private/rss_parser.h index aeab57e3f..3d0ed4e72 100644 --- a/src/base/rss/private/rss_parser.h +++ b/src/base/rss/private/rss_parser.h @@ -31,6 +31,7 @@ #include #include +#include #include #include @@ -65,9 +66,11 @@ namespace RSS void parseRSSChannel(QXmlStreamReader &xml); void parseAtomArticle(QXmlStreamReader &xml); void parseAtomChannel(QXmlStreamReader &xml); + void addArticle(QVariantHash article); QString m_baseUrl; ParsingResult m_result; + QSet m_articleIDs; }; } } diff --git a/src/base/rss/rss_feed.cpp b/src/base/rss/rss_feed.cpp index 0680aa954..90bf70310 100644 --- a/src/base/rss/rss_feed.cpp +++ b/src/base/rss/rss_feed.cpp @@ -411,24 +411,10 @@ int Feed::updateArticles(const QList &loadedArticles) QVector newArticles; newArticles.reserve(loadedArticles.size()); for (QVariantHash article : loadedArticles) { - QVariant &torrentURL = article[Article::KeyTorrentURL]; - if (torrentURL.toString().isEmpty()) - torrentURL = article[Article::KeyLink]; - - // If item does not have an ID, fall back to some other identifier. - QVariant &localId = article[Article::KeyId]; - if (localId.toString().isEmpty()) - localId = article.value(Article::KeyTorrentURL); - if (localId.toString().isEmpty()) - localId = article.value(Article::KeyTitle); - - if (localId.toString().isEmpty()) - continue; - // If article has no publication date we use feed update time as a fallback. // To prevent processing of "out-of-limit" articles we must not assign dates // that are earlier than the dates of existing articles. - const Article *existingArticle = articleByGUID(localId.toString()); + const Article *existingArticle = articleByGUID(article[Article::KeyId].toString()); if (existingArticle) { dummyPubDate = existingArticle->date().addMSecs(-1); continue;