d4708
/
qBittorrent
mirror of https://github.com/d47081/qBittorrent.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Improve RSS parsing logic.

adaptive-webui-19844
Vladimir Golovnev (Glassez) 9 years ago committed by Vladimir Golovnev (qlassez)
parent
28ed981082
commit
6662081044
6 changed files with 259 additions and 350 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 487
      src/base/rss/private/rssparser.cpp
  2. 49
      src/base/rss/private/rssparser.h
  3. 38
      src/base/rss/rssfeed.cpp
  4. 13
      src/base/rss/rssfeed.h
  5. 11
      src/base/rss/rssmanager.cpp
  6. 11
      src/base/rss/rssmanager.h

487
src/base/rss/private/rssparser.cpp
Unescape View File

@ -1,6 +1,7 @@
/* /*
* Bittorrent Client using Qt4 and libtorrent. * Bittorrent Client using Qt and libtorrent.
* Copyright (C) 2012 Christophe Dumez * Copyright (C) 2015 Vladimir Golovnev <glassez@yandex.ru>
* Copyright (C) 2012 Christophe Dumez <chris@qbittorrent.org>
* *
* This program is free software; you can redistribute it and/or * This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License * modify it under the terms of the GNU General Public License
@ -30,248 +31,226 @@
#include <QDebug> #include <QDebug>
#include <QDateTime> #include <QDateTime>
#include <QFile>
#include <QRegExp> #include <QRegExp>
#include <QStringList> #include <QStringList>
#include <QVariant> #include <QVariant>
#include <QXmlStreamReader> #include <QXmlStreamReader>
#include "base/utils/fs.h"
#include "rssparser.h" #include "rssparser.h"
namespace Rss namespace
{ {
namespace Private const char shortDay[][4] = {
"Mon", "Tue", "Wed",
"Thu", "Fri", "Sat",
"Sun"
};
const char longDay[][10] = {
"Monday", "Tuesday", "Wednesday",
"Thursday", "Friday", "Saturday",
"Sunday"
};
const char shortMonth[][4] = {
"Jan", "Feb", "Mar", "Apr",
"May", "Jun", "Jul", "Aug",
"Sep", "Oct", "Nov", "Dec"
};
// Ported to Qt from KDElibs4
QDateTime parseDate(const QString &string)
{ {
struct ParsingJob const QString str = string.trimmed();
{ if (str.isEmpty())
QString feedUrl;
QByteArray feedData;
};
}
}
static const char shortDay[][4] = {
"Mon", "Tue", "Wed",
"Thu", "Fri", "Sat",
"Sun"
};
static const char longDay[][10] = {
"Monday", "Tuesday", "Wednesday",
"Thursday", "Friday", "Saturday",
"Sunday"
};
static const char shortMonth[][4] = {
"Jan", "Feb", "Mar", "Apr",
"May", "Jun", "Jul", "Aug",
"Sep", "Oct", "Nov", "Dec"
};
using namespace Rss::Private;
Parser::Parser(QObject *parent)
: QThread(parent)
, m_running(true)
{
start();
}
Parser::~Parser()
{
m_running = false;
m_waitCondition.wakeOne();
wait();
}
// Ported to Qt from KDElibs4
QDateTime Parser::parseDate(const QString &string)
{
const QString str = string.trimmed();
if (str.isEmpty())
return QDateTime::currentDateTime();
int nyear = 6; // indexes within string to values
int nmonth = 4;
int nday = 2;
int nwday = 1;
int nhour = 7;
int nmin = 8;
int nsec = 9;
// Also accept obsolete form "Weekday, DD-Mon-YY HH:MM:SS ±hhmm"
QRegExp rx("^(?:([A-Z][a-z]+),\\s*)?(\\d{1,2})(\\s+|-)([^-\\s]+)(\\s+|-)(\\d{2,4})\\s+(\\d\\d):(\\d\\d)(?::(\\d\\d))?\\s+(\\S+)$");
QStringList parts;
if (!str.indexOf(rx)) {
// Check that if date has '-' separators, both separators are '-'.
parts = rx.capturedTexts();
bool h1 = (parts[3] == QLatin1String("-"));
bool h2 = (parts[5] == QLatin1String("-"));
if (h1 != h2)
return QDateTime::currentDateTime();
}
else {
// Check for the obsolete form "Wdy Mon DD HH:MM:SS YYYY"
rx = QRegExp("^([A-Z][a-z]+)\\s+(\\S+)\\s+(\\d\\d)\\s+(\\d\\d):(\\d\\d):(\\d\\d)\\s+(\\d\\d\\d\\d)$");
if (str.indexOf(rx))
return QDateTime::currentDateTime(); return QDateTime::currentDateTime();
nyear = 7;
nmonth = 2;
nday = 3;
nwday = 1;
nhour = 4;
nmin = 5;
nsec = 6;
parts = rx.capturedTexts();
}
bool ok[4]; int nyear = 6; // indexes within string to values
const int day = parts[nday].toInt(&ok[0]); int nmonth = 4;
int year = parts[nyear].toInt(&ok[1]); int nday = 2;
const int hour = parts[nhour].toInt(&ok[2]); int nwday = 1;
const int minute = parts[nmin].toInt(&ok[3]); int nhour = 7;
if (!ok[0] || !ok[1] || !ok[2] || !ok[3]) int nmin = 8;
return QDateTime::currentDateTime(); int nsec = 9;
// Also accept obsolete form "Weekday, DD-Mon-YY HH:MM:SS ±hhmm"
int second = 0; QRegExp rx("^(?:([A-Z][a-z]+),\\s*)?(\\d{1,2})(\\s+|-)([^-\\s]+)(\\s+|-)(\\d{2,4})\\s+(\\d\\d):(\\d\\d)(?::(\\d\\d))?\\s+(\\S+)$");
if (!parts[nsec].isEmpty()) { QStringList parts;
second = parts[nsec].toInt(&ok[0]); if (!str.indexOf(rx)) {
if (!ok[0]) // Check that if date has '-' separators, both separators are '-'.
parts = rx.capturedTexts();
bool h1 = (parts[3] == QLatin1String("-"));
bool h2 = (parts[5] == QLatin1String("-"));
if (h1 != h2)
return QDateTime::currentDateTime();
}
else {
// Check for the obsolete form "Wdy Mon DD HH:MM:SS YYYY"
rx = QRegExp("^([A-Z][a-z]+)\\s+(\\S+)\\s+(\\d\\d)\\s+(\\d\\d):(\\d\\d):(\\d\\d)\\s+(\\d\\d\\d\\d)$");
if (str.indexOf(rx))
return QDateTime::currentDateTime();
nyear = 7;
nmonth = 2;
nday = 3;
nwday = 1;
nhour = 4;
nmin = 5;
nsec = 6;
parts = rx.capturedTexts();
}
bool ok[4];
const int day = parts[nday].toInt(&ok[0]);
int year = parts[nyear].toInt(&ok[1]);
const int hour = parts[nhour].toInt(&ok[2]);
const int minute = parts[nmin].toInt(&ok[3]);
if (!ok[0] || !ok[1] || !ok[2] || !ok[3])
return QDateTime::currentDateTime(); return QDateTime::currentDateTime();
}
bool leapSecond = (second == 60); int second = 0;
if (leapSecond) if (!parts[nsec].isEmpty()) {
second = 59; // apparently a leap second - validate below, once time zone is known second = parts[nsec].toInt(&ok[0]);
int month = 0; if (!ok[0])
for ( ; (month < 12) && (parts[nmonth] != shortMonth[month]); ++month); return QDateTime::currentDateTime();
int dayOfWeek = -1; }
if (!parts[nwday].isEmpty()) {
// Look up the weekday name
while (++dayOfWeek < 7 && (shortDay[dayOfWeek] != parts[nwday]));
if (dayOfWeek >= 7)
for (dayOfWeek = 0; dayOfWeek < 7 && (longDay[dayOfWeek] != parts[nwday]); ++dayOfWeek);
}
// if (month >= 12 || dayOfWeek >= 7 bool leapSecond = (second == 60);
// || (dayOfWeek < 0 && format == RFCDateDay)) if (leapSecond)
// return QDateTime; second = 59; // apparently a leap second - validate below, once time zone is known
int i = parts[nyear].size(); int month = 0;
if (i < 4) { for ( ; (month < 12) && (parts[nmonth] != shortMonth[month]); ++month);
// It's an obsolete year specification with less than 4 digits int dayOfWeek = -1;
year += (i == 2 && year < 50) ? 2000 : 1900; if (!parts[nwday].isEmpty()) {
} // Look up the weekday name
while (++dayOfWeek < 7 && (shortDay[dayOfWeek] != parts[nwday]));
if (dayOfWeek >= 7)
for (dayOfWeek = 0; dayOfWeek < 7 && (longDay[dayOfWeek] != parts[nwday]); ++dayOfWeek);
}
// Parse the UTC offset part // if (month >= 12 || dayOfWeek >= 7
int offset = 0; // set default to '-0000' // || (dayOfWeek < 0 && format == RFCDateDay))
bool negOffset = false; // return QDateTime;
if (parts.count() > 10) { int i = parts[nyear].size();
rx = QRegExp("^([+-])(\\d\\d)(\\d\\d)$"); if (i < 4) {
if (!parts[10].indexOf(rx)) { // It's an obsolete year specification with less than 4 digits
// It's a UTC offset ±hhmm year += (i == 2 && year < 50) ? 2000 : 1900;
parts = rx.capturedTexts();
offset = parts[2].toInt(&ok[0]) * 3600;
int offsetMin = parts[3].toInt(&ok[1]);
if (!ok[0] || !ok[1] || offsetMin > 59)
return QDateTime();
offset += offsetMin * 60;
negOffset = (parts[1] == QLatin1String("-"));
if (negOffset)
offset = -offset;
} }
else {
// Check for an obsolete time zone name // Parse the UTC offset part
QByteArray zone = parts[10].toLatin1(); int offset = 0; // set default to '-0000'
if (zone.length() == 1 && isalpha(zone[0]) && toupper(zone[0]) != 'J') { bool negOffset = false;
negOffset = true; // military zone: RFC 2822 treats as '-0000' if (parts.count() > 10) {
rx = QRegExp("^([+-])(\\d\\d)(\\d\\d)$");
if (!parts[10].indexOf(rx)) {
// It's a UTC offset ±hhmm
parts = rx.capturedTexts();
offset = parts[2].toInt(&ok[0]) * 3600;
int offsetMin = parts[3].toInt(&ok[1]);
if (!ok[0] || !ok[1] || offsetMin > 59)
return QDateTime();
offset += offsetMin * 60;
negOffset = (parts[1] == QLatin1String("-"));
if (negOffset)
offset = -offset;
} }
else if (zone != "UT" && zone != "GMT") { // treated as '+0000' else {
offset = (zone == "EDT") // Check for an obsolete time zone name
? -4 * 3600 QByteArray zone = parts[10].toLatin1();
: ((zone == "EST") || (zone == "CDT")) if (zone.length() == 1 && isalpha(zone[0]) && toupper(zone[0]) != 'J') {
? -5 * 3600 negOffset = true; // military zone: RFC 2822 treats as '-0000'
: ((zone == "CST") || (zone == "MDT")) }
? -6 * 3600 else if (zone != "UT" && zone != "GMT") { // treated as '+0000'
: (zone == "MST" || zone == "PDT") offset = (zone == "EDT")
? -7 * 3600 ? -4 * 3600
: (zone == "PST") : ((zone == "EST") || (zone == "CDT"))
? -8 * 3600 ? -5 * 3600
: 0; : ((zone == "CST") || (zone == "MDT"))
if (!offset) { ? -6 * 3600
// Check for any other alphabetic time zone : (zone == "MST" || zone == "PDT")
bool nonalpha = false; ? -7 * 3600
for (int i = 0, end = zone.size(); (i < end) && !nonalpha; ++i) : (zone == "PST")
nonalpha = !isalpha(zone[i]); ? -8 * 3600
if (nonalpha) : 0;
return QDateTime(); if (!offset) {
// TODO: Attempt to recognize the time zone abbreviation? // Check for any other alphabetic time zone
negOffset = true; // unknown time zone: RFC 2822 treats as '-0000' bool nonalpha = false;
for (int i = 0, end = zone.size(); (i < end) && !nonalpha; ++i)
nonalpha = !isalpha(zone[i]);
if (nonalpha)
return QDateTime();
// TODO: Attempt to recognize the time zone abbreviation?
negOffset = true; // unknown time zone: RFC 2822 treats as '-0000'
}
} }
} }
} }
}
QDate qdate(year, month + 1, day); // convert date, and check for out-of-range QDate qdate(year, month + 1, day); // convert date, and check for out-of-range
if (!qdate.isValid()) if (!qdate.isValid())
return QDateTime::currentDateTime(); return QDateTime::currentDateTime();
QTime qTime(hour, minute, second);
QDateTime result(qdate, qTime, Qt::UTC);
if (offset)
result = result.addSecs(-offset);
if (!result.isValid())
return QDateTime::currentDateTime(); // invalid date/time
if (leapSecond) {
// Validate a leap second time. Leap seconds are inserted after 23:59:59 UTC.
// Convert the time to UTC and check that it is 00:00:00.
if ((hour*3600 + minute*60 + 60 - offset + 86400*5) % 86400) // (max abs(offset) is 100 hours)
return QDateTime::currentDateTime(); // the time isn't the last second of the day
}
return result; QTime qTime(hour, minute, second);
} QDateTime result(qdate, qTime, Qt::UTC);
if (offset)
result = result.addSecs(-offset);
if (!result.isValid())
return QDateTime::currentDateTime(); // invalid date/time
if (leapSecond) {
// Validate a leap second time. Leap seconds are inserted after 23:59:59 UTC.
// Convert the time to UTC and check that it is 00:00:00.
if ((hour*3600 + minute*60 + 60 - offset + 86400*5) % 86400) // (max abs(offset) is 100 hours)
return QDateTime::currentDateTime(); // the time isn't the last second of the day
}
void Parser::parseFeedData(const QString &feedUrl, const QByteArray &feedData) return result;
{
qDebug() << Q_FUNC_INFO << feedUrl;
m_mutex.lock();
ParsingJob job = { feedUrl, feedData };
m_queue.enqueue(job);
// Wake up thread.
if (m_queue.count() == 1) {
qDebug() << Q_FUNC_INFO << "Waking up thread";
m_waitCondition.wakeOne();
} }
m_mutex.unlock();
} }
void Parser::clearFeedData(const QString &feedUrl) using namespace Rss::Private;
{
m_mutex.lock();
m_lastBuildDates.remove(feedUrl);
m_mutex.unlock();
}
void Parser::run() // read and create items from a rss document
void Parser::parse(const QByteArray &feedData)
{ {
while (m_running) { qDebug() << Q_FUNC_INFO;
m_mutex.lock();
if (!m_queue.empty()) { QXmlStreamReader xml(feedData);
ParsingJob job = m_queue.dequeue(); bool foundChannel = false;
m_mutex.unlock(); while (xml.readNextStartElement()) {
parseFeed(job); if (xml.name() == "rss") {
// Find channels
while (xml.readNextStartElement()) {
if (xml.name() == "channel") {
parseRSSChannel(xml);
foundChannel = true;
break;
}
else {
qDebug() << "Skip rss item: " << xml.name();
xml.skipCurrentElement();
}
}
break;
}
else if (xml.name() == "feed") { // Atom feed
parseAtomChannel(xml);
foundChannel = true;
break;
} }
else { else {
qDebug() << Q_FUNC_INFO << "Thread is waiting."; qDebug() << "Skip root item: " << xml.name();
m_waitCondition.wait(&m_mutex); xml.skipCurrentElement();
qDebug() << Q_FUNC_INFO << "Thread woke up.";
m_mutex.unlock();
} }
} }
if (xml.hasError())
emit finished(xml.errorString());
else if (!foundChannel)
emit finished(tr("Invalid RSS feed."));
else
emit finished(QString());
} }
void Parser::parseRssArticle(QXmlStreamReader &xml, const QString &feedUrl) void Parser::parseRssArticle(QXmlStreamReader &xml)
{ {
QVariantHash article; QVariantHash article;
@ -332,12 +311,12 @@ void Parser::parseRssArticle(QXmlStreamReader &xml, const QString &feedUrl)
} }
} }
emit newArticle(feedUrl, article); emit newArticle(article);
} }
void Parser::parseRSSChannel(QXmlStreamReader &xml, const QString &feedUrl) void Parser::parseRSSChannel(QXmlStreamReader &xml)
{ {
qDebug() << Q_FUNC_INFO << feedUrl; qDebug() << Q_FUNC_INFO;
Q_ASSERT(xml.isStartElement() && xml.name() == "channel"); Q_ASSERT(xml.isStartElement() && xml.name() == "channel");
while(!xml.atEnd()) { while(!xml.atEnd()) {
@ -346,27 +325,26 @@ void Parser::parseRSSChannel(QXmlStreamReader &xml, const QString &feedUrl)
if (xml.isStartElement()) { if (xml.isStartElement()) {
if (xml.name() == "title") { if (xml.name() == "title") {
QString title = xml.readElementText(); QString title = xml.readElementText();
emit feedTitle(feedUrl, title); emit feedTitle(title);
} }
else if (xml.name() == "lastBuildDate") { else if (xml.name() == "lastBuildDate") {
QString lastBuildDate = xml.readElementText(); QString lastBuildDate = xml.readElementText();
if (!lastBuildDate.isEmpty()) { if (!lastBuildDate.isEmpty()) {
QMutexLocker locker(&m_mutex); if (m_lastBuildDate == lastBuildDate) {
if (m_lastBuildDates.value(feedUrl, "") == lastBuildDate) {
qDebug() << "The RSS feed has not changed since last time, aborting parsing."; qDebug() << "The RSS feed has not changed since last time, aborting parsing.";
return; return;
} }
m_lastBuildDates[feedUrl] = lastBuildDate; m_lastBuildDate = lastBuildDate;
} }
} }
else if (xml.name() == "item") { else if (xml.name() == "item") {
parseRssArticle(xml, feedUrl); parseRssArticle(xml);
} }
} }
} }
} }
void Parser::parseAtomArticle(QXmlStreamReader &xml, const QString &feedUrl, const QString &baseUrl) void Parser::parseAtomArticle(QXmlStreamReader &xml)
{ {
QVariantHash article; QVariantHash article;
bool doubleContent = false; bool doubleContent = false;
@ -392,7 +370,7 @@ void Parser::parseAtomArticle(QXmlStreamReader &xml, const QString &feedUrl, con
// Atom feeds can have relative links, work around this and // Atom feeds can have relative links, work around this and
// take the stress of figuring article full URI from UI // take the stress of figuring article full URI from UI
// Assemble full URI // Assemble full URI
article["news_link"] = ( baseUrl.isEmpty() ? link : baseUrl + link ); article["news_link"] = ( m_baseUrl.isEmpty() ? link : m_baseUrl + link );
} }
else if ((xml.name() == "summary") || (xml.name() == "content")){ else if ((xml.name() == "summary") || (xml.name() == "content")){
@ -453,15 +431,15 @@ void Parser::parseAtomArticle(QXmlStreamReader &xml, const QString &feedUrl, con
} }
} }
emit newArticle(feedUrl, article); emit newArticle(article);
} }
void Parser::parseAtomChannel(QXmlStreamReader &xml, const QString &feedUrl) void Parser::parseAtomChannel(QXmlStreamReader &xml)
{ {
qDebug() << Q_FUNC_INFO << feedUrl; qDebug() << Q_FUNC_INFO;
Q_ASSERT(xml.isStartElement() && xml.name() == "feed"); Q_ASSERT(xml.isStartElement() && xml.name() == "feed");
QString baseURL = xml.attributes().value("xml:base").toString(); m_baseUrl = xml.attributes().value("xml:base").toString();
while (!xml.atEnd()) { while (!xml.atEnd()) {
xml.readNext(); xml.readNext();
@ -469,74 +447,21 @@ void Parser::parseAtomChannel(QXmlStreamReader &xml, const QString &feedUrl)
if (xml.isStartElement()) { if (xml.isStartElement()) {
if (xml.name() == "title") { if (xml.name() == "title") {
QString title = xml.readElementText(); QString title = xml.readElementText();
emit feedTitle(feedUrl, title); emit feedTitle(title);
} }
else if (xml.name() == "updated") { else if (xml.name() == "updated") {
QString lastBuildDate = xml.readElementText(); QString lastBuildDate = xml.readElementText();
if (!lastBuildDate.isEmpty()) { if (!lastBuildDate.isEmpty()) {
QMutexLocker locker(&m_mutex); if (m_lastBuildDate == lastBuildDate) {
if (m_lastBuildDates.value(feedUrl) == lastBuildDate) {
qDebug() << "The RSS feed has not changed since last time, aborting parsing."; qDebug() << "The RSS feed has not changed since last time, aborting parsing.";
return; return;
} }
m_lastBuildDates[feedUrl] = lastBuildDate; m_lastBuildDate = lastBuildDate;
} }
} }
else if (xml.name() == "entry") { else if (xml.name() == "entry") {
parseAtomArticle(xml, feedUrl, baseURL); parseAtomArticle(xml);
} }
} }
} }
} }
// read and create items from a rss document
void Parser::parseFeed(const ParsingJob &job)
{
qDebug() << Q_FUNC_INFO << job.feedUrl;
QXmlStreamReader xml(job.feedData);
bool foundChannel = false;
while (xml.readNextStartElement()) {
if (xml.name() == "rss") {
// Find channels
while (xml.readNextStartElement()) {
if (xml.name() == "channel") {
parseRSSChannel(xml, job.feedUrl);
foundChannel = true;
break;
}
else {
qDebug() << "Skip rss item: " << xml.name();
xml.skipCurrentElement();
}
}
break;
}
else if (xml.name() == "feed") { // Atom feed
parseAtomChannel(xml, job.feedUrl);
foundChannel = true;
break;
}
else {
qDebug() << "Skip root item: " << xml.name();
xml.skipCurrentElement();
}
}
if (xml.hasError()) {
reportFailure(job, xml.errorString());
return;
}
if (!foundChannel) {
reportFailure(job, tr("Invalid RSS feed at '%1'.").arg(job.feedUrl));
return;
}
emit feedParsingFinished(job.feedUrl, QString());
}
void Parser::reportFailure(const ParsingJob &job, const QString &error)
{
emit feedParsingFinished(job.feedUrl, error);
}

49
src/base/rss/private/rssparser.h
Unescape View File

@ -1,6 +1,7 @@
/* /*
* Bittorrent Client using Qt and libtorrent. * Bittorrent Client using Qt and libtorrent.
* Copyright (C) 2012 Christophe Dumez * Copyright (C) 2015 Vladimir Golovnev <glassez@yandex.ru>
* Copyright (C) 2012 Christophe Dumez <chris@qbittorrent.org>
* *
* This program is free software; you can redistribute it and/or * This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License * modify it under the terms of the GNU General Public License
@ -31,12 +32,9 @@
#ifndef RSSPARSER_H #ifndef RSSPARSER_H
#define RSSPARSER_H #define RSSPARSER_H
#include <QHash> #include <QObject>
#include <QMutex> #include <QString>
#include <QQueue>
#include <QThread>
#include <QVariantHash> #include <QVariantHash>
#include <QWaitCondition>
class QXmlStreamReader; class QXmlStreamReader;
@ -44,41 +42,26 @@ namespace Rss
{ {
namespace Private namespace Private
{ {
struct ParsingJob; class Parser: public QObject
class Parser: public QThread
{ {
Q_OBJECT Q_OBJECT
public: public slots:
explicit Parser(QObject *parent = 0); void parse(const QByteArray &feedData);
~Parser();
void parseFeedData(const QString &feedUrl, const QByteArray &feedData);
void clearFeedData(const QString &feedUrl);
signals: signals:
void newArticle(const QString &feedUrl, const QVariantHash &rssArticle); void newArticle(const QVariantHash &rssArticle);
void feedTitle(const QString &feedUrl, const QString &title); void feedTitle(const QString &title);
void feedParsingFinished(const QString &feedUrl, const QString &error); void finished(const QString &error);
private: private:
void run() override; void parseRssArticle(QXmlStreamReader &xml);
void parseRSSChannel(QXmlStreamReader &xml);
static QDateTime parseDate(const QString &string); void parseAtomArticle(QXmlStreamReader &xml);
void parseAtomChannel(QXmlStreamReader &xml);
void parseRssArticle(QXmlStreamReader &xml, const QString &feedUrl);
void parseRSSChannel(QXmlStreamReader &xml, const QString &feedUrl);
void parseAtomArticle(QXmlStreamReader &xml, const QString &feedUrl, const QString &baseUrl);
void parseAtomChannel(QXmlStreamReader &xml, const QString &feedUrl);
void parseFeed(const ParsingJob &job);
void reportFailure(const ParsingJob &job, const QString &error);
bool m_running; QString m_lastBuildDate; // Optimization
QMutex m_mutex; QString m_baseUrl;
QQueue<ParsingJob> m_queue;
QWaitCondition m_waitCondition;
QHash<QString/*feedUrl*/, QString/*lastBuildDate*/> m_lastBuildDates; // Optimization
}; };
} }
} }

38
src/base/rss/rssfeed.cpp
Unescape View File

@ -1,5 +1,6 @@
/* /*
* Bittorrent Client using Qt and libtorrent. * Bittorrent Client using Qt and libtorrent.
* Copyright (C) 2015 Vladimir Golovnev <glassez@yandex.ru>
* Copyright (C) 2010 Christophe Dumez <chris@qbittorrent.org> * Copyright (C) 2010 Christophe Dumez <chris@qbittorrent.org>
* Copyright (C) 2010 Arnaud Demaiziere <arnaud@qbittorrent.org> * Copyright (C) 2010 Arnaud Demaiziere <arnaud@qbittorrent.org>
* *
@ -67,11 +68,13 @@ Feed::Feed(const QString &url, Manager *manager)
, m_loading(false) , m_loading(false)
{ {
qDebug() << Q_FUNC_INFO << m_url; qDebug() << Q_FUNC_INFO << m_url;
m_parser = new Private::Parser;
m_parser->moveToThread(m_manager->workingThread());
connect(this, SIGNAL(destroyed()), m_parser, SLOT(deleteLater()));
// Listen for new RSS downloads // Listen for new RSS downloads
Private::Parser *const parser = m_manager->rssParser(); connect(m_parser, SIGNAL(feedTitle(QString)), SLOT(handleFeedTitle(QString)));
connect(parser, SIGNAL(feedTitle(QString,QString)), SLOT(handleFeedTitle(QString,QString))); connect(m_parser, SIGNAL(newArticle(QVariantHash)), SLOT(handleNewArticle(QVariantHash)));
connect(parser, SIGNAL(newArticle(QString,QVariantHash)), SLOT(handleNewArticle(QString,QVariantHash))); connect(m_parser, SIGNAL(finished(QString)), SLOT(handleParsingFinished(QString)));
connect(parser, SIGNAL(feedParsingFinished(QString,QString)), SLOT(handleParsingFinished(QString,QString)));
// Download the RSS Feed icon // Download the RSS Feed icon
Net::DownloadHandler *handler = Net::DownloadManager::instance()->downloadUrl(iconUrl(), true); Net::DownloadHandler *handler = Net::DownloadManager::instance()->downloadUrl(iconUrl(), true);
@ -87,7 +90,6 @@ Feed::~Feed()
{ {
if (!m_icon.startsWith(":/") && QFile::exists(m_icon)) if (!m_icon.startsWith(":/") && QFile::exists(m_icon))
Utils::Fs::forceRemove(m_icon); Utils::Fs::forceRemove(m_icon);
m_manager->rssParser()->clearFeedData(m_url);
} }
void Feed::saveItemsToDisk() void Feed::saveItemsToDisk()
@ -320,7 +322,6 @@ QString Feed::iconUrl() const
void Feed::handleIconDownloadFinished(const QString &url, const QString &filePath) void Feed::handleIconDownloadFinished(const QString &url, const QString &filePath)
{ {
Q_UNUSED(url); Q_UNUSED(url);
m_icon = filePath; m_icon = filePath;
qDebug() << Q_FUNC_INFO << "icon path:" << m_icon; qDebug() << Q_FUNC_INFO << "icon path:" << m_icon;
m_manager->forwardFeedIconChanged(m_url, m_icon); m_manager->forwardFeedIconChanged(m_url, m_icon);
@ -328,30 +329,31 @@ void Feed::handleIconDownloadFinished(const QString &url, const QString &filePat
void Feed::handleRssDownloadFinished(const QString &url, const QByteArray &data) void Feed::handleRssDownloadFinished(const QString &url, const QByteArray &data)
{ {
qDebug() << Q_FUNC_INFO << "Successfully downloaded RSS feed at" << url; Q_UNUSED(url);
qDebug() << Q_FUNC_INFO << "Successfully downloaded RSS feed at" << m_url;
// Parse the download RSS // Parse the download RSS
m_manager->rssParser()->parseFeedData(m_url, data); QMetaObject::invokeMethod(m_parser, "parse", Qt::QueuedConnection, Q_ARG(QByteArray, data));
} }
void Feed::handleRssDownloadFailed(const QString &url, const QString &error) void Feed::handleRssDownloadFailed(const QString &url, const QString &error)
{ {
Q_UNUSED(url);
m_inErrorState = true; m_inErrorState = true;
m_loading = false; m_loading = false;
m_manager->forwardFeedInfosChanged(m_url, displayName(), m_unreadCount); m_manager->forwardFeedInfosChanged(m_url, displayName(), m_unreadCount);
qWarning() << "Failed to download RSS feed at" << url; qWarning() << "Failed to download RSS feed at" << m_url;
qWarning() << "Reason:" << error; qWarning() << "Reason:" << error;
} }
void Feed::handleFeedTitle(const QString &feedUrl, const QString &title) void Feed::handleFeedTitle(const QString &title)
{ {
if (feedUrl != m_url) return;
if (m_title == title) return; if (m_title == title) return;
m_title = title; m_title = title;
// Notify that we now have something better than a URL to display // Notify that we now have something better than a URL to display
if (m_alias.isEmpty()) if (m_alias.isEmpty())
m_manager->forwardFeedInfosChanged(feedUrl, title, m_unreadCount); m_manager->forwardFeedInfosChanged(m_url, title, m_unreadCount);
} }
void Feed::downloadArticleTorrentIfMatching(const ArticlePtr &article) void Feed::downloadArticleTorrentIfMatching(const ArticlePtr &article)
@ -406,13 +408,11 @@ void Feed::recheckRssItemsForDownload()
} }
} }
void Feed::handleNewArticle(const QString &feedUrl, const QVariantHash &articleData) void Feed::handleNewArticle(const QVariantHash &articleData)
{ {
if (feedUrl != m_url) return;
ArticlePtr article = Article::fromHash(this, articleData); ArticlePtr article = Article::fromHash(this, articleData);
if (article.isNull()) { if (article.isNull()) {
qDebug() << "Article hash corrupted or guid is uncomputable; feed url: " << feedUrl; qDebug() << "Article hash corrupted or guid is uncomputable; feed url: " << m_url;
return; return;
} }
Q_ASSERT(article); Q_ASSERT(article);
@ -424,12 +424,10 @@ void Feed::handleNewArticle(const QString &feedUrl, const QVariantHash &articleD
//m_manager->forwardFeedContentChanged(m_url); //m_manager->forwardFeedContentChanged(m_url);
} }
void Feed::handleParsingFinished(const QString &feedUrl, const QString &error) void Feed::handleParsingFinished(const QString &error)
{ {
if (feedUrl != m_url) return;
if (!error.isEmpty()) { if (!error.isEmpty()) {
qWarning() << "Failed to parse RSS feed at" << feedUrl; qWarning() << "Failed to parse RSS feed at" << m_url;
qWarning() << "Reason:" << error; qWarning() << "Reason:" << error;
} }

13
src/base/rss/rssfeed.h
Unescape View File

@ -1,5 +1,6 @@
/* /*
* Bittorrent Client using Qt and libtorrent. * Bittorrent Client using Qt and libtorrent.
* Copyright (C) 2015 Vladimir Golovnev <glassez@yandex.ru>
* Copyright (C) 2010 Christophe Dumez <chris@qbittorrent.org> * Copyright (C) 2010 Christophe Dumez <chris@qbittorrent.org>
* Copyright (C) 2010 Arnaud Demaiziere <arnaud@qbittorrent.org> * Copyright (C) 2010 Arnaud Demaiziere <arnaud@qbittorrent.org>
* *
@ -51,6 +52,11 @@ namespace Rss
typedef QSharedPointer<Feed> FeedPtr; typedef QSharedPointer<Feed> FeedPtr;
typedef QList<FeedPtr> FeedList; typedef QList<FeedPtr> FeedList;
namespace Private
{
class Parser;
}
bool articleDateRecentThan(const ArticlePtr &left, const ArticlePtr &right); bool articleDateRecentThan(const ArticlePtr &left, const ArticlePtr &right);
class Feed: public QObject, public File class Feed: public QObject, public File
@ -86,9 +92,9 @@ namespace Rss
void handleIconDownloadFinished(const QString &url, const QString &filePath); void handleIconDownloadFinished(const QString &url, const QString &filePath);
void handleRssDownloadFinished(const QString &url, const QByteArray &data); void handleRssDownloadFinished(const QString &url, const QByteArray &data);
void handleRssDownloadFailed(const QString &url, const QString &error); void handleRssDownloadFailed(const QString &url, const QString &error);
void handleFeedTitle(const QString &feedUrl, const QString &title); void handleFeedTitle(const QString &title);
void handleNewArticle(const QString &feedUrl, const QVariantHash &article); void handleNewArticle(const QVariantHash &article);
void handleParsingFinished(const QString &feedUrl, const QString &error); void handleParsingFinished(const QString &error);
void handleArticleRead(); void handleArticleRead();
private: private:
@ -99,6 +105,7 @@ namespace Rss
private: private:
Manager *m_manager; Manager *m_manager;
Private::Parser *m_parser;
ArticleHash m_articles; ArticleHash m_articles;
ArticleList m_articlesByDate; // Articles sorted by date (more recent first) ArticleList m_articlesByDate; // Articles sorted by date (more recent first)
QString m_title; QString m_title;

11
src/base/rss/rssmanager.cpp
Unescape View File

@ -33,7 +33,6 @@
#include "base/logger.h" #include "base/logger.h"
#include "base/preferences.h" #include "base/preferences.h"
#include "private/rssparser.h"
#include "rssfolder.h" #include "rssfolder.h"
#include "rssfeed.h" #include "rssfeed.h"
#include "rssarticle.h" #include "rssarticle.h"
@ -48,9 +47,10 @@ using namespace Rss::Private;
Manager::Manager(QObject *parent) Manager::Manager(QObject *parent)
: QObject(parent) : QObject(parent)
, m_downloadRules(new DownloadRuleList) , m_downloadRules(new DownloadRuleList)
, m_rssParser(new Parser(this))
, m_rootFolder(new Folder) , m_rootFolder(new Folder)
, m_workingThread(new QThread(this))
{ {
m_workingThread->start();
connect(&m_refreshTimer, SIGNAL(timeout()), SLOT(refresh())); connect(&m_refreshTimer, SIGNAL(timeout()), SLOT(refresh()));
m_refreshInterval = Preferences::instance()->getRSSRefreshInterval(); m_refreshInterval = Preferences::instance()->getRSSRefreshInterval();
m_refreshTimer.start(m_refreshInterval * MSECS_PER_MIN); m_refreshTimer.start(m_refreshInterval * MSECS_PER_MIN);
@ -59,8 +59,9 @@ Manager::Manager(QObject *parent)
Manager::~Manager() Manager::~Manager()
{ {
qDebug("Deleting RSSManager..."); qDebug("Deleting RSSManager...");
m_workingThread->quit();
m_workingThread->wait();
delete m_downloadRules; delete m_downloadRules;
delete m_rssParser;
m_rootFolder->saveItemsToDisk(); m_rootFolder->saveItemsToDisk();
saveStreamList(); saveStreamList();
m_rootFolder.clear(); m_rootFolder.clear();
@ -178,9 +179,9 @@ FolderPtr Manager::rootFolder() const
return m_rootFolder; return m_rootFolder;
} }
Parser *Manager::rssParser() const QThread *Manager::workingThread() const
{ {
return m_rssParser; return m_workingThread;
} }
void Manager::refresh() void Manager::refresh()

11
src/base/rss/rssmanager.h
Unescape View File

@ -35,6 +35,7 @@
#include <QObject> #include <QObject>
#include <QTimer> #include <QTimer>
#include <QSharedPointer> #include <QSharedPointer>
#include <QThread>
namespace Rss namespace Rss
{ {
@ -48,11 +49,6 @@ namespace Rss
typedef QSharedPointer<Folder> FolderPtr; typedef QSharedPointer<Folder> FolderPtr;
typedef QSharedPointer<Feed> FeedPtr; typedef QSharedPointer<Feed> FeedPtr;
namespace Private
{
class Parser;
}
typedef QSharedPointer<Manager> ManagerPtr; typedef QSharedPointer<Manager> ManagerPtr;
class Manager: public QObject class Manager: public QObject
@ -65,8 +61,7 @@ namespace Rss
DownloadRuleList *downloadRules() const; DownloadRuleList *downloadRules() const;
FolderPtr rootFolder() const; FolderPtr rootFolder() const;
QThread *workingThread() const;
Private::Parser *rssParser() const;
public slots: public slots:
void refresh(); void refresh();
@ -87,8 +82,8 @@ namespace Rss
QTimer m_refreshTimer; QTimer m_refreshTimer;
uint m_refreshInterval; uint m_refreshInterval;
DownloadRuleList *m_downloadRules; DownloadRuleList *m_downloadRules;
Private::Parser *m_rssParser;
FolderPtr m_rootFolder; FolderPtr m_rootFolder;
QThread *m_workingThread;
}; };
} }

Write Preview
Loading…
Cancel
Save