@ -37,475 +37,505 @@
# include "base/utils/fs.h" # include "base/utils/fs.h"
# include "rssparser.h" # include "rssparser.h"
struct ParsingJob { struct ParsingJob
QString feedUrl ; {
QString filePath ; QString feedUrl ;
QString filePath ;
} ; } ;
static const char shortDay [ ] [ 4 ] = { static const char shortDay [ ] [ 4 ] = {
" Mon " , " Tue " , " Wed " , " Mon " , " Tue " , " Wed " ,
" Thu " , " Fri " , " Sat " , " Thu " , " Fri " , " Sat " ,
" Sun " " Sun "
} ; } ;
static const char longDay [ ] [ 10 ] = { static const char longDay [ ] [ 10 ] = {
" Monday " , " Tuesday " , " Wednesday " , " Monday " , " Tuesday " , " Wednesday " ,
" Thursday " , " Friday " , " Saturday " , " Thursday " , " Friday " , " Saturday " ,
" Sunday " " Sunday "
} ; } ;
static const char shortMonth [ ] [ 4 ] = { static const char shortMonth [ ] [ 4 ] = {
" Jan " , " Feb " , " Mar " , " Apr " , " Jan " , " Feb " , " Mar " , " Apr " ,
" May " , " Jun " , " Jul " , " Aug " , " May " , " Jun " , " Jul " , " Aug " ,
" Sep " , " Oct " , " Nov " , " Dec " " Sep " , " Oct " , " Nov " , " Dec "
} ; } ;
// Ported to Qt4 from KDElibs4
// Ported to Qt from KDElibs4
QDateTime RssParser : : parseDate ( const QString & string ) { QDateTime RssParser : : parseDate ( const QString & string )
const QString str = string . trimmed ( ) ; {
if ( str . isEmpty ( ) ) const QString str = string . trimmed ( ) ;
return QDateTime : : currentDateTime ( ) ; if ( str . isEmpty ( ) )
return QDateTime : : currentDateTime ( ) ;
int nyear = 6 ; // indexes within string to values
int nmonth = 4 ; int nyear = 6 ; // indexes within string to values
int nday = 2 ; int nmonth = 4 ;
int nwday = 1 ; int nday = 2 ;
int nhour = 7 ; int nwday = 1 ;
int nmin = 8 ; int nhour = 7 ;
int nsec = 9 ; int nmin = 8 ;
// Also accept obsolete form "Weekday, DD-Mon-YY HH:MM:SS ±hhmm"
int nsec = 9 ;
QRegExp rx ( " ^(?:([A-Z][a-z]+) , \ \ s * ) ? ( \ \ d { 1 , 2 } ) ( \ \ s + | - ) ( [ ^ - \ \ s ] + ) ( \ \ s + | - ) ( \ \ d { 2 , 4 } ) \ \ s + ( \ \ d \ \ d ) : ( \ \ d \ \ d ) ( ? : : ( \ \ d \ \ d ) ) ? \ \ s + ( \ \ S + ) $ " ); // Also accept obsolete form "Weekday, DD-Mon-YY HH:MM:SS ±hhmm"
QStringList parts ; QRegExp rx ( " ^(?:([A-Z][a-z]+) , \ \ s * ) ? ( \ \ d { 1 , 2 } ) ( \ \ s + | - ) ( [ ^ - \ \ s ] + ) ( \ \ s + | - ) ( \ \ d { 2 , 4 } ) \ \ s + ( \ \ d \ \ d ) : ( \ \ d \ \ d ) ( ? : : ( \ \ d \ \ d ) ) ? \ \ s + ( \ \ S + ) $ " );
if ( ! str . indexOf ( rx ) ) { QStringList parts ;
// Check that if date has '-' separators, both separators are '-'.
if ( ! str . indexOf ( rx ) ) {
parts = rx . capturedTexts ( ) ; // Check that if date has '-' separators, both separators are '-'.
bool h1 = ( parts [ 3 ] = = QLatin1String ( " - " ) ) ; parts = rx . capturedTexts ( ) ;
bool h2 = ( parts [ 5 ] = = QLatin1String ( " - " ) ) ; bool h1 = ( parts [ 3 ] = = QLatin1String ( " - " ) ) ;
if ( h1 ! = h2 ) bool h2 = ( parts [ 5 ] = = QLatin1String ( " - " ) ) ;
return QDateTime : : currentDateTime ( ) ; if ( h1 ! = h2 )
} else { return QDateTime : : currentDateTime ( ) ;
// Check for the obsolete form "Wdy Mon DD HH:MM:SS YYYY"
}
rx = QRegExp ( " ^([A-Z][a-z]+) \\ s+( \\ S+) \\ s+( \\ d \\ d) \\ s+( \\ d \\ d):( \\ d \\ d):( \\ d \\ d) \\ s+( \\ d \\ d \\ d \\ d)$ " ) ; else {
if ( str . indexOf ( rx ) ) // Check for the obsolete form "Wdy Mon DD HH:MM:SS YYYY"
return QDateTime : : currentDateTime ( ) ; rx = QRegExp ( " ^([A-Z][a-z]+) \\ s+( \\ S+) \\ s+( \\ d \\ d) \\ s+( \\ d \\ d):( \\ d \\ d):( \\ d \\ d) \\ s+( \\ d \\ d \\ d \\ d)$ " ) ;
nyear = 7 ; if ( str . indexOf ( rx ) )
nmonth = 2 ; return QDateTime : : currentDateTime ( ) ;
nday = 3 ; nyear = 7 ;
nwday = 1 ; nmonth = 2 ;
nhour = 4 ; nday = 3 ;
nmin = 5 ; nwday = 1 ;
nsec = 6 ; nhour = 4 ;
parts = rx . capturedTexts ( ) ; nmin = 5 ;
} nsec = 6 ;
bool ok [ 4 ] ; parts = rx . capturedTexts ( ) ;
const int day = parts [ nday ] . toInt ( & ok [ 0 ] ) ; }
int year = parts [ nyear ] . toInt ( & ok [ 1 ] ) ;
const int hour = parts [ nhour ] . toInt ( & ok [ 2 ] ) ; bool ok [ 4 ] ;
const int minute = parts [ nmin ] . toInt ( & ok [ 3 ] ) ; const int day = parts [ nday ] . toInt ( & ok [ 0 ] ) ;
if ( ! ok [ 0 ] | | ! ok [ 1 ] | | ! ok [ 2 ] | | ! ok [ 3 ] ) int year = parts [ nyear ] . toInt ( & ok [ 1 ] ) ;
return QDateTime : : currentDateTime ( ) ; const int hour = parts [ nhour ] . toInt ( & ok [ 2 ] ) ;
int second = 0 ; const int minute = parts [ nmin ] . toInt ( & ok [ 3 ] ) ;
if ( ! parts [ nsec ] . isEmpty ( ) ) { if ( ! ok [ 0 ] | | ! ok [ 1 ] | | ! ok [ 2 ] | | ! ok [ 3 ] )
second = parts [ nsec ] . toInt ( & ok [ 0 ] ) ; return QDateTime : : currentDateTime ( ) ;
if ( ! ok [ 0 ] )
return QDateTime : : currentDateTime ( ) ; int second = 0 ;
} if ( ! parts [ nsec ] . isEmpty ( ) ) {
bool leapSecond = ( second = = 60 ) ; second = parts [ nsec ] . toInt ( & ok [ 0 ] ) ;
if ( leapSecond ) if ( ! ok [ 0 ] )
second = 59 ; // apparently a leap second - validate below, once time zone is known
return QDateTime : : currentDateTime ( ) ;
int month = 0 ; }
for ( ; month < 12 & & parts [ nmonth ] ! = shortMonth [ month ] ; + + month ) ;
int dayOfWeek = - 1 ; bool leapSecond = ( second = = 60 ) ;
if ( ! parts [ nwday ] . isEmpty ( ) ) { if ( leapSecond )
// Look up the weekday name
second = 59 ; // apparently a leap second - validate below, once time zone is known
while ( + + dayOfWeek < 7 & & shortDay [ dayOfWeek ] ! = parts [ nwday ] ) ; int month = 0 ;
if ( dayOfWeek > = 7 ) for ( ; ( month < 12 ) & & ( parts [ nmonth ] ! = shortMonth [ month ] ) ; + + month ) ;
for ( dayOfWeek = 0 ; dayOfWeek < 7 & & longDay [ dayOfWeek ] ! = parts [ nwday ] ; + + dayOfWeek ) ; int dayOfWeek = - 1 ;
} if ( ! parts [ nwday ] . isEmpty ( ) ) {
// if (month >= 12 || dayOfWeek >= 7
// Look up the weekday name
// || (dayOfWeek < 0 && format == RFCDateDay))
while ( + + dayOfWeek < 7 & & ( shortDay [ dayOfWeek ] ! = parts [ nwday ] ) ) ;
// return QDateTime;
if ( dayOfWeek > = 7 )
int i = parts [ nyear ] . size ( ) ; for ( dayOfWeek = 0 ; dayOfWeek < 7 & & ( longDay [ dayOfWeek ] ! = parts [ nwday ] ) ; + + dayOfWeek ) ;
if ( i < 4 ) { }
// It's an obsolete year specification with less than 4 digits
year + = ( i = = 2 & & year < 50 ) ? 2000 : 1900 ; // if (month >= 12 || dayOfWeek >= 7
} // || (dayOfWeek < 0 && format == RFCDateDay))
// return QDateTime;
// Parse the UTC offset part
int i = parts [ nyear ] . size ( ) ;
int offset = 0 ; // set default to '-0000'
if ( i < 4 ) {
bool negOffset = false ; // It's an obsolete year specification with less than 4 digits
if ( parts . count ( ) > 10 ) { year + = ( i = = 2 & & year < 50 ) ? 2000 : 1900 ;
rx = QRegExp ( " ^([+-])( \\ d \\ d)( \\ d \\ d)$ " ) ; }
if ( ! parts [ 10 ] . indexOf ( rx ) ) {
// It's a UTC offset ±hhmm
// Parse the UTC offset part
parts = rx . capturedTexts ( ) ; int offset = 0 ; // set default to '-0000'
offset = parts [ 2 ] . toInt ( & ok [ 0 ] ) * 3600 ; bool negOffset = false ;
int offsetMin = parts [ 3 ] . toInt ( & ok [ 1 ] ) ; if ( parts . count ( ) > 10 ) {
if ( ! ok [ 0 ] | | ! ok [ 1 ] | | offsetMin > 59 ) rx = QRegExp ( " ^([+-])( \\ d \\ d)( \\ d \\ d)$ " ) ;
return QDateTime ( ) ; if ( ! parts [ 10 ] . indexOf ( rx ) ) {
offset + = offsetMin * 60 ; // It's a UTC offset ±hhmm
negOffset = ( parts [ 1 ] = = QLatin1String ( " - " ) ) ; parts = rx . capturedTexts ( ) ;
if ( negOffset ) offset = parts [ 2 ] . toInt ( & ok [ 0 ] ) * 3600 ;
offset = - offset ; int offsetMin = parts [ 3 ] . toInt ( & ok [ 1 ] ) ;
} else { if ( ! ok [ 0 ] | | ! ok [ 1 ] | | offsetMin > 59 )
// Check for an obsolete time zone name
return QDateTime ( ) ;
QByteArray zone = parts [ 10 ] . toLatin1 ( ) ; offset + = offsetMin * 60 ;
if ( zone . length ( ) = = 1 & & isalpha ( zone [ 0 ] ) & & toupper ( zone [ 0 ] ) ! = ' J ' ) negOffset = ( parts [ 1 ] = = QLatin1String ( " - " ) ) ;
negOffset = true ; // military zone: RFC 2822 treats as '-0000'
if ( negOffset )
else if ( zone ! = " UT " & & zone ! = " GMT " ) { // treated as '+0000'
offset = - offset ;
offset = ( zone = = " EDT " ) ? - 4 * 3600 }
: ( zone = = " EST " | | zone = = " CDT " ) ? - 5 * 3600 else {
: ( zone = = " CST " | | zone = = " MDT " ) ? - 6 * 3600 // Check for an obsolete time zone name
: ( zone = = " MST " | | zone = = " PDT " ) ? - 7 * 3600 QByteArray zone = parts [ 10 ] . toLatin1 ( ) ;
: ( zone = = " PST " ) ? - 8 * 3600 if ( zone . length ( ) = = 1 & & isalpha ( zone [ 0 ] ) & & toupper ( zone [ 0 ] ) ! = ' J ' ) {
: 0 ; negOffset = true ; // military zone: RFC 2822 treats as '-0000'
if ( ! offset ) { }
// Check for any other alphabetic time zone
else if ( zone ! = " UT " & & zone ! = " GMT " ) { // treated as '+0000'
bool nonalpha = false ; offset = ( zone = = " EDT " )
for ( int i = 0 , end = zone . size ( ) ; i < end & & ! nonalpha ; + + i ) ? - 4 * 3600
nonalpha = ! isalpha ( zone [ i ] ) ; : ( ( zone = = " EST " ) | | ( zone = = " CDT " ) )
if ( nonalpha ) ? - 5 * 3600
return QDateTime ( ) ; : ( ( zone = = " CST " ) | | ( zone = = " MDT " ) )
// TODO: Attempt to recognize the time zone abbreviation?
? - 6 * 3600
negOffset = true ; // unknown time zone: RFC 2822 treats as '-0000'
: ( zone = = " MST " | | zone = = " PDT " )
? - 7 * 3600
: ( zone = = " PST " )
? - 8 * 3600
: 0 ;
if ( ! offset ) {
// Check for any other alphabetic time zone
bool nonalpha = false ;
for ( int i = 0 , end = zone . size ( ) ; ( i < end ) & & ! nonalpha ; + + i )
nonalpha = ! isalpha ( zone [ i ] ) ;
if ( nonalpha )
return QDateTime ( ) ;
// TODO: Attempt to recognize the time zone abbreviation?
negOffset = true ; // unknown time zone: RFC 2822 treats as '-0000'
}
}
} }
}
} }
}
QDate qdate ( year , month + 1 , day ) ; // convert date, and check for out-of-range
QDate qdate ( year , month + 1 , day ) ; // convert date, and check for out-of-range
if ( ! qdate . isValid ( ) ) if ( ! qdate . isValid ( ) )
return QDateTime : : currentDateTime ( ) ; return QDateTime : : currentDateTime ( ) ;
QTime qTime ( hour , minute , second ) ;
QTime qTime ( hour , minute , second ) ;
QDateTime result ( qdate , qTime , Qt : : UTC ) ; QDateTime result ( qdate , qTime , Qt : : UTC ) ;
if ( offset ) if ( offset )
result = result . addSecs ( - offset ) ; result = result . addSecs ( - offset ) ;
if ( ! result . isValid ( ) ) if ( ! result . isValid ( ) )
return QDateTime : : currentDateTime ( ) ; // invalid date/time
return QDateTime : : currentDateTime ( ) ; // invalid date/time
if ( leapSecond ) { if ( leapSecond ) {
// Validate a leap second time. Leap seconds are inserted after 23:59:59 UTC.
// Validate a leap second time. Leap seconds are inserted after 23:59:59 UTC.
// Convert the time to UTC and check that it is 00:00:00.
// Convert the time to UTC and check that it is 00:00:00.
if ( ( hour * 3600 + minute * 60 + 60 - offset + 86400 * 5 ) % 86400 ) // (max abs(offset) is 100 hours)
if ( ( hour * 3600 + minute * 60 + 60 - offset + 86400 * 5 ) % 86400 ) // (max abs(offset) is 100 hours)
return QDateTime : : currentDateTime ( ) ; // the time isn't the last second of the day
return QDateTime : : currentDateTime ( ) ; // the time isn't the last second of the day
} }
return result ;
return result ;
} }
RssParser : : RssParser ( QObject * parent ) : RssParser : : RssParser ( QObject * parent )
QThread ( parent ) , m_running ( true ) : QThread ( parent )
, m_running ( true )
{ {
start ( ) ; start ( ) ;
} }
RssParser : : ~ RssParser ( ) RssParser : : ~ RssParser ( )
{ {
m_running = false ; m_running = false ;
m_waitCondition . wakeOne ( ) ; m_waitCondition . wakeOne ( ) ;
wait ( ) ; wait ( ) ;
} }
void RssParser : : parseRssFile ( const QString & feedUrl , const QString & filePath ) void RssParser : : parseRssFile ( const QString & feedUrl , const QString & filePath )
{ {
qDebug ( ) < < Q_FUNC_INFO < < feedUrl < < filePath ; qDebug ( ) < < Q_FUNC_INFO < < feedUrl < < filePath ;
m_mutex . lock ( ) ; m_mutex . lock ( ) ;
ParsingJob job = { feedUrl , Utils : : Fs : : fromNativePath ( filePath ) } ; ParsingJob job = { feedUrl , Utils : : Fs : : fromNativePath ( filePath ) } ;
m_queue . enqueue ( job ) ; m_queue . enqueue ( job ) ;
// Wake up thread.
// Wake up thread.
if ( m_queue . count ( ) = = 1 ) { if ( m_queue . count ( ) = = 1 ) {
qDebug ( ) < < Q_FUNC_INFO < < " Waking up thread " ; qDebug ( ) < < Q_FUNC_INFO < < " Waking up thread " ;
m_waitCondition . wakeOne ( ) ; m_waitCondition . wakeOne ( ) ;
} }
m_mutex . unlock ( ) ; m_mutex . unlock ( ) ;
} }
void RssParser : : clearFeedData ( const QString & feedUrl ) void RssParser : : clearFeedData ( const QString & feedUrl )
{ {
m_mutex . lock ( ) ; m_mutex . lock ( ) ;
m_lastBuildDates . remove ( feedUrl ) ; m_lastBuildDates . remove ( feedUrl ) ;
m_mutex . unlock ( ) ; m_mutex . unlock ( ) ;
} }
void RssParser : : run ( ) void RssParser : : run ( )
{ {
while ( m_running ) { while ( m_running ) {
m_mutex . lock ( ) ; m_mutex . lock ( ) ;
if ( ! m_queue . empty ( ) ) { if ( ! m_queue . empty ( ) ) {
ParsingJob job = m_queue . dequeue ( ) ; ParsingJob job = m_queue . dequeue ( ) ;
m_mutex . unlock ( ) ; m_mutex . unlock ( ) ;
parseFeed ( job ) ; parseFeed ( job ) ;
} else { }
qDebug ( ) < < Q_FUNC_INFO < < " Thread is waiting. " ; else {
m_waitCondition . wait ( & m_mutex ) ; qDebug ( ) < < Q_FUNC_INFO < < " Thread is waiting. " ;
qDebug ( ) < < Q_FUNC_INFO < < " Thread woke up. " ; m_waitCondition . wait ( & m_mutex ) ;
m_mutex . unlock ( ) ; qDebug ( ) < < Q_FUNC_INFO < < " Thread woke up. " ;
m_mutex . unlock ( ) ;
}
} }
}
} }
void RssParser : : parseRssArticle ( QXmlStreamReader & xml , const QString & feedUrl ) void RssParser : : parseRssArticle ( QXmlStreamReader & xml , const QString & feedUrl )
{ {
QVariantHash article ; QVariantHash article ;
while ( ! xml . atEnd ( ) ) { while ( ! xml . atEnd ( ) ) {
xml . readNext ( ) ; xml . readNext ( ) ;
if ( xml . isEndElement ( ) & & xml . name ( ) = = " item " ) if ( xml . isEndElement ( ) & & xml . name ( ) = = " item " )
break ; break ;
if ( xml . isStartElement ( ) ) { if ( xml . isStartElement ( ) ) {
if ( xml . name ( ) = = " title " ) if ( xml . name ( ) = = " title " ) {
article [ " title " ] = xml . readElementText ( ) . trimmed ( ) ; article [ " title " ] = xml . readElementText ( ) . trimmed ( ) ;
else if ( xml . name ( ) = = " enclosure " ) { }
if ( xml . attributes ( ) . value ( " type " ) = = " application/x-bittorrent " ) else if ( xml . name ( ) = = " enclosure " ) {
article [ " torrent_url " ] = xml . attributes ( ) . value ( " url " ) . toString ( ) ; if ( xml . attributes ( ) . value ( " type " ) = = " application/x-bittorrent " )
} article [ " torrent_url " ] = xml . attributes ( ) . value ( " url " ) . toString ( ) ;
else if ( xml . name ( ) = = " link " ) { }
QString link = xml . readElementText ( ) . trimmed ( ) ; else if ( xml . name ( ) = = " link " ) {
if ( link . startsWith ( " magnet: " , Qt : : CaseInsensitive ) ) QString link = xml . readElementText ( ) . trimmed ( ) ;
article [ " torrent_url " ] = link ; // magnet link instead of a news URL
if ( link . startsWith ( " magnet: " , Qt : : CaseInsensitive ) )
else article [ " torrent_url " ] = link ; // magnet link instead of a news URL
article [ " news_link " ] = link ; else
} article [ " news_link " ] = link ;
else if ( xml . name ( ) = = " description " ) }
article [ " description " ] = xml . readElementText ( ) . trimmed ( ) ; else if ( xml . name ( ) = = " description " ) {
else if ( xml . name ( ) = = " pubDate " ) article [ " description " ] = xml . readElementText ( ) . trimmed ( ) ;
article [ " date " ] = parseDate ( xml . readElementText ( ) . trimmed ( ) ) ; }
else if ( xml . name ( ) = = " author " ) else if ( xml . name ( ) = = " pubDate " ) {
article [ " author " ] = xml . readElementText ( ) . trimmed ( ) ; article [ " date " ] = parseDate ( xml . readElementText ( ) . trimmed ( ) ) ;
else if ( xml . name ( ) = = " guid " ) }
article [ " id " ] = xml . readElementText ( ) . trimmed ( ) ; else if ( xml . name ( ) = = " author " ) {
article [ " author " ] = xml . readElementText ( ) . trimmed ( ) ;
}
else if ( xml . name ( ) = = " guid " ) {
article [ " id " ] = xml . readElementText ( ) . trimmed ( ) ;
}
}
} }
}
if ( ! article . contains ( " torrent_url " ) & & article . contains ( " news_link " ) ) if ( ! article . contains ( " torrent_url " ) & & article . contains ( " news_link " ) )
article [ " torrent_url " ] = article [ " news_link " ] ; article [ " torrent_url " ] = article [ " news_link " ] ;
if ( ! article . contains ( " id " ) ) { if ( ! article . contains ( " id " ) ) {
// Item does not have a guid, fall back to some other identifier
// Item does not have a guid, fall back to some other identifier
const QString link = article . value ( " news_link " ) . toString ( ) ; const QString link = article . value ( " news_link " ) . toString ( ) ;
if ( ! link . isEmpty ( ) ) if ( ! link . isEmpty ( ) ) {
article [ " id " ] = link ; article [ " id " ] = link ;
else { }
const QString title = article . value ( " title " ) . toString ( ) ; else {
if ( ! title . isEmpty ( ) ) const QString title = article . value ( " title " ) . toString ( ) ;
article [ " id " ] = title ; if ( ! title . isEmpty ( ) ) {
else { article [ " id " ] = title ;
qWarning ( ) < < " Item has no guid, link or title, ignoring it... " ; }
return ; else {
} qWarning ( ) < < " Item has no guid, link or title, ignoring it... " ;
return ;
}
}
} }
}
emit newArticle ( feedUrl , article ) ; emit newArticle ( feedUrl , article ) ;
} }
void RssParser : : parseRSSChannel ( QXmlStreamReader & xml , const QString & feedUrl ) void RssParser : : parseRSSChannel ( QXmlStreamReader & xml , const QString & feedUrl )
{ {
qDebug ( ) < < Q_FUNC_INFO < < feedUrl ; qDebug ( ) < < Q_FUNC_INFO < < feedUrl ;
Q_ASSERT ( xml . isStartElement ( ) & & xml . name ( ) = = " channel " ) ; Q_ASSERT ( xml . isStartElement ( ) & & xml . name ( ) = = " channel " ) ;
while ( ! xml . atEnd ( ) ) { while ( ! xml . atEnd ( ) ) {
xml . readNext ( ) ; xml . readNext ( ) ;
if ( xml . isStartElement ( ) ) { if ( xml . isStartElement ( ) ) {
if ( xml . name ( ) = = " title " ) { if ( xml . name ( ) = = " title " ) {
QString title = xml . readElementText ( ) ; QString title = xml . readElementText ( ) ;
emit feedTitle ( feedUrl , title ) ; emit feedTitle ( feedUrl , title ) ;
} }
else if ( xml . name ( ) = = " lastBuildDate " ) { else if ( xml . name ( ) = = " lastBuildDate " ) {
QString lastBuildDate = xml . readElementText ( ) ; QString lastBuildDate = xml . readElementText ( ) ;
if ( ! lastBuildDate . isEmpty ( ) ) { if ( ! lastBuildDate . isEmpty ( ) ) {
QMutexLocker locker ( & m_mutex ) ; QMutexLocker locker ( & m_mutex ) ;
if ( m_lastBuildDates . value ( feedUrl , " " ) = = lastBuildDate ) { if ( m_lastBuildDates . value ( feedUrl , " " ) = = lastBuildDate ) {
qDebug ( ) < < " The RSS feed has not changed since last time, aborting parsing. " ; qDebug ( ) < < " The RSS feed has not changed since last time, aborting parsing. " ;
return ; return ;
} }
m_lastBuildDates [ feedUrl ] = lastBuildDate ; m_lastBuildDates [ feedUrl ] = lastBuildDate ;
}
}
else if ( xml . name ( ) = = " item " ) {
parseRssArticle ( xml , feedUrl ) ;
}
} }
}
else if ( xml . name ( ) = = " item " ) {
parseRssArticle ( xml , feedUrl ) ;
}
} }
}
} }
void RssParser : : parseAtomArticle ( QXmlStreamReader & xml , const QString & feedUrl , const QString & baseUrl ) void RssParser : : parseAtomArticle ( QXmlStreamReader & xml , const QString & feedUrl , const QString & baseUrl )
{ {
QVariantHash article ; QVariantHash article ;
bool double_content = false ; bool doubleContent = false ;
while ( ! xml . atEnd ( ) ) {
xml . readNext ( ) ;
if ( xml . isEndElement ( ) & & xml . name ( ) = = " entry " )
break ;
if ( xml . isStartElement ( ) ) {
if ( xml . name ( ) = = " title " ) {
article [ " title " ] = xml . readElementText ( ) . trimmed ( ) ;
}
else if ( xml . name ( ) = = " link " ) {
QString link = ( xml . attributes ( ) . isEmpty ( ) ?
xml . readElementText ( ) . trimmed ( ) :
xml . attributes ( ) . value ( " href " ) . toString ( ) ) ;
if ( link . startsWith ( " magnet: " , Qt : : CaseInsensitive ) )
article [ " torrent_url " ] = link ; // magnet link instead of a news URL
else
// Atom feeds can have relative links, work around this and
// take the stress of figuring article full URI from UI
// Assemble full URI
article [ " news_link " ] = ( baseUrl . isEmpty ( ) ? link : baseUrl + link ) ;
}
else if ( xml . name ( ) = = " summary " | | xml . name ( ) = = " content " ) {
if ( double_content ) { // Duplicate content -> ignore
xml . readNext ( ) ;
while ( xml . name ( ) ! = " summary " & & xml . name ( ) ! = " content " )
xml . readNext ( ) ;
continue ;
}
// Try to also parse broken articles, which don't use html '&' escapes
while ( ! xml . atEnd ( ) ) {
// Actually works great for non-broken content too
QString feedText = xml . readElementText ( QXmlStreamReader : : IncludeChildElements ) ;
if ( ! feedText . isEmpty ( ) )
article [ " description " ] = feedText . trimmed ( ) ;
double_content = true ;
}
else if ( xml . name ( ) = = " updated " ) {
// ATOM uses standard compliant date, don't do fancy stuff
QDateTime articleDate = QDateTime : : fromString ( xml . readElementText ( ) . trimmed ( ) , Qt : : ISODate ) ;
article [ " date " ] = ( articleDate . isValid ( ) ?
articleDate :
QDateTime : : currentDateTime ( ) ) ;
}
else if ( xml . name ( ) = = " author " ) {
xml . readNext ( ) ; xml . readNext ( ) ;
while ( xml . name ( ) ! = " author " ) {
if ( xml . name ( ) = = " name " ) if ( xml . isEndElement ( ) & & ( xml . name ( ) = = " entry " ) )
article [ " author " ] = xml . readElementText ( ) . trimmed ( ) ; break ;
xml . readNext ( ) ;
if ( xml . isStartElement ( ) ) {
if ( xml . name ( ) = = " title " ) {
article [ " title " ] = xml . readElementText ( ) . trimmed ( ) ;
}
else if ( xml . name ( ) = = " link " ) {
QString link = ( xml . attributes ( ) . isEmpty ( ) ?
xml . readElementText ( ) . trimmed ( ) :
xml . attributes ( ) . value ( " href " ) . toString ( ) ) ;
if ( link . startsWith ( " magnet: " , Qt : : CaseInsensitive ) )
article [ " torrent_url " ] = link ; // magnet link instead of a news URL
else
// Atom feeds can have relative links, work around this and
// take the stress of figuring article full URI from UI
// Assemble full URI
article [ " news_link " ] = ( baseUrl . isEmpty ( ) ? link : baseUrl + link ) ;
}
else if ( ( xml . name ( ) = = " summary " ) | | ( xml . name ( ) = = " content " ) ) {
if ( doubleContent ) { // Duplicate content -> ignore
xml . readNext ( ) ;
while ( ( xml . name ( ) ! = " summary " ) & & ( xml . name ( ) ! = " content " ) )
xml . readNext ( ) ;
continue ;
}
// Try to also parse broken articles, which don't use html '&' escapes
// Actually works great for non-broken content too
QString feedText = xml . readElementText ( QXmlStreamReader : : IncludeChildElements ) ;
if ( ! feedText . isEmpty ( ) )
article [ " description " ] = feedText . trimmed ( ) ;
doubleContent = true ;
}
else if ( xml . name ( ) = = " updated " ) {
// ATOM uses standard compliant date, don't do fancy stuff
QDateTime articleDate = QDateTime : : fromString ( xml . readElementText ( ) . trimmed ( ) , Qt : : ISODate ) ;
article [ " date " ] = ( articleDate . isValid ( ) ? articleDate : QDateTime : : currentDateTime ( ) ) ;
}
else if ( xml . name ( ) = = " author " ) {
xml . readNext ( ) ;
while ( xml . name ( ) ! = " author " ) {
if ( xml . name ( ) = = " name " )
article [ " author " ] = xml . readElementText ( ) . trimmed ( ) ;
xml . readNext ( ) ;
}
}
else if ( xml . name ( ) = = " id " ) {
article [ " id " ] = xml . readElementText ( ) . trimmed ( ) ;
}
} }
}
else if ( xml . name ( ) = = " id " )
article [ " id " ] = xml . readElementText ( ) . trimmed ( ) ;
} }
}
if ( ! article . contains ( " torrent_url " ) & & article . contains ( " news_link " ) ) if ( ! article . contains ( " torrent_url " ) & & article . contains ( " news_link " ) )
article [ " torrent_url " ] = article [ " news_link " ] ; article [ " torrent_url " ] = article [ " news_link " ] ;
if ( ! article . contains ( " id " ) ) { if ( ! article . contains ( " id " ) ) {
// Item does not have a guid, fall back to some other identifier
// Item does not have a guid, fall back to some other identifier
const QString link = article . value ( " news_link " ) . toString ( ) ; const QString link = article . value ( " news_link " ) . toString ( ) ;
if ( ! link . isEmpty ( ) ) if ( ! link . isEmpty ( ) ) {
article [ " id " ] = link ; article [ " id " ] = link ;
else { }
const QString title = article . value ( " title " ) . toString ( ) ; else {
if ( ! title . isEmpty ( ) ) const QString title = article . value ( " title " ) . toString ( ) ;
article [ " id " ] = title ; if ( ! title . isEmpty ( ) ) {
else { article [ " id " ] = title ;
qWarning ( ) < < " Item has no guid, link or title, ignoring it... " ; }
return ; else {
} qWarning ( ) < < " Item has no guid, link or title, ignoring it... " ;
return ;
}
}
} }
}
emit newArticle ( feedUrl , article ) ; emit newArticle ( feedUrl , article ) ;
} }
void RssParser : : parseAtomChannel ( QXmlStreamReader & xml , const QString & feedUrl ) void RssParser : : parseAtomChannel ( QXmlStreamReader & xml , const QString & feedUrl )
{ {
qDebug ( ) < < Q_FUNC_INFO < < feedUrl ; qDebug ( ) < < Q_FUNC_INFO < < feedUrl ;
Q_ASSERT ( xml . isStartElement ( ) & & xml . name ( ) = = " feed " ) ; Q_ASSERT ( xml . isStartElement ( ) & & xml . name ( ) = = " feed " ) ;
QString baseURL = xml . attributes ( ) . value ( " xml:base " ) . toString ( ) ; QString baseURL = xml . attributes ( ) . value ( " xml:base " ) . toString ( ) ;
while ( ! xml . atEnd ( ) ) { while ( ! xml . atEnd ( ) ) {
xml . readNext ( ) ; xml . readNext ( ) ;
if ( xml . isStartElement ( ) ) { if ( xml . isStartElement ( ) ) {
if ( xml . name ( ) = = " title " ) { if ( xml . name ( ) = = " title " ) {
QString title = xml . readElementText ( ) ; QString title = xml . readElementText ( ) ;
emit feedTitle ( feedUrl , title ) ; emit feedTitle ( feedUrl , title ) ;
} }
else if ( xml . name ( ) = = " updated " ) { else if ( xml . name ( ) = = " updated " ) {
QString lastBuildDate = xml . readElementText ( ) ; QString lastBuildDate = xml . readElementText ( ) ;
if ( ! lastBuildDate . isEmpty ( ) ) { if ( ! lastBuildDate . isEmpty ( ) ) {
QMutexLocker locker ( & m_mutex ) ; QMutexLocker locker ( & m_mutex ) ;
if ( m_lastBuildDates . value ( feedUrl ) = = lastBuildDate ) { if ( m_lastBuildDates . value ( feedUrl ) = = lastBuildDate ) {
qDebug ( ) < < " The RSS feed has not changed since last time, aborting parsing. " ; qDebug ( ) < < " The RSS feed has not changed since last time, aborting parsing. " ;
return ; return ;
} }
m_lastBuildDates [ feedUrl ] = lastBuildDate ; m_lastBuildDates [ feedUrl ] = lastBuildDate ;
}
}
else if ( xml . name ( ) = = " entry " ) {
parseAtomArticle ( xml , feedUrl , baseURL ) ;
}
} }
}
else if ( xml . name ( ) = = " entry " ) {
parseAtomArticle ( xml , feedUrl , baseURL ) ;
}
} }
}
} }
// read and create items from a rss document
// read and create items from a rss document
void RssParser : : parseFeed ( const ParsingJob & job ) void RssParser : : parseFeed ( const ParsingJob & job )
{ {
qDebug ( ) < < Q_FUNC_INFO < < job . feedUrl < < job . filePath ; qDebug ( ) < < Q_FUNC_INFO < < job . feedUrl < < job . filePath ;
QFile fileRss ( job . filePath ) ; QFile fileRss ( job . filePath ) ;
if ( ! fileRss . open ( QIODevice : : ReadOnly | QIODevice : : Text ) ) { if ( ! fileRss . open ( QIODevice : : ReadOnly | QIODevice : : Text ) ) {
reportFailure ( job , tr ( " Failed to open downloaded RSS file. " ) ) ; reportFailure ( job , tr ( " Failed to open downloaded RSS file. " ) ) ;
return ; return ;
} }
QXmlStreamReader xml ( & fileRss ) ;
QXmlStreamReader xml ( & fileRss ) ;
bool found_channel = false ; bool foundChannel = false ;
while ( xml . readNextStartElement ( ) ) { while ( xml . readNextStartElement ( ) ) {
if ( xml . name ( ) = = " rss " ) { if ( xml . name ( ) = = " rss " ) {
// Find channels
// Find channels
while ( xml . readNextStartElement ( ) ) { while ( xml . readNextStartElement ( ) ) {
if ( xml . name ( ) = = " channel " ) { if ( xml . name ( ) = = " channel " ) {
parseRSSChannel ( xml , job . feedUrl ) ; parseRSSChannel ( xml , job . feedUrl ) ;
found_channel = true ; foundChannel = true ;
break ; break ;
} else { }
qDebug ( ) < < " Skip rss item: " < < xml . name ( ) ; else {
xml . skipCurrentElement ( ) ; qDebug ( ) < < " Skip rss item: " < < xml . name ( ) ;
xml . skipCurrentElement ( ) ;
}
}
break ;
}
else if ( xml . name ( ) = = " feed " ) { // Atom feed
parseAtomChannel ( xml , job . feedUrl ) ;
foundChannel = true ;
break ;
}
else {
qDebug ( ) < < " Skip root item: " < < xml . name ( ) ;
xml . skipCurrentElement ( ) ;
} }
}
break ;
} }
else if ( xml . name ( ) = = " feed " ) { // Atom feed
parseAtomChannel ( xml , job . feedUrl ) ; if ( xml . hasError ( ) ) {
found_channel = true ; reportFailure ( job , xml . errorString ( ) ) ;
break ; return ;
} else {
qDebug ( ) < < " Skip root item: " < < xml . name ( ) ;
xml . skipCurrentElement ( ) ;
} }
}
if ( ! foundChannel ) {
if ( xml . hasError ( ) ) { reportFailure ( job , tr ( " Invalid RSS feed at '%1'. " ) . arg ( job . feedUrl ) ) ;
reportFailure ( job , xml . errorString ( ) ) ; return ;
return ; }
}
// Clean up
if ( ! found_channel ) { fileRss . close ( ) ;
reportFailure ( job , tr ( " Invalid RSS feed at '%1'. " ) . arg ( job . feedUrl ) ) ; emit feedParsingFinished ( job . feedUrl , QString ( ) ) ;
return ; Utils : : Fs : : forceRemove ( job . filePath ) ;
}
// Clean up
fileRss . close ( ) ;
emit feedParsingFinished ( job . feedUrl , QString ( ) ) ;
Utils : : Fs : : forceRemove ( job . filePath ) ;
} }
void RssParser : : reportFailure ( const ParsingJob & job , const QString & error ) void RssParser : : reportFailure ( const ParsingJob & job , const QString & error )
{ {
emit feedParsingFinished ( job . feedUrl , error ) ; emit feedParsingFinished ( job . feedUrl , error ) ;
Utils : : Fs : : forceRemove ( job . filePath ) ; Utils : : Fs : : forceRemove ( job . filePath ) ;
} }
Vladimir Golovnev (Glassez)
9 years ago
committed by