d4708/qBittorrent
1
0
Fork 0
mirror of https://github.com/d47081/qBittorrent.git synced 2025-01-11 07:18:08 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #5287 from elFarto/master

Browse Source 
Implement RSS Smart Filter
This commit is contained in:
Vladimir Golovnev 2018-02-16 13:43:38 +03:00 committed by GitHub
commit 98a1c111b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 233 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/base/rss/rss_autodownloader.cpp
View File

@ -64,6 +64,7 @@ const QString ConfFolderName(QStringLiteral("rss"));
const QString RulesFileName(QStringLiteral("download_rules.json"));
const QString SettingsKey_ProcessingEnabled(QStringLiteral("RSS/AutoDownloader/EnableProcessing"));
const QString SettingsKey_SmartEpisodeFilter(QStringLiteral("RSS/AutoDownloader/SmartEpisodeFilter"));
namespace
{
@ -95,6 +96,11 @@ using namespace RSS;
QPointer<AutoDownloader> AutoDownloader::m_instance = nullptr;
QString computeSmartFilterRegex(const QStringList &filters)
{
return QString("(?:_|\\b)(?:%1)(?:_|\\b)").arg(filters.join(QString(")|(?:")));
}
AutoDownloader::AutoDownloader()
: m_processingEnabled(SettingsStorage::instance()->loadValue(SettingsKey_ProcessingEnabled, false).toBool())
, m_processingTimer(new QTimer(this))
@ -123,6 +129,13 @@ AutoDownloader::AutoDownloader()
connect(BitTorrent::Session::instance(), &BitTorrent::Session::downloadFromUrlFailed
, this, &AutoDownloader::handleTorrentDownloadFailed);
// initialise the smart episode regex
const QString regex = computeSmartFilterRegex(smartEpisodeFilters());
m_smartEpisodeRegex = QRegularExpression(regex,
QRegularExpression::CaseInsensitiveOption
| QRegularExpression::ExtendedPatternSyntaxOption
| QRegularExpression::UseUnicodePropertiesOption);
load();
m_processingTimer->setSingleShot(true);
@ -266,6 +279,37 @@ void AutoDownloader::importRulesFromLegacyFormat(const QByteArray &data)
insertRule(AutoDownloadRule::fromLegacyDict(val.toHash()));
}
QStringList AutoDownloader::smartEpisodeFilters() const
{
const QVariant filtersSetting = SettingsStorage::instance()->loadValue(SettingsKey_SmartEpisodeFilter);
if (filtersSetting.isNull()) {
QStringList filters = {
"s(\\d+)e(\\d+)", // Format 1: s01e01
"(\\d+)x(\\d+)", // Format 2: 01x01
"(\\d{4}[.\\-]\\d{1,2}[.\\-]\\d{1,2})", // Format 3: 2017.01.01
"(\\d{1,2}[.\\-]\\d{1,2}[.\\-]\\d{4})" // Format 4: 01.01.2017
};
return filters;
}
return filtersSetting.toStringList();
}
QRegularExpression AutoDownloader::smartEpisodeRegex() const
{
return m_smartEpisodeRegex;
}
void AutoDownloader::setSmartEpisodeFilters(const QStringList &filters)
{
SettingsStorage::instance()->storeValue(SettingsKey_SmartEpisodeFilter, filters);
const QString regex = computeSmartFilterRegex(filters);
m_smartEpisodeRegex.setPattern(regex);
}
void AutoDownloader::process()
{
if (m_processingQueue.isEmpty()) return; // processing was disabled
@ -333,6 +377,8 @@ void AutoDownloader::processJob(const QSharedPointer<ProcessingJob> &job)
}
rule.setLastMatch(articleDate);
rule.appendLastComputedEpisode();
m_dirty = true;
storeDeferred();

6
src/base/rss/rss_autodownloader.h
View File

@ -35,6 +35,7 @@
#include <QList>
#include <QObject>
#include <QPointer>
#include <QRegularExpression>
#include <QSharedPointer>
class QThread;
@ -80,6 +81,10 @@ namespace RSS
bool isProcessingEnabled() const;
void setProcessingEnabled(bool enabled);
QStringList smartEpisodeFilters() const;
void setSmartEpisodeFilters(const QStringList &filters);
QRegularExpression smartEpisodeRegex() const;
bool hasRule(const QString &ruleName) const;
AutoDownloadRule ruleByName(const QString &ruleName) const;
QList<AutoDownloadRule> rules() const;
@ -132,5 +137,6 @@ namespace RSS
QHash<QString, QSharedPointer<ProcessingJob>> m_waitingJobs;
bool m_dirty = false;
QBasicTimer m_savingTimer;
QRegularExpression m_smartEpisodeRegex;
};
}

97
src/base/rss/rss_autodownloadrule.cpp
View File

@ -34,7 +34,6 @@
#include <QHash>
#include <QJsonArray>
#include <QJsonObject>
#include <QRegExp>
#include <QRegularExpression>
#include <QSharedData>
#include <QString>
@ -46,6 +45,7 @@
#include "../utils/string.h"
#include "rss_feed.h"
#include "rss_article.h"
#include "rss_autodownloader.h"
namespace
{
@ -100,6 +100,8 @@ const QString Str_AssignedCategory(QStringLiteral("assignedCategory"));
const QString Str_LastMatch(QStringLiteral("lastMatch"));
const QString Str_IgnoreDays(QStringLiteral("ignoreDays"));
const QString Str_AddPaused(QStringLiteral("addPaused"));
const QString Str_SmartFilter(QStringLiteral("smartFilter"));
const QString Str_PreviouslyMatched(QStringLiteral("previouslyMatchedEpisodes"));
namespace RSS
{
@ -120,6 +122,10 @@ namespace RSS
QString category;
TriStateBool addPaused = TriStateBool::Undefined;
bool smartFilter = false;
QStringList previouslyMatchedEpisodes;
mutable QString lastComputedEpisode;
mutable QHash<QString, QRegularExpression> cachedRegexes;
bool operator==(const AutoDownloadRuleData &other) const
@ -135,13 +141,38 @@ namespace RSS
&& (lastMatch == other.lastMatch)
&& (savePath == other.savePath)
&& (category == other.category)
&& (addPaused == other.addPaused);
&& (addPaused == other.addPaused)
&& (smartFilter == other.smartFilter);
}
};
}
using namespace RSS;
QString computeEpisodeName(const QString &article)
{
const QRegularExpression episodeRegex = AutoDownloader::instance()->smartEpisodeRegex();
const QRegularExpressionMatch match = episodeRegex.match(article);
// See if we can extract an season/episode number or date from the title
if (!match.hasMatch())
return QString();
QStringList ret;
for (int i = 1; i <= match.lastCapturedIndex(); ++i) {
QString cap = match.captured(i);
if (cap.isEmpty())
continue;
bool isInt = false;
int x = cap.toInt(&isInt);
ret.append(isInt ? QString::number(x) : cap);
}
return ret.join('x');
}
AutoDownloadRule::AutoDownloadRule(const QString &name)
: m_dataPtr(new AutoDownloadRuleData)
{
@ -197,6 +228,9 @@ bool AutoDownloadRule::matches(const QString &articleTitle, const QString &expre
bool AutoDownloadRule::matches(const QString &articleTitle) const
{
// Reset the lastComputedEpisode, we don't want to leak it between matches
m_dataPtr->lastComputedEpisode.clear();
if (!m_dataPtr->mustContain.empty()) {
bool logged = false;
bool foundMustContain = false;
@ -334,6 +368,20 @@ bool AutoDownloadRule::matches(const QString &articleTitle) const
return false;
}
if (useSmartFilter()) {
// now see if this episode has been downloaded before
const QString episodeStr = computeEpisodeName(articleTitle);
if (!episodeStr.isEmpty()) {
bool previouslyMatched = m_dataPtr->previouslyMatchedEpisodes.contains(episodeStr);
bool isRepack = articleTitle.contains("REPACK", Qt::CaseInsensitive) || articleTitle.contains("PROPER", Qt::CaseInsensitive);
if (previouslyMatched && !isRepack)
return false;
m_dataPtr->lastComputedEpisode = episodeStr;
}
}
// qDebug() << "Matched article:" << articleTitle;
return true;
}
@ -367,7 +415,9 @@ QJsonObject AutoDownloadRule::toJsonObject() const
, {Str_AssignedCategory, assignedCategory()}
, {Str_LastMatch, lastMatch().toString(Qt::RFC2822Date)}
, {Str_IgnoreDays, ignoreDays()}
, {Str_AddPaused, triStateBoolToJsonValue(addPaused())}};
, {Str_AddPaused, triStateBoolToJsonValue(addPaused())}
, {Str_SmartFilter, useSmartFilter()}
, {Str_PreviouslyMatched, QJsonArray::fromStringList(previouslyMatchedEpisodes())}};
}
AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, const QString &name)
@ -384,6 +434,7 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co
rule.setAddPaused(jsonValueToTriStateBool(jsonObj.value(Str_AddPaused)));
rule.setLastMatch(QDateTime::fromString(jsonObj.value(Str_LastMatch).toString(), Qt::RFC2822Date));
rule.setIgnoreDays(jsonObj.value(Str_IgnoreDays).toInt());
rule.setUseSmartFilter(jsonObj.value(Str_SmartFilter).toBool(false));
const QJsonValue feedsVal = jsonObj.value(Str_AffectedFeeds);
QStringList feedURLs;
@ -393,6 +444,17 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co
feedURLs << urlVal.toString();
rule.setFeedURLs(feedURLs);
const QJsonValue previouslyMatchedVal = jsonObj.value(Str_PreviouslyMatched);
QStringList previouslyMatched;
if (previouslyMatchedVal.isString()) {
previouslyMatched << previouslyMatchedVal.toString();
}
else {
foreach (const QJsonValue &val, previouslyMatchedVal.toArray())
previouslyMatched << val.toString();
}
rule.setPreviouslyMatchedEpisodes(previouslyMatched);
return rule;
}
@ -549,6 +611,16 @@ QString AutoDownloadRule::mustNotContain() const
return m_dataPtr->mustNotContain.join("|");
}
bool AutoDownloadRule::useSmartFilter() const
{
return m_dataPtr->smartFilter;
}
void AutoDownloadRule::setUseSmartFilter(bool enabled)
{
m_dataPtr->smartFilter = enabled;
}
bool AutoDownloadRule::useRegex() const
{
return m_dataPtr->useRegex;
@ -560,6 +632,25 @@ void AutoDownloadRule::setUseRegex(bool enabled)
m_dataPtr->cachedRegexes.clear();
}
QStringList AutoDownloadRule::previouslyMatchedEpisodes() const
{
return m_dataPtr->previouslyMatchedEpisodes;
}
void AutoDownloadRule::setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes)
{
m_dataPtr->previouslyMatchedEpisodes = previouslyMatchedEpisodes;
}
void AutoDownloadRule::appendLastComputedEpisode()
{
if (!m_dataPtr->lastComputedEpisode.isEmpty()) {
// TODO: probably need to add a marker for PROPER/REPACK to avoid duplicate downloads
m_dataPtr->previouslyMatchedEpisodes.append(m_dataPtr->lastComputedEpisode);
m_dataPtr->lastComputedEpisode.clear();
}
}
QString AutoDownloadRule::episodeFilter() const
{
return m_dataPtr->episodeFilter;

6
src/base/rss/rss_autodownloadrule.h
View File

@ -66,9 +66,15 @@ namespace RSS
void setLastMatch(const QDateTime &lastMatch);
bool useRegex() const;
void setUseRegex(bool enabled);
bool useSmartFilter() const;
void setUseSmartFilter(bool enabled);
QString episodeFilter() const;
void setEpisodeFilter(const QString &e);
void appendLastComputedEpisode();
QStringList previouslyMatchedEpisodes() const;
void setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes);
QString savePath() const;
void setSavePath(const QString &savePath);
TriStateBool addPaused() const;

4
src/gui/optionsdlg.cpp
View File

@ -377,6 +377,7 @@ OptionsDialog::OptionsDialog(QWidget *parent)
// RSS tab
connect(m_ui->checkRSSEnable, &QCheckBox::toggled, this, &OptionsDialog::enableApplyButton);
connect(m_ui->checkRSSAutoDownloaderEnable, &QCheckBox::toggled, this, &OptionsDialog::enableApplyButton);
connect(m_ui->textSmartEpisodeFilters, &QPlainTextEdit::textChanged, this, &OptionsDialog::enableApplyButton);
connect(m_ui->spinRSSRefreshInterval, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton);
connect(m_ui->spinRSSMaxArticlesPerFeed, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton);
connect(m_ui->btnEditRules, &QPushButton::clicked, [this]() { AutomatedRssDownloader(this).exec(); });
@ -553,6 +554,7 @@ void OptionsDialog::saveOptions()
RSS::Session::instance()->setMaxArticlesPerFeed(m_ui->spinRSSMaxArticlesPerFeed->value());
RSS::Session::instance()->setProcessingEnabled(m_ui->checkRSSEnable->isChecked());
RSS::AutoDownloader::instance()->setProcessingEnabled(m_ui->checkRSSAutoDownloaderEnable->isChecked());
RSS::AutoDownloader::instance()->setSmartEpisodeFilters(m_ui->textSmartEpisodeFilters->toPlainText().split('\n', QString::SplitBehavior::SkipEmptyParts));
auto session = BitTorrent::Session::instance();
@ -780,6 +782,8 @@ void OptionsDialog::loadOptions()
m_ui->checkRSSEnable->setChecked(RSS::Session::instance()->isProcessingEnabled());
m_ui->checkRSSAutoDownloaderEnable->setChecked(RSS::AutoDownloader::instance()->isProcessingEnabled());
m_ui->textSmartEpisodeFilters->setPlainText(RSS::AutoDownloader::instance()->smartEpisodeFilters().join('\n'));
m_ui->spinRSSRefreshInterval->setValue(RSS::Session::instance()->refreshInterval());
m_ui->spinRSSMaxArticlesPerFeed->setValue(RSS::Session::instance()->maxArticlesPerFeed());

14
src/gui/optionsdlg.ui
View File

@ -2699,6 +2699,18 @@
</layout>
</widget>
</item>
<item>
<widget class="QGroupBox" name="groupRSSSmartEpisodeFilter">
<property name="title">
<string>RSS Smart Episode Filters</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout_31">
<item>
<widget class="QPlainTextEdit" name="textSmartEpisodeFilters"/>
</item>
</layout>
</widget>
</item>
<item>
<spacer name="verticalSpacer_5">
<property name="orientation">
@ -2707,7 +2719,7 @@
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>267</height>
<height>200</height>
</size>
</property>
</spacer>

28
src/gui/rss/automatedrssdownloader.cpp
View File

@ -113,6 +113,7 @@ AutomatedRssDownloader::AutomatedRssDownloader(QWidget *parent)
connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::handleRuleDefinitionChanged);
connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::updateMustLineValidity);
connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::updateMustNotLineValidity);
connect(m_ui->checkSmart, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::handleRuleDefinitionChanged);
connect(m_ui->listFeeds, &QListWidget::itemChanged, this, &AutomatedRssDownloader::handleFeedCheckStateChange);
@ -255,6 +256,9 @@ void AutomatedRssDownloader::updateRuleDefinitionBox()
m_ui->checkRegex->blockSignals(true);
m_ui->checkRegex->setChecked(m_currentRule.useRegex());
m_ui->checkRegex->blockSignals(false);
m_ui->checkSmart->blockSignals(true);
m_ui->checkSmart->setChecked(m_currentRule.useSmartFilter());
m_ui->checkSmart->blockSignals(false);
m_ui->comboCategory->setCurrentIndex(m_ui->comboCategory->findText(m_currentRule.assignedCategory()));
if (m_currentRule.assignedCategory().isEmpty())
m_ui->comboCategory->clearEditText();
@ -299,6 +303,7 @@ void AutomatedRssDownloader::clearRuleDefinitionBox()
m_ui->comboCategory->clearEditText();
m_ui->comboCategory->setCurrentIndex(-1);
m_ui->checkRegex->setChecked(false);
m_ui->checkSmart->setChecked(false);
m_ui->spinIgnorePeriod->setValue(0);
m_ui->comboAddPaused->clearEditText();
m_ui->comboAddPaused->setCurrentIndex(-1);
@ -323,6 +328,7 @@ void AutomatedRssDownloader::updateEditedRule()
m_currentRule.setEnabled(m_currentRuleItem->checkState() != Qt::Unchecked);
m_currentRule.setUseRegex(m_ui->checkRegex->isChecked());
m_currentRule.setUseSmartFilter(m_ui->checkSmart->isChecked());
m_currentRule.setMustContain(m_ui->lineContains->text());
m_currentRule.setMustNotContain(m_ui->lineNotContains->text());
m_currentRule.setEpisodeFilter(m_ui->lineEFilter->text());
@ -465,7 +471,9 @@ void AutomatedRssDownloader::displayRulesListMenu()
QAction *addAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-add"), tr("Add new rule..."));
QAction *delAct = nullptr;
QAction *renameAct = nullptr;
QAction *clearAct = nullptr;
const QList<QListWidgetItem *> selection = m_ui->listRules->selectedItems();
if (!selection.isEmpty()) {
if (selection.count() == 1) {
delAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-remove"), tr("Delete rule"));
@ -475,6 +483,8 @@ void AutomatedRssDownloader::displayRulesListMenu()
else {
delAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-remove"), tr("Delete selected rules"));
}
menu.addSeparator();
clearAct = menu.addAction(GuiIconProvider::instance()->getIcon("edit-clear"), tr("Clear downloaded episodes..."));
}
QAction *act = menu.exec(QCursor::pos());
@ -486,6 +496,8 @@ void AutomatedRssDownloader::displayRulesListMenu()
on_removeRuleBtn_clicked();
else if (act == renameAct)
renameSelectedRule();
else if (act == clearAct)
clearSelectedRuleDownloadedEpisodeList();
}
void AutomatedRssDownloader::renameSelectedRule()
@ -518,6 +530,20 @@ void AutomatedRssDownloader::handleRuleCheckStateChange(QListWidgetItem *ruleIte
m_ui->listRules->setCurrentItem(ruleItem);
}
void AutomatedRssDownloader::clearSelectedRuleDownloadedEpisodeList()
{
const QMessageBox::StandardButton reply = QMessageBox::question(
this,
tr("Clear downloaded episodes"),
tr("Are you sure you want to clear the list of downloaded episodes for the selected rule?"),
QMessageBox::Yes | QMessageBox::No);
if (reply == QMessageBox::Yes) {
m_currentRule.setPreviouslyMatchedEpisodes(QStringList());
handleRuleDefinitionChanged();
}
}
void AutomatedRssDownloader::handleFeedCheckStateChange(QListWidgetItem *feedItem)
{
const QString feedURL = feedItem->data(Qt::UserRole).toString();
@ -750,7 +776,7 @@ void AutomatedRssDownloader::handleRuleRenamed(const QString &ruleName, const QS
void AutomatedRssDownloader::handleRuleChanged(const QString &ruleName)
{
auto item = m_itemsByRuleName.value(ruleName);
if (item != m_currentRuleItem)
if (item && (item != m_currentRuleItem))
item->setCheckState(RSS::AutoDownloader::instance()->ruleByName(ruleName).isEnabled() ? Qt::Checked : Qt::Unchecked);
}

1
src/gui/rss/automatedrssdownloader.h
View File

@ -71,6 +71,7 @@ private slots:
void displayRulesListMenu();
void renameSelectedRule();
void updateRuleDefinitionBox();
void clearSelectedRuleDownloadedEpisodeList();
void updateFieldsToolTips(bool regex);
void updateMustLineValidity();
void updateMustNotLineValidity();

38
src/gui/rss/automatedrssdownloader.ui
View File

@ -6,8 +6,8 @@
<rect>
<x>0</x>
<y>0</y>
<width>816</width>
<height>523</height>
<width>818</width>
<height>571</height>
</rect>
</property>
<property name="windowTitle">
@ -106,6 +106,17 @@
</property>
</widget>
</item>
<item>
<widget class="QCheckBox" name="checkSmart">
<property name="toolTip">
<string>Smart Episode Filter will check the episode number to prevent downloading of duplicates.
Supports the formats: S01E01, 1x1, 2017.01.01 and 01.01.2017 (Date formats also support - as a separator)</string>
</property>
<property name="text">
<string>Use Smart Episode Filter</string>
</property>
</widget>
</item>
<item>
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="0">
@ -405,6 +416,9 @@
</item>
<item>
<widget class="QDialogButtonBox" name="buttonBox">
<property name="focusPolicy">
<enum>Qt::StrongFocus</enum>
</property>
<property name="standardButtons">
<set>QDialogButtonBox::Close</set>
</property>
@ -414,6 +428,26 @@
</item>
</layout>
</widget>
<tabstops>
<tabstop>removeRuleBtn</tabstop>
<tabstop>addRuleBtn</tabstop>
<tabstop>listRules</tabstop>
<tabstop>checkRegex</tabstop>
<tabstop>checkSmart</tabstop>
<tabstop>lineContains</tabstop>
<tabstop>lineNotContains</tabstop>
<tabstop>lineEFilter</tabstop>
<tabstop>comboCategory</tabstop>
<tabstop>saveDiffDir_check</tabstop>
<tabstop>lineSavePath</tabstop>
<tabstop>browseSP</tabstop>
<tabstop>spinIgnorePeriod</tabstop>
<tabstop>comboAddPaused</tabstop>
<tabstop>listFeeds</tabstop>
<tabstop>treeMatchingArticles</tabstop>
<tabstop>importBtn</tabstop>
<tabstop>exportBtn</tabstop>
</tabstops>
<resources/>
<connections>
<connection>