diff --git a/src/base/rss/rss_autodownloader.cpp b/src/base/rss/rss_autodownloader.cpp index 188658f64..8d0a43dc5 100644 --- a/src/base/rss/rss_autodownloader.cpp +++ b/src/base/rss/rss_autodownloader.cpp @@ -64,6 +64,7 @@ const QString ConfFolderName(QStringLiteral("rss")); const QString RulesFileName(QStringLiteral("download_rules.json")); const QString SettingsKey_ProcessingEnabled(QStringLiteral("RSS/AutoDownloader/EnableProcessing")); +const QString SettingsKey_SmartEpisodeFilter(QStringLiteral("RSS/AutoDownloader/SmartEpisodeFilter")); namespace { @@ -95,6 +96,11 @@ using namespace RSS; QPointer AutoDownloader::m_instance = nullptr; +QString computeSmartFilterRegex(const QStringList &filters) +{ + return QString("(?:_|\\b)(?:%1)(?:_|\\b)").arg(filters.join(QString(")|(?:"))); +} + AutoDownloader::AutoDownloader() : m_processingEnabled(SettingsStorage::instance()->loadValue(SettingsKey_ProcessingEnabled, false).toBool()) , m_processingTimer(new QTimer(this)) @@ -123,6 +129,13 @@ AutoDownloader::AutoDownloader() connect(BitTorrent::Session::instance(), &BitTorrent::Session::downloadFromUrlFailed , this, &AutoDownloader::handleTorrentDownloadFailed); + // initialise the smart episode regex + const QString regex = computeSmartFilterRegex(smartEpisodeFilters()); + m_smartEpisodeRegex = QRegularExpression(regex, + QRegularExpression::CaseInsensitiveOption + | QRegularExpression::ExtendedPatternSyntaxOption + | QRegularExpression::UseUnicodePropertiesOption); + load(); m_processingTimer->setSingleShot(true); @@ -266,6 +279,37 @@ void AutoDownloader::importRulesFromLegacyFormat(const QByteArray &data) insertRule(AutoDownloadRule::fromLegacyDict(val.toHash())); } +QStringList AutoDownloader::smartEpisodeFilters() const +{ + const QVariant filtersSetting = SettingsStorage::instance()->loadValue(SettingsKey_SmartEpisodeFilter); + + if (filtersSetting.isNull()) { + QStringList filters = { + "s(\\d+)e(\\d+)", // Format 1: s01e01 + "(\\d+)x(\\d+)", // Format 2: 01x01 + "(\\d{4}[.\\-]\\d{1,2}[.\\-]\\d{1,2})", // Format 3: 2017.01.01 + "(\\d{1,2}[.\\-]\\d{1,2}[.\\-]\\d{4})" // Format 4: 01.01.2017 + }; + + return filters; + } + + return filtersSetting.toStringList(); +} + +QRegularExpression AutoDownloader::smartEpisodeRegex() const +{ + return m_smartEpisodeRegex; +} + +void AutoDownloader::setSmartEpisodeFilters(const QStringList &filters) +{ + SettingsStorage::instance()->storeValue(SettingsKey_SmartEpisodeFilter, filters); + + const QString regex = computeSmartFilterRegex(filters); + m_smartEpisodeRegex.setPattern(regex); +} + void AutoDownloader::process() { if (m_processingQueue.isEmpty()) return; // processing was disabled @@ -333,6 +377,8 @@ void AutoDownloader::processJob(const QSharedPointer &job) } rule.setLastMatch(articleDate); + rule.appendLastComputedEpisode(); + m_dirty = true; storeDeferred(); diff --git a/src/base/rss/rss_autodownloader.h b/src/base/rss/rss_autodownloader.h index ba9c0ac39..5a56b488e 100644 --- a/src/base/rss/rss_autodownloader.h +++ b/src/base/rss/rss_autodownloader.h @@ -35,6 +35,7 @@ #include #include #include +#include #include class QThread; @@ -80,6 +81,10 @@ namespace RSS bool isProcessingEnabled() const; void setProcessingEnabled(bool enabled); + QStringList smartEpisodeFilters() const; + void setSmartEpisodeFilters(const QStringList &filters); + QRegularExpression smartEpisodeRegex() const; + bool hasRule(const QString &ruleName) const; AutoDownloadRule ruleByName(const QString &ruleName) const; QList rules() const; @@ -132,5 +137,6 @@ namespace RSS QHash> m_waitingJobs; bool m_dirty = false; QBasicTimer m_savingTimer; + QRegularExpression m_smartEpisodeRegex; }; } diff --git a/src/base/rss/rss_autodownloadrule.cpp b/src/base/rss/rss_autodownloadrule.cpp index 83a9c343e..dda3c2d07 100644 --- a/src/base/rss/rss_autodownloadrule.cpp +++ b/src/base/rss/rss_autodownloadrule.cpp @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -46,6 +45,7 @@ #include "../utils/string.h" #include "rss_feed.h" #include "rss_article.h" +#include "rss_autodownloader.h" namespace { @@ -100,6 +100,8 @@ const QString Str_AssignedCategory(QStringLiteral("assignedCategory")); const QString Str_LastMatch(QStringLiteral("lastMatch")); const QString Str_IgnoreDays(QStringLiteral("ignoreDays")); const QString Str_AddPaused(QStringLiteral("addPaused")); +const QString Str_SmartFilter(QStringLiteral("smartFilter")); +const QString Str_PreviouslyMatched(QStringLiteral("previouslyMatchedEpisodes")); namespace RSS { @@ -120,6 +122,10 @@ namespace RSS QString category; TriStateBool addPaused = TriStateBool::Undefined; + bool smartFilter = false; + QStringList previouslyMatchedEpisodes; + + mutable QString lastComputedEpisode; mutable QHash cachedRegexes; bool operator==(const AutoDownloadRuleData &other) const @@ -135,13 +141,38 @@ namespace RSS && (lastMatch == other.lastMatch) && (savePath == other.savePath) && (category == other.category) - && (addPaused == other.addPaused); + && (addPaused == other.addPaused) + && (smartFilter == other.smartFilter); } }; } using namespace RSS; +QString computeEpisodeName(const QString &article) +{ + const QRegularExpression episodeRegex = AutoDownloader::instance()->smartEpisodeRegex(); + const QRegularExpressionMatch match = episodeRegex.match(article); + + // See if we can extract an season/episode number or date from the title + if (!match.hasMatch()) + return QString(); + + QStringList ret; + for (int i = 1; i <= match.lastCapturedIndex(); ++i) { + QString cap = match.captured(i); + + if (cap.isEmpty()) + continue; + + bool isInt = false; + int x = cap.toInt(&isInt); + + ret.append(isInt ? QString::number(x) : cap); + } + return ret.join('x'); +} + AutoDownloadRule::AutoDownloadRule(const QString &name) : m_dataPtr(new AutoDownloadRuleData) { @@ -197,6 +228,9 @@ bool AutoDownloadRule::matches(const QString &articleTitle, const QString &expre bool AutoDownloadRule::matches(const QString &articleTitle) const { + // Reset the lastComputedEpisode, we don't want to leak it between matches + m_dataPtr->lastComputedEpisode.clear(); + if (!m_dataPtr->mustContain.empty()) { bool logged = false; bool foundMustContain = false; @@ -334,6 +368,20 @@ bool AutoDownloadRule::matches(const QString &articleTitle) const return false; } + if (useSmartFilter()) { + // now see if this episode has been downloaded before + const QString episodeStr = computeEpisodeName(articleTitle); + + if (!episodeStr.isEmpty()) { + bool previouslyMatched = m_dataPtr->previouslyMatchedEpisodes.contains(episodeStr); + bool isRepack = articleTitle.contains("REPACK", Qt::CaseInsensitive) || articleTitle.contains("PROPER", Qt::CaseInsensitive); + if (previouslyMatched && !isRepack) + return false; + + m_dataPtr->lastComputedEpisode = episodeStr; + } + } + // qDebug() << "Matched article:" << articleTitle; return true; } @@ -367,7 +415,9 @@ QJsonObject AutoDownloadRule::toJsonObject() const , {Str_AssignedCategory, assignedCategory()} , {Str_LastMatch, lastMatch().toString(Qt::RFC2822Date)} , {Str_IgnoreDays, ignoreDays()} - , {Str_AddPaused, triStateBoolToJsonValue(addPaused())}}; + , {Str_AddPaused, triStateBoolToJsonValue(addPaused())} + , {Str_SmartFilter, useSmartFilter()} + , {Str_PreviouslyMatched, QJsonArray::fromStringList(previouslyMatchedEpisodes())}}; } AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, const QString &name) @@ -384,6 +434,7 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co rule.setAddPaused(jsonValueToTriStateBool(jsonObj.value(Str_AddPaused))); rule.setLastMatch(QDateTime::fromString(jsonObj.value(Str_LastMatch).toString(), Qt::RFC2822Date)); rule.setIgnoreDays(jsonObj.value(Str_IgnoreDays).toInt()); + rule.setUseSmartFilter(jsonObj.value(Str_SmartFilter).toBool(false)); const QJsonValue feedsVal = jsonObj.value(Str_AffectedFeeds); QStringList feedURLs; @@ -393,6 +444,17 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co feedURLs << urlVal.toString(); rule.setFeedURLs(feedURLs); + const QJsonValue previouslyMatchedVal = jsonObj.value(Str_PreviouslyMatched); + QStringList previouslyMatched; + if (previouslyMatchedVal.isString()) { + previouslyMatched << previouslyMatchedVal.toString(); + } + else { + foreach (const QJsonValue &val, previouslyMatchedVal.toArray()) + previouslyMatched << val.toString(); + } + rule.setPreviouslyMatchedEpisodes(previouslyMatched); + return rule; } @@ -549,6 +611,16 @@ QString AutoDownloadRule::mustNotContain() const return m_dataPtr->mustNotContain.join("|"); } +bool AutoDownloadRule::useSmartFilter() const +{ + return m_dataPtr->smartFilter; +} + +void AutoDownloadRule::setUseSmartFilter(bool enabled) +{ + m_dataPtr->smartFilter = enabled; +} + bool AutoDownloadRule::useRegex() const { return m_dataPtr->useRegex; @@ -560,6 +632,25 @@ void AutoDownloadRule::setUseRegex(bool enabled) m_dataPtr->cachedRegexes.clear(); } +QStringList AutoDownloadRule::previouslyMatchedEpisodes() const +{ + return m_dataPtr->previouslyMatchedEpisodes; +} + +void AutoDownloadRule::setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes) +{ + m_dataPtr->previouslyMatchedEpisodes = previouslyMatchedEpisodes; +} + +void AutoDownloadRule::appendLastComputedEpisode() +{ + if (!m_dataPtr->lastComputedEpisode.isEmpty()) { + // TODO: probably need to add a marker for PROPER/REPACK to avoid duplicate downloads + m_dataPtr->previouslyMatchedEpisodes.append(m_dataPtr->lastComputedEpisode); + m_dataPtr->lastComputedEpisode.clear(); + } +} + QString AutoDownloadRule::episodeFilter() const { return m_dataPtr->episodeFilter; diff --git a/src/base/rss/rss_autodownloadrule.h b/src/base/rss/rss_autodownloadrule.h index b28d7b47a..ec6c76979 100644 --- a/src/base/rss/rss_autodownloadrule.h +++ b/src/base/rss/rss_autodownloadrule.h @@ -66,9 +66,15 @@ namespace RSS void setLastMatch(const QDateTime &lastMatch); bool useRegex() const; void setUseRegex(bool enabled); + bool useSmartFilter() const; + void setUseSmartFilter(bool enabled); QString episodeFilter() const; void setEpisodeFilter(const QString &e); + void appendLastComputedEpisode(); + QStringList previouslyMatchedEpisodes() const; + void setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes); + QString savePath() const; void setSavePath(const QString &savePath); TriStateBool addPaused() const; diff --git a/src/gui/optionsdlg.cpp b/src/gui/optionsdlg.cpp index 95a43d0dc..360b664d4 100644 --- a/src/gui/optionsdlg.cpp +++ b/src/gui/optionsdlg.cpp @@ -377,6 +377,7 @@ OptionsDialog::OptionsDialog(QWidget *parent) // RSS tab connect(m_ui->checkRSSEnable, &QCheckBox::toggled, this, &OptionsDialog::enableApplyButton); connect(m_ui->checkRSSAutoDownloaderEnable, &QCheckBox::toggled, this, &OptionsDialog::enableApplyButton); + connect(m_ui->textSmartEpisodeFilters, &QPlainTextEdit::textChanged, this, &OptionsDialog::enableApplyButton); connect(m_ui->spinRSSRefreshInterval, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton); connect(m_ui->spinRSSMaxArticlesPerFeed, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton); connect(m_ui->btnEditRules, &QPushButton::clicked, [this]() { AutomatedRssDownloader(this).exec(); }); @@ -553,6 +554,7 @@ void OptionsDialog::saveOptions() RSS::Session::instance()->setMaxArticlesPerFeed(m_ui->spinRSSMaxArticlesPerFeed->value()); RSS::Session::instance()->setProcessingEnabled(m_ui->checkRSSEnable->isChecked()); RSS::AutoDownloader::instance()->setProcessingEnabled(m_ui->checkRSSAutoDownloaderEnable->isChecked()); + RSS::AutoDownloader::instance()->setSmartEpisodeFilters(m_ui->textSmartEpisodeFilters->toPlainText().split('\n', QString::SplitBehavior::SkipEmptyParts)); auto session = BitTorrent::Session::instance(); @@ -780,6 +782,8 @@ void OptionsDialog::loadOptions() m_ui->checkRSSEnable->setChecked(RSS::Session::instance()->isProcessingEnabled()); m_ui->checkRSSAutoDownloaderEnable->setChecked(RSS::AutoDownloader::instance()->isProcessingEnabled()); + m_ui->textSmartEpisodeFilters->setPlainText(RSS::AutoDownloader::instance()->smartEpisodeFilters().join('\n')); + m_ui->spinRSSRefreshInterval->setValue(RSS::Session::instance()->refreshInterval()); m_ui->spinRSSMaxArticlesPerFeed->setValue(RSS::Session::instance()->maxArticlesPerFeed()); diff --git a/src/gui/optionsdlg.ui b/src/gui/optionsdlg.ui index 4319b27c3..25a39397b 100644 --- a/src/gui/optionsdlg.ui +++ b/src/gui/optionsdlg.ui @@ -2699,6 +2699,18 @@ + + + + RSS Smart Episode Filters + + + + + + + + @@ -2707,7 +2719,7 @@ 20 - 267 + 200 diff --git a/src/gui/rss/automatedrssdownloader.cpp b/src/gui/rss/automatedrssdownloader.cpp index a00ab4750..133659961 100644 --- a/src/gui/rss/automatedrssdownloader.cpp +++ b/src/gui/rss/automatedrssdownloader.cpp @@ -113,6 +113,7 @@ AutomatedRssDownloader::AutomatedRssDownloader(QWidget *parent) connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::handleRuleDefinitionChanged); connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::updateMustLineValidity); connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::updateMustNotLineValidity); + connect(m_ui->checkSmart, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::handleRuleDefinitionChanged); connect(m_ui->listFeeds, &QListWidget::itemChanged, this, &AutomatedRssDownloader::handleFeedCheckStateChange); @@ -255,6 +256,9 @@ void AutomatedRssDownloader::updateRuleDefinitionBox() m_ui->checkRegex->blockSignals(true); m_ui->checkRegex->setChecked(m_currentRule.useRegex()); m_ui->checkRegex->blockSignals(false); + m_ui->checkSmart->blockSignals(true); + m_ui->checkSmart->setChecked(m_currentRule.useSmartFilter()); + m_ui->checkSmart->blockSignals(false); m_ui->comboCategory->setCurrentIndex(m_ui->comboCategory->findText(m_currentRule.assignedCategory())); if (m_currentRule.assignedCategory().isEmpty()) m_ui->comboCategory->clearEditText(); @@ -299,6 +303,7 @@ void AutomatedRssDownloader::clearRuleDefinitionBox() m_ui->comboCategory->clearEditText(); m_ui->comboCategory->setCurrentIndex(-1); m_ui->checkRegex->setChecked(false); + m_ui->checkSmart->setChecked(false); m_ui->spinIgnorePeriod->setValue(0); m_ui->comboAddPaused->clearEditText(); m_ui->comboAddPaused->setCurrentIndex(-1); @@ -323,6 +328,7 @@ void AutomatedRssDownloader::updateEditedRule() m_currentRule.setEnabled(m_currentRuleItem->checkState() != Qt::Unchecked); m_currentRule.setUseRegex(m_ui->checkRegex->isChecked()); + m_currentRule.setUseSmartFilter(m_ui->checkSmart->isChecked()); m_currentRule.setMustContain(m_ui->lineContains->text()); m_currentRule.setMustNotContain(m_ui->lineNotContains->text()); m_currentRule.setEpisodeFilter(m_ui->lineEFilter->text()); @@ -465,7 +471,9 @@ void AutomatedRssDownloader::displayRulesListMenu() QAction *addAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-add"), tr("Add new rule...")); QAction *delAct = nullptr; QAction *renameAct = nullptr; + QAction *clearAct = nullptr; const QList selection = m_ui->listRules->selectedItems(); + if (!selection.isEmpty()) { if (selection.count() == 1) { delAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-remove"), tr("Delete rule")); @@ -475,6 +483,8 @@ void AutomatedRssDownloader::displayRulesListMenu() else { delAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-remove"), tr("Delete selected rules")); } + menu.addSeparator(); + clearAct = menu.addAction(GuiIconProvider::instance()->getIcon("edit-clear"), tr("Clear downloaded episodes...")); } QAction *act = menu.exec(QCursor::pos()); @@ -486,6 +496,8 @@ void AutomatedRssDownloader::displayRulesListMenu() on_removeRuleBtn_clicked(); else if (act == renameAct) renameSelectedRule(); + else if (act == clearAct) + clearSelectedRuleDownloadedEpisodeList(); } void AutomatedRssDownloader::renameSelectedRule() @@ -518,6 +530,20 @@ void AutomatedRssDownloader::handleRuleCheckStateChange(QListWidgetItem *ruleIte m_ui->listRules->setCurrentItem(ruleItem); } +void AutomatedRssDownloader::clearSelectedRuleDownloadedEpisodeList() +{ + const QMessageBox::StandardButton reply = QMessageBox::question( + this, + tr("Clear downloaded episodes"), + tr("Are you sure you want to clear the list of downloaded episodes for the selected rule?"), + QMessageBox::Yes | QMessageBox::No); + + if (reply == QMessageBox::Yes) { + m_currentRule.setPreviouslyMatchedEpisodes(QStringList()); + handleRuleDefinitionChanged(); + } +} + void AutomatedRssDownloader::handleFeedCheckStateChange(QListWidgetItem *feedItem) { const QString feedURL = feedItem->data(Qt::UserRole).toString(); @@ -750,7 +776,7 @@ void AutomatedRssDownloader::handleRuleRenamed(const QString &ruleName, const QS void AutomatedRssDownloader::handleRuleChanged(const QString &ruleName) { auto item = m_itemsByRuleName.value(ruleName); - if (item != m_currentRuleItem) + if (item && (item != m_currentRuleItem)) item->setCheckState(RSS::AutoDownloader::instance()->ruleByName(ruleName).isEnabled() ? Qt::Checked : Qt::Unchecked); } diff --git a/src/gui/rss/automatedrssdownloader.h b/src/gui/rss/automatedrssdownloader.h index d336fbc60..5d24fe315 100644 --- a/src/gui/rss/automatedrssdownloader.h +++ b/src/gui/rss/automatedrssdownloader.h @@ -71,6 +71,7 @@ private slots: void displayRulesListMenu(); void renameSelectedRule(); void updateRuleDefinitionBox(); + void clearSelectedRuleDownloadedEpisodeList(); void updateFieldsToolTips(bool regex); void updateMustLineValidity(); void updateMustNotLineValidity(); diff --git a/src/gui/rss/automatedrssdownloader.ui b/src/gui/rss/automatedrssdownloader.ui index e77b1385a..5ae379665 100644 --- a/src/gui/rss/automatedrssdownloader.ui +++ b/src/gui/rss/automatedrssdownloader.ui @@ -6,8 +6,8 @@ 0 0 - 816 - 523 + 818 + 571 @@ -106,6 +106,17 @@ + + + + Smart Episode Filter will check the episode number to prevent downloading of duplicates. +Supports the formats: S01E01, 1x1, 2017.01.01 and 01.01.2017 (Date formats also support - as a separator) + + + Use Smart Episode Filter + + + @@ -405,6 +416,9 @@ + + Qt::StrongFocus + QDialogButtonBox::Close @@ -414,6 +428,26 @@ + + removeRuleBtn + addRuleBtn + listRules + checkRegex + checkSmart + lineContains + lineNotContains + lineEFilter + comboCategory + saveDiffDir_check + lineSavePath + browseSP + spinIgnorePeriod + comboAddPaused + listFeeds + treeMatchingArticles + importBtn + exportBtn +