Merge pull request #5287 from elFarto/master

Implement RSS Smart Filter

src/base/rss/rss_autodownloader.cpp

@@ -64,6 +64,7 @@ const QString ConfFolderName(QStringLiteral("rss"));
 const QString RulesFileName(QStringLiteral("download_rules.json"));
 
 const QString SettingsKey_ProcessingEnabled(QStringLiteral("RSS/AutoDownloader/EnableProcessing"));
+const QString SettingsKey_SmartEpisodeFilter(QStringLiteral("RSS/AutoDownloader/SmartEpisodeFilter"));
 
 namespace
 {
@@ -95,6 +96,11 @@ using namespace RSS;
 
 QPointer<AutoDownloader> AutoDownloader::m_instance = nullptr;
 
+QString computeSmartFilterRegex(const QStringList &filters)
+{
+    return QString("(?:_|\\b)(?:%1)(?:_|\\b)").arg(filters.join(QString(")|(?:")));
+}
+
 AutoDownloader::AutoDownloader()
     : m_processingEnabled(SettingsStorage::instance()->loadValue(SettingsKey_ProcessingEnabled, false).toBool())
     , m_processingTimer(new QTimer(this))
@@ -123,6 +129,13 @@ AutoDownloader::AutoDownloader()
     connect(BitTorrent::Session::instance(), &BitTorrent::Session::downloadFromUrlFailed
             , this, &AutoDownloader::handleTorrentDownloadFailed);
 
+    // initialise the smart episode regex
+    const QString regex = computeSmartFilterRegex(smartEpisodeFilters());
+    m_smartEpisodeRegex = QRegularExpression(regex,
+                                             QRegularExpression::CaseInsensitiveOption
+                                             | QRegularExpression::ExtendedPatternSyntaxOption
+                                             | QRegularExpression::UseUnicodePropertiesOption);
+
     load();
 
     m_processingTimer->setSingleShot(true);
@@ -266,6 +279,37 @@ void AutoDownloader::importRulesFromLegacyFormat(const QByteArray &data)
         insertRule(AutoDownloadRule::fromLegacyDict(val.toHash()));
 }
 
+QStringList AutoDownloader::smartEpisodeFilters() const
+{
+    const QVariant filtersSetting = SettingsStorage::instance()->loadValue(SettingsKey_SmartEpisodeFilter);
+
+    if (filtersSetting.isNull()) {
+        QStringList filters = {
+            "s(\\d+)e(\\d+)",                       // Format 1: s01e01
+            "(\\d+)x(\\d+)",                        // Format 2: 01x01
+            "(\\d{4}[.\\-]\\d{1,2}[.\\-]\\d{1,2})", // Format 3: 2017.01.01
+            "(\\d{1,2}[.\\-]\\d{1,2}[.\\-]\\d{4})"  // Format 4: 01.01.2017
+        };
+
+        return filters;
+    }
+
+    return filtersSetting.toStringList();
+}
+
+QRegularExpression AutoDownloader::smartEpisodeRegex() const
+{
+    return m_smartEpisodeRegex;
+}
+
+void AutoDownloader::setSmartEpisodeFilters(const QStringList &filters)
+{
+    SettingsStorage::instance()->storeValue(SettingsKey_SmartEpisodeFilter, filters);
+
+    const QString regex = computeSmartFilterRegex(filters);
+    m_smartEpisodeRegex.setPattern(regex);
+}
+
 void AutoDownloader::process()
 {
     if (m_processingQueue.isEmpty()) return; // processing was disabled
@@ -333,6 +377,8 @@ void AutoDownloader::processJob(const QSharedPointer<ProcessingJob> &job)
         }
 
         rule.setLastMatch(articleDate);
+        rule.appendLastComputedEpisode();
+
         m_dirty = true;
         storeDeferred();
 

src/base/rss/rss_autodownloader.h

@@ -35,6 +35,7 @@
 #include <QList>
 #include <QObject>
 #include <QPointer>
+#include <QRegularExpression>
 #include <QSharedPointer>
 
 class QThread;
@@ -80,6 +81,10 @@ namespace RSS
         bool isProcessingEnabled() const;
         void setProcessingEnabled(bool enabled);
 
+        QStringList smartEpisodeFilters() const;
+        void setSmartEpisodeFilters(const QStringList &filters);
+        QRegularExpression smartEpisodeRegex() const;
+
         bool hasRule(const QString &ruleName) const;
         AutoDownloadRule ruleByName(const QString &ruleName) const;
         QList<AutoDownloadRule> rules() const;
@@ -132,5 +137,6 @@ namespace RSS
         QHash<QString, QSharedPointer<ProcessingJob>> m_waitingJobs;
         bool m_dirty = false;
         QBasicTimer m_savingTimer;
+        QRegularExpression m_smartEpisodeRegex;
     };
 }

src/base/rss/rss_autodownloadrule.cpp

@@ -34,7 +34,6 @@
 #include <QHash>
 #include <QJsonArray>
 #include <QJsonObject>
-#include <QRegExp>
 #include <QRegularExpression>
 #include <QSharedData>
 #include <QString>
@@ -46,6 +45,7 @@
 #include "../utils/string.h"
 #include "rss_feed.h"
 #include "rss_article.h"
+#include "rss_autodownloader.h"
 
 namespace
 {
@@ -100,6 +100,8 @@ const QString Str_AssignedCategory(QStringLiteral("assignedCategory"));
 const QString Str_LastMatch(QStringLiteral("lastMatch"));
 const QString Str_IgnoreDays(QStringLiteral("ignoreDays"));
 const QString Str_AddPaused(QStringLiteral("addPaused"));
+const QString Str_SmartFilter(QStringLiteral("smartFilter"));
+const QString Str_PreviouslyMatched(QStringLiteral("previouslyMatchedEpisodes"));
 
 namespace RSS
 {
@@ -120,6 +122,10 @@ namespace RSS
         QString category;
         TriStateBool addPaused = TriStateBool::Undefined;
 
+        bool smartFilter = false;
+        QStringList previouslyMatchedEpisodes;
+
+        mutable QString lastComputedEpisode;
         mutable QHash<QString, QRegularExpression> cachedRegexes;
 
         bool operator==(const AutoDownloadRuleData &other) const
@@ -135,13 +141,38 @@ namespace RSS
                     && (lastMatch == other.lastMatch)
                     && (savePath == other.savePath)
                     && (category == other.category)
-                    && (addPaused == other.addPaused);
+                    && (addPaused == other.addPaused)
+                    && (smartFilter == other.smartFilter);
         }
     };
 }
 
 using namespace RSS;
 
+QString computeEpisodeName(const QString &article)
+{
+    const QRegularExpression episodeRegex = AutoDownloader::instance()->smartEpisodeRegex();
+    const QRegularExpressionMatch match = episodeRegex.match(article);
+
+    // See if we can extract an season/episode number or date from the title
+    if (!match.hasMatch())
+        return QString();
+
+    QStringList ret;
+    for (int i = 1; i <= match.lastCapturedIndex(); ++i) {
+        QString cap = match.captured(i);
+
+        if (cap.isEmpty())
+            continue;
+
+        bool isInt = false;
+        int x = cap.toInt(&isInt);
+
+        ret.append(isInt ? QString::number(x) : cap);
+    }
+    return ret.join('x');
+}
+
 AutoDownloadRule::AutoDownloadRule(const QString &name)
     : m_dataPtr(new AutoDownloadRuleData)
 {
@@ -197,6 +228,9 @@ bool AutoDownloadRule::matches(const QString &articleTitle, const QString &expre
 
 bool AutoDownloadRule::matches(const QString &articleTitle) const
 {
+    // Reset the lastComputedEpisode, we don't want to leak it between matches
+    m_dataPtr->lastComputedEpisode.clear();
+
     if (!m_dataPtr->mustContain.empty()) {
         bool logged = false;
         bool foundMustContain = false;
@@ -334,6 +368,20 @@ bool AutoDownloadRule::matches(const QString &articleTitle) const
         return false;
     }
 
+    if (useSmartFilter()) {
+        // now see if this episode has been downloaded before
+        const QString episodeStr = computeEpisodeName(articleTitle);
+
+        if (!episodeStr.isEmpty()) {
+            bool previouslyMatched = m_dataPtr->previouslyMatchedEpisodes.contains(episodeStr);
+            bool isRepack = articleTitle.contains("REPACK", Qt::CaseInsensitive) || articleTitle.contains("PROPER", Qt::CaseInsensitive);
+            if (previouslyMatched && !isRepack)
+                return false;
+
+            m_dataPtr->lastComputedEpisode = episodeStr;
+        }
+    }
+
 //    qDebug() << "Matched article:" << articleTitle;
     return true;
 }
@@ -367,7 +415,9 @@ QJsonObject AutoDownloadRule::toJsonObject() const
         , {Str_AssignedCategory, assignedCategory()}
         , {Str_LastMatch, lastMatch().toString(Qt::RFC2822Date)}
         , {Str_IgnoreDays, ignoreDays()}
-        , {Str_AddPaused, triStateBoolToJsonValue(addPaused())}};
+        , {Str_AddPaused, triStateBoolToJsonValue(addPaused())}
+        , {Str_SmartFilter, useSmartFilter()}
+        , {Str_PreviouslyMatched, QJsonArray::fromStringList(previouslyMatchedEpisodes())}};
 }
 
 AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, const QString &name)
@@ -384,6 +434,7 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co
     rule.setAddPaused(jsonValueToTriStateBool(jsonObj.value(Str_AddPaused)));
     rule.setLastMatch(QDateTime::fromString(jsonObj.value(Str_LastMatch).toString(), Qt::RFC2822Date));
     rule.setIgnoreDays(jsonObj.value(Str_IgnoreDays).toInt());
+    rule.setUseSmartFilter(jsonObj.value(Str_SmartFilter).toBool(false));
 
     const QJsonValue feedsVal = jsonObj.value(Str_AffectedFeeds);
     QStringList feedURLs;
@@ -393,6 +444,17 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co
         feedURLs << urlVal.toString();
     rule.setFeedURLs(feedURLs);
 
+    const QJsonValue previouslyMatchedVal = jsonObj.value(Str_PreviouslyMatched);
+    QStringList previouslyMatched;
+    if (previouslyMatchedVal.isString()) {
+        previouslyMatched << previouslyMatchedVal.toString();
+    }
+    else {
+        foreach (const QJsonValue &val, previouslyMatchedVal.toArray())
+            previouslyMatched << val.toString();
+    }
+    rule.setPreviouslyMatchedEpisodes(previouslyMatched);
+
     return rule;
 }
 
@@ -549,6 +611,16 @@ QString AutoDownloadRule::mustNotContain() const
     return m_dataPtr->mustNotContain.join("|");
 }
 
+bool AutoDownloadRule::useSmartFilter() const
+{
+    return m_dataPtr->smartFilter;
+}
+
+void AutoDownloadRule::setUseSmartFilter(bool enabled)
+{
+    m_dataPtr->smartFilter = enabled;
+}
+
 bool AutoDownloadRule::useRegex() const
 {
     return m_dataPtr->useRegex;
@@ -560,6 +632,25 @@ void AutoDownloadRule::setUseRegex(bool enabled)
     m_dataPtr->cachedRegexes.clear();
 }
 
+QStringList AutoDownloadRule::previouslyMatchedEpisodes() const
+{
+    return m_dataPtr->previouslyMatchedEpisodes;
+}
+
+void AutoDownloadRule::setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes)
+{
+    m_dataPtr->previouslyMatchedEpisodes = previouslyMatchedEpisodes;
+}
+
+void AutoDownloadRule::appendLastComputedEpisode()
+{
+    if (!m_dataPtr->lastComputedEpisode.isEmpty()) {
+        // TODO: probably need to add a marker for PROPER/REPACK to avoid duplicate downloads
+        m_dataPtr->previouslyMatchedEpisodes.append(m_dataPtr->lastComputedEpisode);
+        m_dataPtr->lastComputedEpisode.clear();
+    }
+}
+
 QString AutoDownloadRule::episodeFilter() const
 {
     return m_dataPtr->episodeFilter;

src/base/rss/rss_autodownloadrule.h

@@ -66,9 +66,15 @@ namespace RSS
         void setLastMatch(const QDateTime &lastMatch);
         bool useRegex() const;
         void setUseRegex(bool enabled);
+        bool useSmartFilter() const;
+        void setUseSmartFilter(bool enabled);
         QString episodeFilter() const;
         void setEpisodeFilter(const QString &e);
 
+        void appendLastComputedEpisode();
+        QStringList previouslyMatchedEpisodes() const;
+        void setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes);
+
         QString savePath() const;
         void setSavePath(const QString &savePath);
         TriStateBool addPaused() const;

src/gui/optionsdlg.cpp

@@ -377,6 +377,7 @@ OptionsDialog::OptionsDialog(QWidget *parent)
     // RSS tab
     connect(m_ui->checkRSSEnable, &QCheckBox::toggled, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->checkRSSAutoDownloaderEnable, &QCheckBox::toggled, this, &OptionsDialog::enableApplyButton);
+    connect(m_ui->textSmartEpisodeFilters, &QPlainTextEdit::textChanged, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->spinRSSRefreshInterval, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->spinRSSMaxArticlesPerFeed, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->btnEditRules, &QPushButton::clicked, [this]() { AutomatedRssDownloader(this).exec(); });
@@ -553,6 +554,7 @@ void OptionsDialog::saveOptions()
     RSS::Session::instance()->setMaxArticlesPerFeed(m_ui->spinRSSMaxArticlesPerFeed->value());
     RSS::Session::instance()->setProcessingEnabled(m_ui->checkRSSEnable->isChecked());
     RSS::AutoDownloader::instance()->setProcessingEnabled(m_ui->checkRSSAutoDownloaderEnable->isChecked());
+    RSS::AutoDownloader::instance()->setSmartEpisodeFilters(m_ui->textSmartEpisodeFilters->toPlainText().split('\n', QString::SplitBehavior::SkipEmptyParts));
 
     auto session = BitTorrent::Session::instance();
 
@@ -780,6 +782,8 @@ void OptionsDialog::loadOptions()
 
     m_ui->checkRSSEnable->setChecked(RSS::Session::instance()->isProcessingEnabled());
     m_ui->checkRSSAutoDownloaderEnable->setChecked(RSS::AutoDownloader::instance()->isProcessingEnabled());
+    m_ui->textSmartEpisodeFilters->setPlainText(RSS::AutoDownloader::instance()->smartEpisodeFilters().join('\n'));
+
     m_ui->spinRSSRefreshInterval->setValue(RSS::Session::instance()->refreshInterval());
     m_ui->spinRSSMaxArticlesPerFeed->setValue(RSS::Session::instance()->maxArticlesPerFeed());
 

src/gui/optionsdlg.ui

@@ -2699,6 +2699,18 @@
               </layout>
              </widget>
             </item>
+            <item>
+             <widget class="QGroupBox" name="groupRSSSmartEpisodeFilter">
+              <property name="title">
+               <string>RSS Smart Episode Filters</string>
+              </property>
+              <layout class="QVBoxLayout" name="verticalLayout_31">
+               <item>
+                <widget class="QPlainTextEdit" name="textSmartEpisodeFilters"/>
+               </item>
+              </layout>
+             </widget>
+            </item>
             <item>
              <spacer name="verticalSpacer_5">
               <property name="orientation">
@@ -2707,7 +2719,7 @@
               <property name="sizeHint" stdset="0">
                <size>
                 <width>20</width>
-                <height>267</height>
+                <height>200</height>
                </size>
               </property>
              </spacer>

src/gui/rss/automatedrssdownloader.cpp

@@ -113,6 +113,7 @@ AutomatedRssDownloader::AutomatedRssDownloader(QWidget *parent)
     connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::handleRuleDefinitionChanged);
     connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::updateMustLineValidity);
     connect(m_ui->checkRegex, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::updateMustNotLineValidity);
+    connect(m_ui->checkSmart, &QCheckBox::stateChanged, this, &AutomatedRssDownloader::handleRuleDefinitionChanged);
 
     connect(m_ui->listFeeds, &QListWidget::itemChanged, this, &AutomatedRssDownloader::handleFeedCheckStateChange);
 
@@ -255,6 +256,9 @@ void AutomatedRssDownloader::updateRuleDefinitionBox()
         m_ui->checkRegex->blockSignals(true);
         m_ui->checkRegex->setChecked(m_currentRule.useRegex());
         m_ui->checkRegex->blockSignals(false);
+        m_ui->checkSmart->blockSignals(true);
+        m_ui->checkSmart->setChecked(m_currentRule.useSmartFilter());
+        m_ui->checkSmart->blockSignals(false);
         m_ui->comboCategory->setCurrentIndex(m_ui->comboCategory->findText(m_currentRule.assignedCategory()));
         if (m_currentRule.assignedCategory().isEmpty())
             m_ui->comboCategory->clearEditText();
@@ -299,6 +303,7 @@ void AutomatedRssDownloader::clearRuleDefinitionBox()
     m_ui->comboCategory->clearEditText();
     m_ui->comboCategory->setCurrentIndex(-1);
     m_ui->checkRegex->setChecked(false);
+    m_ui->checkSmart->setChecked(false);
     m_ui->spinIgnorePeriod->setValue(0);
     m_ui->comboAddPaused->clearEditText();
     m_ui->comboAddPaused->setCurrentIndex(-1);
@@ -323,6 +328,7 @@ void AutomatedRssDownloader::updateEditedRule()
 
     m_currentRule.setEnabled(m_currentRuleItem->checkState() != Qt::Unchecked);
     m_currentRule.setUseRegex(m_ui->checkRegex->isChecked());
+    m_currentRule.setUseSmartFilter(m_ui->checkSmart->isChecked());
     m_currentRule.setMustContain(m_ui->lineContains->text());
     m_currentRule.setMustNotContain(m_ui->lineNotContains->text());
     m_currentRule.setEpisodeFilter(m_ui->lineEFilter->text());
@@ -465,7 +471,9 @@ void AutomatedRssDownloader::displayRulesListMenu()
     QAction *addAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-add"), tr("Add new rule..."));
     QAction *delAct = nullptr;
     QAction *renameAct = nullptr;
+    QAction *clearAct = nullptr;
     const QList<QListWidgetItem *> selection = m_ui->listRules->selectedItems();
+
     if (!selection.isEmpty()) {
         if (selection.count() == 1) {
             delAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-remove"), tr("Delete rule"));
@@ -475,6 +483,8 @@ void AutomatedRssDownloader::displayRulesListMenu()
         else {
             delAct = menu.addAction(GuiIconProvider::instance()->getIcon("list-remove"), tr("Delete selected rules"));
         }
+        menu.addSeparator();
+        clearAct = menu.addAction(GuiIconProvider::instance()->getIcon("edit-clear"), tr("Clear downloaded episodes..."));
     }
 
     QAction *act = menu.exec(QCursor::pos());
@@ -486,6 +496,8 @@ void AutomatedRssDownloader::displayRulesListMenu()
         on_removeRuleBtn_clicked();
     else if (act == renameAct)
         renameSelectedRule();
+    else if (act == clearAct)
+        clearSelectedRuleDownloadedEpisodeList();
 }
 
 void AutomatedRssDownloader::renameSelectedRule()
@@ -518,6 +530,20 @@ void AutomatedRssDownloader::handleRuleCheckStateChange(QListWidgetItem *ruleIte
     m_ui->listRules->setCurrentItem(ruleItem);
 }
 
+void AutomatedRssDownloader::clearSelectedRuleDownloadedEpisodeList()
+{
+    const QMessageBox::StandardButton reply = QMessageBox::question(
+                this,
+                tr("Clear downloaded episodes"),
+                tr("Are you sure you want to clear the list of downloaded episodes for the selected rule?"),
+                QMessageBox::Yes | QMessageBox::No);
+
+    if (reply == QMessageBox::Yes) {
+        m_currentRule.setPreviouslyMatchedEpisodes(QStringList());
+        handleRuleDefinitionChanged();
+    }
+}
+
 void AutomatedRssDownloader::handleFeedCheckStateChange(QListWidgetItem *feedItem)
 {
     const QString feedURL = feedItem->data(Qt::UserRole).toString();
@@ -750,7 +776,7 @@ void AutomatedRssDownloader::handleRuleRenamed(const QString &ruleName, const QS
 void AutomatedRssDownloader::handleRuleChanged(const QString &ruleName)
 {
     auto item = m_itemsByRuleName.value(ruleName);
-    if (item != m_currentRuleItem)
+    if (item && (item != m_currentRuleItem))
         item->setCheckState(RSS::AutoDownloader::instance()->ruleByName(ruleName).isEnabled() ? Qt::Checked : Qt::Unchecked);
 }
 

src/gui/rss/automatedrssdownloader.h

@@ -71,6 +71,7 @@ private slots:
     void displayRulesListMenu();
     void renameSelectedRule();
     void updateRuleDefinitionBox();
+    void clearSelectedRuleDownloadedEpisodeList();
     void updateFieldsToolTips(bool regex);
     void updateMustLineValidity();
     void updateMustNotLineValidity();

src/gui/rss/automatedrssdownloader.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>816</width>
+    <width>818</width>
-    <height>523</height>
+    <height>571</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -106,6 +106,17 @@
             </property>
            </widget>
           </item>
+          <item>
+           <widget class="QCheckBox" name="checkSmart">
+            <property name="toolTip">
+		    <string>Smart Episode Filter will check the episode number to prevent downloading of duplicates.
+Supports the formats: S01E01, 1x1, 2017.01.01 and 01.01.2017 (Date formats also support - as a separator)</string>
+            </property>
+            <property name="text">
+             <string>Use Smart Episode Filter</string>
+            </property>
+           </widget>
+          </item>
           <item>
            <layout class="QGridLayout" name="gridLayout">
             <item row="0" column="0">
@@ -405,6 +416,9 @@
      </item>
      <item>
       <widget class="QDialogButtonBox" name="buttonBox">
+       <property name="focusPolicy">
+        <enum>Qt::StrongFocus</enum>
+       </property>
        <property name="standardButtons">
         <set>QDialogButtonBox::Close</set>
        </property>
@@ -414,6 +428,26 @@
    </item>
   </layout>
  </widget>
+ <tabstops>
+  <tabstop>removeRuleBtn</tabstop>
+  <tabstop>addRuleBtn</tabstop>
+  <tabstop>listRules</tabstop>
+  <tabstop>checkRegex</tabstop>
+  <tabstop>checkSmart</tabstop>
+  <tabstop>lineContains</tabstop>
+  <tabstop>lineNotContains</tabstop>
+  <tabstop>lineEFilter</tabstop>
+  <tabstop>comboCategory</tabstop>
+  <tabstop>saveDiffDir_check</tabstop>
+  <tabstop>lineSavePath</tabstop>
+  <tabstop>browseSP</tabstop>
+  <tabstop>spinIgnorePeriod</tabstop>
+  <tabstop>comboAddPaused</tabstop>
+  <tabstop>listFeeds</tabstop>
+  <tabstop>treeMatchingArticles</tabstop>
+  <tabstop>importBtn</tabstop>
+  <tabstop>exportBtn</tabstop>
+ </tabstops>
  <resources/>
  <connections>
   <connection>