From ad5c88be3d31665f6eba9954443e0b4a4aec3df1 Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Thu, 28 Jun 2018 15:50:50 +0800 Subject: [PATCH 1/7] Simplify function --- src/base/search/searchpluginmanager.cpp | 31 ++++++++++--------------- src/base/search/searchpluginmanager.h | 2 +- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/base/search/searchpluginmanager.cpp b/src/base/search/searchpluginmanager.cpp index 892b3186a..c00bc80f6 100644 --- a/src/base/search/searchpluginmanager.cpp +++ b/src/base/search/searchpluginmanager.cpp @@ -540,32 +540,25 @@ QString SearchPluginManager::pluginPath(const QString &name) return QString("%1/%2.py").arg(pluginsLocation(), name); } -PluginVersion SearchPluginManager::getPluginVersion(QString filePath) +PluginVersion SearchPluginManager::getPluginVersion(const QString &filePath) { - QFile plugin(filePath); - if (!plugin.exists()) { - qDebug("%s plugin does not exist, returning 0.0", qUtf8Printable(filePath)); - return {}; - } - - if (!plugin.open(QIODevice::ReadOnly | QIODevice::Text)) + QFile pluginFile(filePath); + if (!pluginFile.open(QIODevice::ReadOnly | QIODevice::Text)) return {}; - const PluginVersion invalidVersion; - - PluginVersion version; - while (!plugin.atEnd()) { - const QString line = QString(plugin.readLine()).remove(' '); + while (!pluginFile.atEnd()) { + const QString line = QString(pluginFile.readLine()).remove(' '); if (!line.startsWith("#VERSION:", Qt::CaseInsensitive)) continue; const QString versionStr = line.mid(9); - version = PluginVersion::tryParse(versionStr, invalidVersion); - if (version == invalidVersion) { - LogMsg(tr("Search plugin '%1' contains invalid version string ('%2')") - .arg(Utils::Fs::fileName(filePath), line), Log::MsgType::WARNING); - } + const PluginVersion version = PluginVersion::tryParse(versionStr, {}); + if (version.isValid()) + return version; + LogMsg(tr("Search plugin '%1' contains invalid version string ('%2')") + .arg(Utils::Fs::fileName(filePath), versionStr), Log::MsgType::WARNING); break; } - return version; + + return {}; } diff --git a/src/base/search/searchpluginmanager.h b/src/base/search/searchpluginmanager.h index 7f2b93ab3..bf12a745a 100644 --- a/src/base/search/searchpluginmanager.h +++ b/src/base/search/searchpluginmanager.h @@ -79,7 +79,7 @@ public: SearchHandler *startSearch(const QString &pattern, const QString &category, const QStringList &usedPlugins); SearchDownloadHandler *downloadTorrent(const QString &siteUrl, const QString &url); - static PluginVersion getPluginVersion(QString filePath); + static PluginVersion getPluginVersion(const QString &filePath); static QString categoryFullName(const QString &categoryName); QString pluginFullName(const QString &pluginName); static QString pluginsLocation(); From 361afb401b5d59c6435b2a67a86980d66c60c9ee Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Thu, 28 Jun 2018 15:55:14 +0800 Subject: [PATCH 2/7] Replace less-efficient QProcess::setEnvironment Also small refactor --- src/base/search/searchpluginmanager.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/base/search/searchpluginmanager.cpp b/src/base/search/searchpluginmanager.cpp index c00bc80f6..71bc33938 100644 --- a/src/base/search/searchpluginmanager.cpp +++ b/src/base/search/searchpluginmanager.cpp @@ -433,15 +433,13 @@ void SearchPluginManager::updateNova() void SearchPluginManager::update() { QProcess nova; - nova.setEnvironment(QProcess::systemEnvironment()); - QStringList params; - params << Utils::Fs::toNativePath(engineLocation() + "/nova2.py"); - params << "--capabilities"; + nova.setProcessEnvironment(QProcessEnvironment::systemEnvironment()); + + const QStringList params {Utils::Fs::toNativePath(engineLocation() + "/nova2.py"), "--capabilities"}; nova.start(Utils::ForeignApps::pythonInfo().executableName, params, QIODevice::ReadOnly); - nova.waitForStarted(); nova.waitForFinished(); - QString capabilities = QString(nova.readAll()); + QString capabilities = nova.readAll(); QDomDocument xmlDoc; if (!xmlDoc.setContent(capabilities)) { qWarning() << "Could not parse Nova search engine capabilities, msg: " << capabilities.toLocal8Bit().data(); @@ -466,7 +464,8 @@ void SearchPluginManager::update() plugin->fullName = engineElem.elementsByTagName("name").at(0).toElement().text(); plugin->url = engineElem.elementsByTagName("url").at(0).toElement().text(); - foreach (QString cat, engineElem.elementsByTagName("categories").at(0).toElement().text().split(" ")) { + const auto categories = engineElem.elementsByTagName("categories").at(0).toElement().text().split(' '); + for (QString cat : categories) { cat = cat.trimmed(); if (!cat.isEmpty()) plugin->supportedCategories << cat; From 5c50c5b24d7a9f3b1a24869bd20d9f01190d7a5e Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Thu, 28 Jun 2018 16:06:16 +0800 Subject: [PATCH 3/7] Cache SearchPluginManager::engineLocation() result Also the folder is only created on first usage. --- src/base/search/searchpluginmanager.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/base/search/searchpluginmanager.cpp b/src/base/search/searchpluginmanager.cpp index 71bc33938..c94fe5ed5 100644 --- a/src/base/search/searchpluginmanager.cpp +++ b/src/base/search/searchpluginmanager.cpp @@ -322,13 +322,16 @@ QString SearchPluginManager::pluginsLocation() QString SearchPluginManager::engineLocation() { - QString folder = "nova"; - if (Utils::ForeignApps::pythonInfo().version.majorNumber() >= 3) - folder = "nova3"; - const QString location = Utils::Fs::expandPathAbs(specialFolderLocation(SpecialFolder::Data) + folder); - QDir locationDir(location); - if (!locationDir.exists()) + static QString location; + if (location.isEmpty()) { + const QString folder = (Utils::ForeignApps::pythonInfo().version.majorNumber() >= 3) + ? "nova3" : "nova"; + location = Utils::Fs::expandPathAbs(specialFolderLocation(SpecialFolder::Data) + folder); + + const QDir locationDir(location); locationDir.mkpath(locationDir.absolutePath()); + } + return location; } From f951bf678d14792819091081fb3aa1fd02780582 Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Thu, 28 Jun 2018 16:13:52 +0800 Subject: [PATCH 4/7] Rename function --- src/base/search/searchpluginmanager.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/base/search/searchpluginmanager.cpp b/src/base/search/searchpluginmanager.cpp index c94fe5ed5..03579524e 100644 --- a/src/base/search/searchpluginmanager.cpp +++ b/src/base/search/searchpluginmanager.cpp @@ -54,10 +54,10 @@ namespace { - inline void removePythonScriptIfExists(const QString &scriptPath) + void removePythonScript(const QString &path) { - Utils::Fs::forceRemove(scriptPath); - Utils::Fs::forceRemove(scriptPath + "c"); + Utils::Fs::forceRemove(path); + Utils::Fs::forceRemove(path + 'c'); // python2 pyc files } } @@ -209,8 +209,7 @@ void SearchPluginManager::installPlugin_impl(const QString &name, const QString if (QFile::exists(destPath)) { // Backup in case install fails QFile::copy(destPath, destPath + ".bak"); - Utils::Fs::forceRemove(destPath); - Utils::Fs::forceRemove(destPath + "c"); + removePythonScript(destPath); updated = true; } // Copy the plugin @@ -389,13 +388,13 @@ void SearchPluginManager::updateNova() // Copy search plugin files (if necessary) QString filePath = searchDir.absoluteFilePath("nova2.py"); if (getPluginVersion(":/" + novaFolder + "/nova2.py") > getPluginVersion(filePath)) { - removePythonScriptIfExists(filePath); + removePythonScript(filePath); QFile::copy(":/" + novaFolder + "/nova2.py", filePath); } filePath = searchDir.absoluteFilePath("nova2dl.py"); if (getPluginVersion(":/" + novaFolder + "/nova2dl.py") > getPluginVersion(filePath)) { - removePythonScriptIfExists(filePath); + removePythonScript(filePath); QFile::copy(":/" + novaFolder + "/nova2dl.py", filePath); } @@ -404,28 +403,28 @@ void SearchPluginManager::updateNova() filePath = searchDir.absoluteFilePath("novaprinter.py"); if (getPluginVersion(":/" + novaFolder + "/novaprinter.py") > getPluginVersion(filePath)) { - removePythonScriptIfExists(filePath); + removePythonScript(filePath); QFile::copy(":/" + novaFolder + "/novaprinter.py", filePath); } filePath = searchDir.absoluteFilePath("helpers.py"); if (getPluginVersion(":/" + novaFolder + "/helpers.py") > getPluginVersion(filePath)) { - removePythonScriptIfExists(filePath); + removePythonScript(filePath); QFile::copy(":/" + novaFolder + "/helpers.py", filePath); } filePath = searchDir.absoluteFilePath("socks.py"); - removePythonScriptIfExists(filePath); + removePythonScript(filePath); QFile::copy(":/" + novaFolder + "/socks.py", filePath); if (novaFolder.endsWith("nova")) { filePath = searchDir.absoluteFilePath("fix_encoding.py"); - removePythonScriptIfExists(filePath); + removePythonScript(filePath); QFile::copy(":/" + novaFolder + "/fix_encoding.py", filePath); } else if (novaFolder.endsWith("nova3")) { filePath = searchDir.absoluteFilePath("sgmllib3.py"); - removePythonScriptIfExists(filePath); + removePythonScript(filePath); QFile::copy(":/" + novaFolder + "/sgmllib3.py", filePath); } From 77b71e392e757e25f4dbfcf7717673de673b332d Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Thu, 28 Jun 2018 16:16:46 +0800 Subject: [PATCH 5/7] Refactor function In SearchPluginManager::updateNova(), omit removing __pycache__ folder and pyc files, those files will be recreated anyway. Add const to variables --- src/base/search/searchpluginmanager.cpp | 75 +++++++++---------------- 1 file changed, 26 insertions(+), 49 deletions(-) diff --git a/src/base/search/searchpluginmanager.cpp b/src/base/search/searchpluginmanager.cpp index 03579524e..afbe1c7b9 100644 --- a/src/base/search/searchpluginmanager.cpp +++ b/src/base/search/searchpluginmanager.cpp @@ -369,67 +369,44 @@ void SearchPluginManager::pluginDownloadFailed(const QString &url, const QString // Update nova.py search plugin if necessary void SearchPluginManager::updateNova() { - qDebug("Updating nova"); - // create nova directory if necessary - QDir searchDir(engineLocation()); - QString novaFolder = Utils::ForeignApps::pythonInfo().version.majorNumber() >= 3 ? "searchengine/nova3" : "searchengine/nova"; + const QDir searchDir(engineLocation()); + const QString novaFolder = Utils::ForeignApps::pythonInfo().version.majorNumber() >= 3 + ? "searchengine/nova3" : "searchengine/nova"; + QFile packageFile(searchDir.absoluteFilePath("__init__.py")); - packageFile.open(QIODevice::WriteOnly | QIODevice::Text); + packageFile.open(QIODevice::WriteOnly); packageFile.close(); - if (!searchDir.exists("engines")) - searchDir.mkdir("engines"); - Utils::Fs::removeDirRecursive(searchDir.absoluteFilePath("__pycache__")); + + searchDir.mkdir("engines"); QFile packageFile2(searchDir.absolutePath() + "/engines/__init__.py"); - packageFile2.open(QIODevice::WriteOnly | QIODevice::Text); + packageFile2.open(QIODevice::WriteOnly); packageFile2.close(); // Copy search plugin files (if necessary) - QString filePath = searchDir.absoluteFilePath("nova2.py"); - if (getPluginVersion(":/" + novaFolder + "/nova2.py") > getPluginVersion(filePath)) { - removePythonScript(filePath); - QFile::copy(":/" + novaFolder + "/nova2.py", filePath); - } - - filePath = searchDir.absoluteFilePath("nova2dl.py"); - if (getPluginVersion(":/" + novaFolder + "/nova2dl.py") > getPluginVersion(filePath)) { - removePythonScript(filePath); - QFile::copy(":/" + novaFolder + "/nova2dl.py", filePath); - } - - filePath = searchDir.absoluteFilePath("fix_encoding.py"); - QFile::copy(":/" + novaFolder + "/fix_encoding.py", filePath); - - filePath = searchDir.absoluteFilePath("novaprinter.py"); - if (getPluginVersion(":/" + novaFolder + "/novaprinter.py") > getPluginVersion(filePath)) { - removePythonScript(filePath); - QFile::copy(":/" + novaFolder + "/novaprinter.py", filePath); - } + const auto updateFile = [&novaFolder](const QString &filename, const bool compareVersion) + { + const QString filePathBundled = ":/" + novaFolder + '/' + filename; + const QString filePathDisk = QDir(engineLocation()).absoluteFilePath(filename); - filePath = searchDir.absoluteFilePath("helpers.py"); - if (getPluginVersion(":/" + novaFolder + "/helpers.py") > getPluginVersion(filePath)) { - removePythonScript(filePath); - QFile::copy(":/" + novaFolder + "/helpers.py", filePath); - } + if (compareVersion && (getPluginVersion(filePathBundled) <= getPluginVersion(filePathDisk))) + return; - filePath = searchDir.absoluteFilePath("socks.py"); - removePythonScript(filePath); - QFile::copy(":/" + novaFolder + "/socks.py", filePath); + Utils::Fs::forceRemove(filePathDisk); + QFile::copy(filePathBundled, filePathDisk); + }; - if (novaFolder.endsWith("nova")) { - filePath = searchDir.absoluteFilePath("fix_encoding.py"); - removePythonScript(filePath); - QFile::copy(":/" + novaFolder + "/fix_encoding.py", filePath); - } - else if (novaFolder.endsWith("nova3")) { - filePath = searchDir.absoluteFilePath("sgmllib3.py"); - removePythonScript(filePath); - QFile::copy(":/" + novaFolder + "/sgmllib3.py", filePath); - } + updateFile("helpers.py", true); + updateFile("nova2.py", true); + updateFile("nova2dl.py", true); + updateFile("novaprinter.py", true); + updateFile("socks.py", false); - QDir destDir(pluginsLocation()); - Utils::Fs::removeDirRecursive(destDir.absoluteFilePath("__pycache__")); + if (Utils::ForeignApps::pythonInfo().version.majorNumber() >= 3) + updateFile("sgmllib3.py", false); + else + updateFile("fix_encoding.py", false); } void SearchPluginManager::update() From 8c32302377f1ed70a049e05e584cf03eb10dc092 Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Thu, 28 Jun 2018 17:11:18 +0800 Subject: [PATCH 6/7] Clear python cache conditionally Clear the cache artifacts on plugin install and plugin uninstall events. --- src/base/search/searchpluginmanager.cpp | 32 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/base/search/searchpluginmanager.cpp b/src/base/search/searchpluginmanager.cpp index afbe1c7b9..1cca208e6 100644 --- a/src/base/search/searchpluginmanager.cpp +++ b/src/base/search/searchpluginmanager.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -54,10 +55,29 @@ namespace { - void removePythonScript(const QString &path) + void clearPythonCache(const QString &path) { - Utils::Fs::forceRemove(path); - Utils::Fs::forceRemove(path + 'c'); // python2 pyc files + // remove python cache artifacts in `path` and subdirs + + QStringList dirs = {path}; + QDirIterator iter {path, (QDir::AllDirs | QDir::NoDotAndDotDot), QDirIterator::Subdirectories}; + while (iter.hasNext()) + dirs += iter.next(); + + for (const QString &dir : qAsConst(dirs)) { + // python 3: remove "__pycache__" folders + if (dir.endsWith("/__pycache__")) { + Utils::Fs::removeDirRecursive(dir); + continue; + } + + // python 2: remove "*.pyc" files + const QStringList files = QDir(dir).entryList(QDir::Files); + for (const QString file : files) { + if (file.endsWith(".pyc")) + Utils::Fs::forceRemove(file); + } + } } } @@ -169,6 +189,8 @@ void SearchPluginManager::installPlugin(const QString &source) { qDebug("Asked to install plugin at %s", qUtf8Printable(source)); + clearPythonCache(engineLocation()); + if (Utils::Misc::isUrl(source)) { using namespace Net; DownloadHandler *handler = DownloadManager::instance()->downloadUrl(source, true); @@ -209,7 +231,7 @@ void SearchPluginManager::installPlugin_impl(const QString &name, const QString if (QFile::exists(destPath)) { // Backup in case install fails QFile::copy(destPath, destPath + ".bak"); - removePythonScript(destPath); + Utils::Fs::forceRemove(destPath); updated = true; } // Copy the plugin @@ -241,6 +263,8 @@ void SearchPluginManager::installPlugin_impl(const QString &name, const QString bool SearchPluginManager::uninstallPlugin(const QString &name) { + clearPythonCache(engineLocation()); + // remove it from hard drive QDir pluginsFolder(pluginsLocation()); QStringList filters; From 2528c6e39a16af80df4d6aab301e500e75335cbe Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Sun, 1 Jul 2018 21:34:10 +0800 Subject: [PATCH 7/7] Fix python version detection Closes #9146. --- src/base/utils/foreignapps.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/base/utils/foreignapps.cpp b/src/base/utils/foreignapps.cpp index af45243e1..488d0c32c 100644 --- a/src/base/utils/foreignapps.cpp +++ b/src/base/utils/foreignapps.cpp @@ -31,6 +31,7 @@ #include #include +#include #include #include "base/logger.h" @@ -56,8 +57,13 @@ namespace if (outputSplit.size() <= 1) return false; + // User reports: `python --version` -> "Python 3.6.6+" + // So trim off unrelated characters + const QString versionStr = outputSplit[1]; + const int idx = versionStr.indexOf(QRegularExpression("[^\\.\\d]")); + try { - info = {exeName, outputSplit[1]}; + info = {exeName, versionStr.left(idx)}; } catch (const std::runtime_error &err) { return false;