From d5209d7ddf3cf1536ab85d9c157cd17a1bb2974e Mon Sep 17 00:00:00 2001
From: Douman <douman@gmx.se>
Date: Wed, 10 Feb 2016 16:02:50 +0300
Subject: [PATCH] Align search engine url getting mechanism. Closes #4778

1. Switch to retrieve_url instead of low-level HTTPConnection module usage
---
 src/searchengine/nova/engines/demonoid.py     | 21 ++++----------
 src/searchengine/nova/engines/extratorrent.py | 22 +++++----------
 src/searchengine/nova/engines/mininova.py     | 22 +++++----------
 src/searchengine/nova/engines/piratebay.py    | 28 ++++++-------------
 src/searchengine/nova/engines/versions.txt    |  8 +++---
 src/searchengine/nova3/engines/demonoid.py    | 21 ++++----------
 .../nova3/engines/extratorrent.py             | 22 +++++----------
 src/searchengine/nova3/engines/mininova.py    | 22 +++++----------
 src/searchengine/nova3/engines/piratebay.py   | 22 +++++----------
 src/searchengine/nova3/engines/versions.txt   |  8 +++---
 10 files changed, 64 insertions(+), 132 deletions(-)

diff --git a/src/searchengine/nova/engines/demonoid.py b/src/searchengine/nova/engines/demonoid.py
index b1a549a27..f10e401a5 100644
--- a/src/searchengine/nova/engines/demonoid.py
+++ b/src/searchengine/nova/engines/demonoid.py
@@ -1,4 +1,4 @@
-#VERSION: 1.1
+#VERSION: 1.2
 #AUTHORS: Douman (custparasite@gmx.se)
 #CONTRIBUTORS: Diego de las Heras (ngosang@hotmail.es)
 
@@ -27,13 +27,12 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from HTMLParser import HTMLParser
-from httplib import HTTPSConnection as https
 from re import compile as re_compile
 from re import DOTALL
 from itertools import islice
 #qBt
 from novaprinter import prettyPrinter
-from helpers import download_file
+from helpers import download_file, retrieve_url
 
 class demonoid(object):
     """ Search engine class """
@@ -120,18 +119,12 @@ class demonoid(object):
 
     def search(self, what, cat='all'):
         """ Performs search """
-        connection = https("www.demonoid.pw")
-
         #prepare query
         cat = self.supported_categories[cat.lower()]
-        query = "".join(("/files/?category=", cat, "&subcategory=All&quality=All&seeded=2&external=2&query=", what, "&to=1&uid=0&sort=S"))
+        query = "".join((self.url, "/files/?category=", cat, "&subcategory=All&quality=All&seeded=2&external=2&query=", what, "&to=1&uid=0&sort=S"))
 
-        connection.request("GET", query)
-        response = connection.getresponse()
-        if response.status != 200:
-            return
+        data = retrieve_url(query)
 
-        data = response.read().decode("utf-8")
         add_res_list = re_compile("/files.*page=[0-9]+")
         torrent_list = re_compile("start torrent list -->(.*)<!-- end torrent", DOTALL)
         data = torrent_list.search(data).group(0)
@@ -144,10 +137,8 @@ class demonoid(object):
 
         if list_results:
             for search_query in islice((add_res_list.search(result).group(0) for result in list_results[1].split(" | ")), 0, 5):
-                connection.request("GET", search_query)
-                response = connection.getresponse()
-                parser.feed(torrent_list.search(response.read().decode('utf-8')).group(0))
+                response = retrieve_url(self.url + search_query)
+                parser.feed(torrent_list.search(response).group(0))
                 parser.close()
 
-        connection.close()
         return
diff --git a/src/searchengine/nova/engines/extratorrent.py b/src/searchengine/nova/engines/extratorrent.py
index e8e5d6cec..88f0851ad 100644
--- a/src/searchengine/nova/engines/extratorrent.py
+++ b/src/searchengine/nova/engines/extratorrent.py
@@ -1,4 +1,4 @@
-#VERSION: 2.02
+#VERSION: 2.03
 #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
 #CONTRIBUTORS: Diego de las Heras (ngosang@hotmail.es)
 
@@ -27,10 +27,9 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from HTMLParser import HTMLParser
-from httplib import HTTPConnection as http
 #qBt
 from novaprinter import prettyPrinter
-from helpers import download_file
+from helpers import download_file, retrieve_url
 
 class extratorrent(object):
     """ Search engine class """
@@ -140,25 +139,18 @@ class extratorrent(object):
 
     def search(self, what, cat="all"):
         """ Performs search """
-        connection = http("extratorrent.cc")
+        query = "".join((self.url, "/advanced_search/?with=", what, "&s_cat=", self.supported_categories[cat]))
 
-        query = "".join(("/advanced_search/?with=", what, "&s_cat=", self.supported_categories[cat]))
-
-        connection.request("GET", query)
-        response = connection.getresponse()
-        if response.status != 200:
-            return
+        response = retrieve_url(query)
 
         list_searches = []
         parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
-        parser.feed(response.read().decode('utf-8'))
+        parser.feed(response)
         parser.close()
 
         for search_query in list_searches:
-            connection.request("GET", search_query)
-            response = connection.getresponse()
-            parser.feed(response.read().decode('utf-8'))
+            response = retrieve_url(self.url + search_query)
+            parser.feed(response)
             parser.close()
 
-        connection.close()
         return
diff --git a/src/searchengine/nova/engines/mininova.py b/src/searchengine/nova/engines/mininova.py
index 6eab04499..485af2170 100644
--- a/src/searchengine/nova/engines/mininova.py
+++ b/src/searchengine/nova/engines/mininova.py
@@ -1,4 +1,4 @@
-#VERSION: 2.00
+#VERSION: 2.01
 #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
 #CONTRIBUTORS: Diego de las Heras (ngosang@hotmail.es)
 
@@ -27,9 +27,8 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from HTMLParser import HTMLParser
-from httplib import HTTPConnection as http
 from novaprinter import prettyPrinter
-from helpers import download_file
+from helpers import download_file, retrieve_url
 
 class mininova(object):
     """ Search engine class """
@@ -123,26 +122,19 @@ class mininova(object):
 
     def search(self, what, cat="all"):
         """ Performs search """
-        connection = http("www.mininova.org")
+        query = "/".join((self.url, "search", what, self.supported_categories[cat], "seeds"))
 
-        query = "/".join(("/search", what, self.supported_categories[cat], "seeds"))
-
-        connection.request("GET", query)
-        response = connection.getresponse()
-        if response.status != 200:
-            return
+        response = retrieve_url(query)
 
         list_searches = []
         parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
-        parser.feed(response.read().decode('utf-8'))
+        parser.feed(response)
         parser.close()
 
         parser.next_queries = False
         for search_query in list_searches:
-            connection.request("GET", search_query)
-            response = connection.getresponse()
-            parser.feed(response.read().decode('utf-8'))
+            response = retrieve_url(self.url + search_query)
+            parser.feed(response)
             parser.close()
 
-        connection.close()
         return
diff --git a/src/searchengine/nova/engines/piratebay.py b/src/searchengine/nova/engines/piratebay.py
index 69cb2a0d4..d690d8ad6 100644
--- a/src/searchengine/nova/engines/piratebay.py
+++ b/src/searchengine/nova/engines/piratebay.py
@@ -1,4 +1,4 @@
-#VERSION: 2.13
+#VERSION: 2.14
 #AUTHORS: Fabien Devaux (fab@gnux.info)
 #CONTRIBUTORS: Christophe Dumez (chris@qbittorrent.org)
 #              Arthur (custparasite@gmx.se)
@@ -29,10 +29,9 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from HTMLParser import HTMLParser
-from httplib import HTTPSConnection as https
 #qBt
 from novaprinter import prettyPrinter
-from helpers import download_file
+from helpers import download_file, retrieve_url
 
 class piratebay(object):
     """ Search engine class """
@@ -141,10 +140,8 @@ class piratebay(object):
                 if self.save_item == "size":
                     temp_data = data.split()
                     if "Size" in temp_data:
-                        self.current_item[self.save_item] = temp_data[2]
-                    elif "ULed" in temp_data:
-                        temp_string = self.current_item[self.save_item]
-                        self.current_item[self.save_item] = " ".join((temp_string, temp_data[0][:-1]))
+                        indx = temp_data.index("Size")
+                        self.current_item[self.save_item] = temp_data[indx + 1] + " " + temp_data[indx + 2]
 
                 elif self.save_item == "name":
                     # names with special characters like '&' are splitted in several pieces
@@ -159,28 +156,21 @@ class piratebay(object):
 
     def search(self, what, cat='all'):
         """ Performs search """
-        connection = https("thepiratebay.se")
-
         #prepare query. 7 is filtering by seeders
         cat = cat.lower()
-        query = "/".join(("/search", what, "0", "7", self.supported_categories[cat]))
+        query = "/".join((self.url, "search", what, "0", "7", self.supported_categories[cat]))
 
-        connection.request("GET", query)
-        response = connection.getresponse()
-        if response.status != 200:
-            return
+        response = retrieve_url(query)
 
         list_searches = []
         parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
-        parser.feed(response.read().decode('utf-8'))
+        parser.feed(response)
         parser.close()
 
         parser.add_query = False
         for search_query in list_searches:
-            connection.request("GET", search_query)
-            response = connection.getresponse()
-            parser.feed(response.read().decode('utf-8'))
+            response = retrieve_url(self.url + search_query)
+            parser.feed(response)
             parser.close()
 
-        connection.close()
         return
diff --git a/src/searchengine/nova/engines/versions.txt b/src/searchengine/nova/engines/versions.txt
index 410562fa8..b7578189e 100644
--- a/src/searchengine/nova/engines/versions.txt
+++ b/src/searchengine/nova/engines/versions.txt
@@ -1,9 +1,9 @@
 btdigg: 1.30
-demonoid: 1.1
-extratorrent: 2.02
+demonoid: 1.2
+extratorrent: 2.03
 kickasstorrents: 1.28
 legittorrents: 2.00
-mininova: 2.00
-piratebay: 2.13
+mininova: 2.01
+piratebay: 2.14
 torrentreactor: 1.40
 torrentz: 2.17
diff --git a/src/searchengine/nova3/engines/demonoid.py b/src/searchengine/nova3/engines/demonoid.py
index 4557059d5..296992675 100644
--- a/src/searchengine/nova3/engines/demonoid.py
+++ b/src/searchengine/nova3/engines/demonoid.py
@@ -1,4 +1,4 @@
-#VERSION: 1.1
+#VERSION: 1.2
 #AUTHORS: Douman (custparasite@gmx.se)
 #CONTRIBUTORS: Diego de las Heras (ngosang@hotmail.es)
 
@@ -27,13 +27,12 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from html.parser import HTMLParser
-from http.client import HTTPSConnection as https
 from re import compile as re_compile
 from re import DOTALL
 from itertools import islice
 #qBt
 from novaprinter import prettyPrinter
-from helpers import download_file
+from helpers import download_file, retrieve_url
 
 class demonoid(object):
     """ Search engine class """
@@ -120,18 +119,12 @@ class demonoid(object):
 
     def search(self, what, cat='all'):
         """ Performs search """
-        connection = https("www.demonoid.pw")
-
         #prepare query
         cat = self.supported_categories[cat.lower()]
-        query = "".join(("/files/?category=", cat, "&subcategory=All&quality=All&seeded=2&external=2&query=", what, "&to=1&uid=0&sort=S"))
+        query = "".join((self.url, "/files/?category=", cat, "&subcategory=All&quality=All&seeded=2&external=2&query=", what, "&to=1&uid=0&sort=S"))
 
-        connection.request("GET", query)
-        response = connection.getresponse()
-        if response.status != 200:
-            return
+        data = retrieve_url(query)
 
-        data = response.read().decode("utf-8")
         add_res_list = re_compile("/files.*page=[0-9]+")
         torrent_list = re_compile("start torrent list -->(.*)<!-- end torrent", DOTALL)
         data = torrent_list.search(data).group(0)
@@ -144,10 +137,8 @@ class demonoid(object):
 
         if list_results:
             for search_query in islice((add_res_list.search(result).group(0) for result in list_results[1].split(" | ")), 0, 5):
-                connection.request("GET", search_query)
-                response = connection.getresponse()
-                parser.feed(torrent_list.search(response.read().decode('utf-8')).group(0))
+                response = retrieve_url(self.url + search_query)
+                parser.feed(torrent_list.search(response).group(0))
                 parser.close()
 
-        connection.close()
         return
diff --git a/src/searchengine/nova3/engines/extratorrent.py b/src/searchengine/nova3/engines/extratorrent.py
index 2d2713e14..f6cc3da8f 100644
--- a/src/searchengine/nova3/engines/extratorrent.py
+++ b/src/searchengine/nova3/engines/extratorrent.py
@@ -1,4 +1,4 @@
-#VERSION: 2.02
+#VERSION: 2.03
 #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
 #CONTRIBUTORS: Diego de las Heras (ngosang@hotmail.es)
 
@@ -27,10 +27,9 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from html.parser import HTMLParser
-from http.client import HTTPConnection as http
 #qBt
 from novaprinter import prettyPrinter
-from helpers import download_file
+from helpers import download_file, retrieve_url
 
 class extratorrent(object):
     """ Search engine class """
@@ -140,25 +139,18 @@ class extratorrent(object):
 
     def search(self, what, cat="all"):
         """ Performs search """
-        connection = http("extratorrent.cc")
+        query = "".join((self.url, "/advanced_search/?with=", what, "&s_cat=", self.supported_categories[cat]))
 
-        query = "".join(("/advanced_search/?with=", what, "&s_cat=", self.supported_categories[cat]))
-
-        connection.request("GET", query)
-        response = connection.getresponse()
-        if response.status != 200:
-            return
+        response = retrieve_url(query)
 
         list_searches = []
         parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
-        parser.feed(response.read().decode('utf-8'))
+        parser.feed(response)
         parser.close()
 
         for search_query in list_searches:
-            connection.request("GET", search_query)
-            response = connection.getresponse()
-            parser.feed(response.read().decode('utf-8'))
+            response = retrieve_url(self.url + search_query)
+            parser.feed(response)
             parser.close()
 
-        connection.close()
         return
diff --git a/src/searchengine/nova3/engines/mininova.py b/src/searchengine/nova3/engines/mininova.py
index ef5d94bea..86205f267 100644
--- a/src/searchengine/nova3/engines/mininova.py
+++ b/src/searchengine/nova3/engines/mininova.py
@@ -1,4 +1,4 @@
-#VERSION: 2.00
+#VERSION: 2.01
 #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
 #CONTRIBUTORS: Diego de las Heras (ngosang@hotmail.es)
 
@@ -27,9 +27,8 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from html.parser import HTMLParser
-from http.client import HTTPConnection as http
 from novaprinter import prettyPrinter
-from helpers import download_file
+from helpers import download_file, retrieve_url
 
 class mininova(object):
     """ Search engine class """
@@ -123,26 +122,19 @@ class mininova(object):
 
     def search(self, what, cat="all"):
         """ Performs search """
-        connection = http("www.mininova.org")
+        query = "/".join((self.url, "search", what, self.supported_categories[cat], "seeds"))
 
-        query = "/".join(("/search", what, self.supported_categories[cat], "seeds"))
-
-        connection.request("GET", query)
-        response = connection.getresponse()
-        if response.status != 200:
-            return
+        response = retrieve_url(query)
 
         list_searches = []
         parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
-        parser.feed(response.read().decode('utf-8'))
+        parser.feed(response)
         parser.close()
 
         parser.next_queries = False
         for search_query in list_searches:
-            connection.request("GET", search_query)
-            response = connection.getresponse()
-            parser.feed(response.read().decode('utf-8'))
+            response = retrieve_url(self.url + search_query)
+            parser.feed(response)
             parser.close()
 
-        connection.close()
         return
diff --git a/src/searchengine/nova3/engines/piratebay.py b/src/searchengine/nova3/engines/piratebay.py
index d7951d37f..62793bc7d 100644
--- a/src/searchengine/nova3/engines/piratebay.py
+++ b/src/searchengine/nova3/engines/piratebay.py
@@ -1,4 +1,4 @@
-#VERSION: 2.13
+#VERSION: 2.14
 #AUTHORS: Fabien Devaux (fab@gnux.info)
 #CONTRIBUTORS: Christophe Dumez (chris@qbittorrent.org)
 #              Arthur (custparasite@gmx.se)
@@ -29,10 +29,9 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from html.parser import HTMLParser
-from http.client import HTTPSConnection as https
 #qBt
 from novaprinter import prettyPrinter
-from helpers import download_file
+from helpers import download_file, retrieve_url
 
 class piratebay(object):
     """ Search engine class """
@@ -157,28 +156,21 @@ class piratebay(object):
 
     def search(self, what, cat='all'):
         """ Performs search """
-        connection = https("thepiratebay.se")
-
         #prepare query. 7 is filtering by seeders
         cat = cat.lower()
-        query = "/".join(("/search", what, "0", "7", self.supported_categories[cat]))
+        query = "/".join((self.url, "search", what, "0", "7", self.supported_categories[cat]))
 
-        connection.request("GET", query)
-        response = connection.getresponse()
-        if response.status != 200:
-            return
+        response = retrieve_url(query)
 
         list_searches = []
         parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
-        parser.feed(response.read().decode('utf-8'))
+        parser.feed(response)
         parser.close()
 
         parser.add_query = False
         for search_query in list_searches:
-            connection.request("GET", search_query)
-            response = connection.getresponse()
-            parser.feed(response.read().decode('utf-8'))
+            response = retrieve_url(self.url + search_query)
+            parser.feed(response)
             parser.close()
 
-        connection.close()
         return
diff --git a/src/searchengine/nova3/engines/versions.txt b/src/searchengine/nova3/engines/versions.txt
index 410562fa8..b7578189e 100644
--- a/src/searchengine/nova3/engines/versions.txt
+++ b/src/searchengine/nova3/engines/versions.txt
@@ -1,9 +1,9 @@
 btdigg: 1.30
-demonoid: 1.1
-extratorrent: 2.02
+demonoid: 1.2
+extratorrent: 2.03
 kickasstorrents: 1.28
 legittorrents: 2.00
-mininova: 2.00
-piratebay: 2.13
+mininova: 2.01
+piratebay: 2.14
 torrentreactor: 1.40
 torrentz: 2.17