Merge pull request #4997 from ngosang/fix_python2_encoding

[Search engine] Python2 encoding and URL getting mechanism in BTDigg

src/searchengine/nova/engines/btdigg.py

@@ -1,5 +1,6 @@
-#VERSION: 1.30
+#VERSION: 1.31
 #AUTHORS: BTDigg team (research@btdigg.org)
+# Contributors: Diego de las Heras (ngosang@hotmail.es)
 
 #                    GNU GENERAL PUBLIC LICENSE
 #                       Version 3, 29 June 2007
@@ -16,12 +17,11 @@
 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 #    GNU General Public License for more details.
 
+from novaprinter import prettyPrinter
+from helpers import retrieve_url, download_file
 import urllib
-import urllib2
 import sys
 
-from novaprinter import prettyPrinter
-
 class btdigg(object):
     url = 'https://btdigg.org'
     name = 'BTDigg'
@@ -37,10 +37,8 @@ class btdigg(object):
         i = 0
         results = 0
         while i < 3:
-            u = urllib2.urlopen('https://api.btdigg.org/api/public-8e9a50f8335b964f/s01?%s' % urllib.urlencode(dict(q = req, p = i)))
-            for line in u:
-                try:
-                    line = line.decode('utf8')
+            data = retrieve_url('https://api.btdigg.org/api/public-8e9a50f8335b964f/s01?%s' % urllib.urlencode(dict(q = req, p = i)))
+            for line in data.splitlines():
                 if line.startswith('#'):
                     continue
 
@@ -60,8 +58,6 @@ class btdigg(object):
 
                 prettyPrinter(res)
                 results += 1
-                except:
-                    pass
 
             if results == 0:
                 break

src/searchengine/nova/engines/torrentz.py

@@ -1,4 +1,4 @@
-#VERSION: 2.17
+#VERSION: 2.18
 #AUTHORS: Diego de las Heras (ngosang@hotmail.es)
 
 # Redistribution and use in source and binary forms, with or without
@@ -92,8 +92,8 @@ class torrentz(object):
                 # display item
                 self.td_counter = None
                 self.current_item['engine_url'] = self.url
-                if self.current_item['name'].find(' \xc2'):
-                    self.current_item['name'] = self.current_item['name'].split(' \xc2')[0]
+                if self.current_item['name'].find(u' \xc2'):
+                    self.current_item['name'] = self.current_item['name'].split(u' \xc2')[0]
                 self.current_item['link'] += '&' + urlencode({'dn' : self.current_item['name']})
                 self.current_item['name'] = self.current_item['name'].decode('utf8')
 

src/searchengine/nova/engines/versions.txt

@@ -1,4 +1,4 @@
-btdigg: 1.30
+btdigg: 1.31
 demonoid: 1.2
 extratorrent: 2.04
 kickasstorrents: 1.28
@@ -6,4 +6,4 @@ legittorrents: 2.00
 mininova: 2.01
 piratebay: 2.14
 torrentreactor: 1.41
-torrentz: 2.17
+torrentz: 2.18

src/searchengine/nova/helpers.py

@@ -1,7 +1,9 @@
-#VERSION: 1.40
+#VERSION: 1.41
 
 # Author:
 #  Christophe DUMEZ (chris@qbittorrent.org)
+# Contributors:
+#  Diego de las Heras (ngosang@hotmail.es)
 
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -86,7 +88,7 @@ def retrieve_url(url):
         pass
     dat = dat.decode(charset, 'replace')
     dat = htmlentitydecode(dat)
-    return dat.encode('utf-8', 'replace')
+    return dat
 
 def download_file(url, referer=None):
     """ Download file at url and write it to a file, return the path to the file and the url """

src/searchengine/nova3/engines/btdigg.py

@@ -1,5 +1,6 @@
-#VERSION: 1.30
+#VERSION: 1.31
 #AUTHORS: BTDigg team (research@btdigg.org)
+# Contributors: Diego de las Heras (ngosang@hotmail.es)
 
 #                    GNU GENERAL PUBLIC LICENSE
 #                       Version 3, 29 June 2007
@@ -16,11 +17,10 @@
 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 #    GNU General Public License for more details.
 
-import urllib.request, urllib.parse, urllib.error
-import urllib.request, urllib.error, urllib.parse
-import sys
-
 from novaprinter import prettyPrinter
+from helpers import retrieve_url, download_file
+import urllib
+import sys
 
 class btdigg(object):
     url = 'https://btdigg.org'
@@ -37,10 +37,8 @@ class btdigg(object):
         i = 0
         results = 0
         while i < 3:
-            u = urllib.request.urlopen('https://api.btdigg.org/api/public-8e9a50f8335b964f/s01?%s' % urllib.parse.urlencode(dict(q = req, p = i)))
-            for line in u:
-                try:
-                    line = line.decode('utf8')
+            data = retrieve_url('https://api.btdigg.org/api/public-8e9a50f8335b964f/s01?%s' % urllib.parse.urlencode(dict(q = req, p = i)))
+            for line in data.splitlines():
                 if line.startswith('#'):
                     continue
 
@@ -60,8 +58,6 @@ class btdigg(object):
 
                 prettyPrinter(res)
                 results += 1
-                except:
-                    pass
 
             if results == 0:
                 break

src/searchengine/nova3/engines/versions.txt

@@ -1,4 +1,4 @@
-btdigg: 1.30
+btdigg: 1.31
 demonoid: 1.2
 extratorrent: 2.04
 kickasstorrents: 1.28