From a2e9210665babdde44c9c0d7b1bc46ad9652e59d Mon Sep 17 00:00:00 2001
From: Christophe Dumez <chris@qbittorrent.org>
Date: Thu, 26 Mar 2009 16:49:44 +0000
Subject: [PATCH] - Fixed unicode in mininova and btjunkie search engines

---
 src/search.qrc                         |  3 +-
 src/searchEngine.cpp                   |  8 ++++
 src/search_engine/engines/btjunkie.py  | 16 +++----
 src/search_engine/engines/mininova.py  | 11 +++--
 src/search_engine/engines/versions.txt |  4 +-
 src/search_engine/helpers.py           | 59 ++++++++++++++++++++++++++
 src/search_engine/nova2.py             |  8 ++--
 src/search_engine/novaprinter.py       |  8 ++--
 8 files changed, 95 insertions(+), 22 deletions(-)
 create mode 100644 src/search_engine/helpers.py

diff --git a/src/search.qrc b/src/search.qrc
index 074110bca..735122372 100644
--- a/src/search.qrc
+++ b/src/search.qrc
@@ -2,6 +2,7 @@
 <qresource>
   <file>search_engine/nova2.py</file>
   <file>search_engine/novaprinter.py</file>
+  <file>search_engine/helpers.py</file>
   <file>search_engine/engines/btjunkie.png</file>
   <file>search_engine/engines/btjunkie.py</file>
   <file>search_engine/engines/isohunt.png</file>
@@ -13,4 +14,4 @@
   <file>search_engine/engines/torrentreactor.png</file>
   <file>search_engine/engines/torrentreactor.py</file>
 </qresource>
-</RCC>
\ No newline at end of file
+</RCC>
diff --git a/src/searchEngine.cpp b/src/searchEngine.cpp
index ca0bf7e57..fb6d8c3a4 100644
--- a/src/searchEngine.cpp
+++ b/src/searchEngine.cpp
@@ -303,6 +303,14 @@ void SearchEngine::updateNova() {
     }
     QFile::copy(":/search_engine/novaprinter.py", filePath);
   }
+  QFile(misc::qBittorrentPath()+"search_engine"+QDir::separator()+"helpers.py").setPermissions(perm);
+  filePath = misc::qBittorrentPath()+"search_engine"+QDir::separator()+"helpers.py";
+  if(misc::getPluginVersion(":/search_engine/helpers.py") > misc::getPluginVersion(filePath)) {
+    if(QFile::exists(filePath)){
+      QFile::remove(filePath);
+    }
+    QFile::copy(":/search_engine/helpers.py", filePath);
+  }
   QString destDir = misc::qBittorrentPath()+"search_engine"+QDir::separator()+"engines"+QDir::separator();
   QDir shipped_subDir(":/search_engine/engines/");
   QStringList files = shipped_subDir.entryList();
diff --git a/src/search_engine/engines/btjunkie.py b/src/search_engine/engines/btjunkie.py
index 4febeddcb..57d663509 100644
--- a/src/search_engine/engines/btjunkie.py
+++ b/src/search_engine/engines/btjunkie.py
@@ -1,4 +1,4 @@
-#VERSION: 2.0
+#VERSION: 2.1
 #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
 
 # Redistribution and use in source and binary forms, with or without
@@ -27,8 +27,8 @@
 
 
 from novaprinter import prettyPrinter
+from helpers import retrieve_url
 import sgmllib
-import urllib
 import re
 
 class btjunkie(object):
@@ -72,11 +72,6 @@ class btjunkie(object):
         if not self.current_item.has_key('leech'):
           self.current_item['leech'] = ''
         self.current_item['leech']+= data.strip()
-        
-    def start_font(self, attr):
-      if isinstance(self.th_counter,int):
-        if self.th_counter == 0:
-          self.current_item['name'] += ' '
       
     def start_th(self,attr):
         if isinstance(self.th_counter,int):
@@ -99,7 +94,12 @@ class btjunkie(object):
     while True and i<11:
       results = []
       parser = self.SimpleSGMLParser(results, self.url)
-      dat = urllib.urlopen(self.url+'/search?q=%s&o=52&p=%d'%(what,i)).read()
+      dat = retrieve_url(self.url+'/search?q=%s&o=52&p=%d'%(what,i))
+      # Remove <font> tags from page
+      p = re.compile( '<[/]?font.*?>')
+      dat = p.sub('', dat)
+      #print dat
+      #return
       results_re = re.compile('(?s)class="tab_results">.*')
       for match in results_re.finditer(dat):
         res_tab = match.group(0)
diff --git a/src/search_engine/engines/mininova.py b/src/search_engine/engines/mininova.py
index cbd07ed0f..400b155cd 100644
--- a/src/search_engine/engines/mininova.py
+++ b/src/search_engine/engines/mininova.py
@@ -1,4 +1,4 @@
-#VERSION: 1.13
+#VERSION: 1.2
 #AUTHORS: Fabien Devaux (fab@gnux.info)
 
 # Redistribution and use in source and binary forms, with or without
@@ -26,7 +26,7 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from novaprinter import prettyPrinter
-import urllib
+from helpers import retrieve_url
 from xml.dom import minidom
 import re
 
@@ -64,12 +64,15 @@ class mininova(object):
 				return ''.join([ get_text(n) for n in txt.childNodes])
 		page = 1
 		while True and page<11:
+			file = open('/home/chris/mytest.txt', 'w')
+			file.write(self.url+'/search/%s/seeds/%d'%(what, page))
+			file.close()
 			res = 0
-			dat = urllib.urlopen(self.url+'/search/%s/seeds/%d'%(what, page)).read().decode('utf-8', 'replace')
+			dat = retrieve_url(self.url+'/search/%s/seeds/%d'%(what, page))
 			dat = re.sub("<a href=\"http://www.boardreader.com/index.php.*\"", "<a href=\"plop\"", dat)
 			dat = re.sub("<=", "&lt;=", dat)
 			dat = re.sub("&\s", "&amp; ", dat)
-			x = minidom.parseString(dat.encode('utf-8', 'replace'))
+			x = minidom.parseString(dat)
 			table = x.getElementsByTagName('table').item(0)
 			if not table: return
 			for tr in table.getElementsByTagName('tr'):
diff --git a/src/search_engine/engines/versions.txt b/src/search_engine/engines/versions.txt
index 71cc1f535..3e1cbc65c 100644
--- a/src/search_engine/engines/versions.txt
+++ b/src/search_engine/engines/versions.txt
@@ -1,5 +1,5 @@
 isohunt: 1.1
 torrentreactor: 1.02
-btjunkie: 2.0
-mininova: 1.13
+btjunkie: 2.1
+mininova: 1.2
 piratebay: 1.04
diff --git a/src/search_engine/helpers.py b/src/search_engine/helpers.py
new file mode 100644
index 000000000..48b9e4582
--- /dev/null
+++ b/src/search_engine/helpers.py
@@ -0,0 +1,59 @@
+#VERSION: 1.0
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#    * Neither the name of the author nor the names of its contributors may be
+#      used to endorse or promote products derived from this software without
+#      specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import re, htmlentitydefs
+import urllib2
+
+def htmlentitydecode(s):
+    # First convert alpha entities (such as &eacute;)
+    # (Inspired from http://mail.python.org/pipermail/python-list/2007-June/443813.html)
+    def entity2char(m):
+        entity = m.group(1)
+        if entity in htmlentitydefs.name2codepoint:
+            return unichr(htmlentitydefs.name2codepoint[entity])
+        return u" "  # Unknown entity: We replace with a space.
+    t = re.sub(u'&(%s);' % u'|'.join(htmlentitydefs.name2codepoint), entity2char, s)
+  
+    # Then convert numerical entities (such as &#233;)
+    t = re.sub(u'&#(\d+);', lambda x: unichr(int(x.group(1))), t)
+   
+    # Then convert hexa entities (such as &#x00E9;)
+    return re.sub(u'&#x(\w+);', lambda x: unichr(int(x.group(1),16)), t)
+    
+def retrieve_url(url):
+    """ Return the content of the url page as a string """
+    req = urllib2.Request(url)
+    response = urllib2.urlopen(req)
+    dat = response.read()
+    info = response.info()
+    charset = 'utf-8'
+    try:
+        ignore, charset = info['Content-Type'].split('charset=')
+    except:
+        pass
+    dat = dat.decode(charset)
+    dat = htmlentitydecode(dat)
+    return dat.encode('utf-8', 'replace')
diff --git a/src/search_engine/nova2.py b/src/search_engine/nova2.py
index 96b419b62..1ef4dfa38 100755
--- a/src/search_engine/nova2.py
+++ b/src/search_engine/nova2.py
@@ -98,18 +98,18 @@ if __name__ == '__main__':
 		engines_list = supported_engines
 
 	what = '+'.join(sys.argv[2:])
-
 	threads = []
 	for engine in engines_list:
-		try:
+		if 1:
+		#try:
 			if THREADED:
 				exec "l = EngineLauncher(%s(), what)" % engine
 				threads.append(l)
 				l.start()
 			else:
 				engine().search(what)
-		except:
-			pass
+		#except:
+		#	pass
 	if THREADED:
 		for t in threads:
 			t.join()
diff --git a/src/search_engine/novaprinter.py b/src/search_engine/novaprinter.py
index 2234a56f5..267d69d77 100644
--- a/src/search_engine/novaprinter.py
+++ b/src/search_engine/novaprinter.py
@@ -1,4 +1,4 @@
-#VERSION: 1.11
+#VERSION: 1.2
 
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -26,7 +26,9 @@
 
 def prettyPrinter(dictionnary):
 	dictionnary['size'] = anySizeToBytes(dictionnary['size'])
-	print "%(link)s|%(name)s|%(size)s|%(seeds)s|%(leech)s|%(engine_url)s" % dictionnary
+	if isinstance(dictionnary['name'], unicode):
+		dictionnary['name'] = dictionnary['name'].encode('utf-8')
+	print  dictionnary['link'],'|',dictionnary['name'],'|',dictionnary['size'],'|',dictionnary['seeds'],'|',dictionnary['leech'],'|',dictionnary['engine_url']
 
 def anySizeToBytes(size_string):
 	"""
@@ -42,7 +44,7 @@ def anySizeToBytes(size_string):
 			size = size[:-len(unit)]
 		except:
 			return -1
-        if len(size) == 0:
+	if len(size) == 0:
 		return -1
 	size = float(size)
 	short_unit = unit.upper()[0]