Browse Source

- Fixed unicode in mininova and btjunkie search engines

adaptive-webui-19844
Christophe Dumez 16 years ago
parent
commit
a2e9210665
  1. 3
      src/search.qrc
  2. 8
      src/searchEngine.cpp
  3. 16
      src/search_engine/engines/btjunkie.py
  4. 11
      src/search_engine/engines/mininova.py
  5. 4
      src/search_engine/engines/versions.txt
  6. 59
      src/search_engine/helpers.py
  7. 8
      src/search_engine/nova2.py
  8. 8
      src/search_engine/novaprinter.py

3
src/search.qrc

@ -2,6 +2,7 @@ @@ -2,6 +2,7 @@
<qresource>
<file>search_engine/nova2.py</file>
<file>search_engine/novaprinter.py</file>
<file>search_engine/helpers.py</file>
<file>search_engine/engines/btjunkie.png</file>
<file>search_engine/engines/btjunkie.py</file>
<file>search_engine/engines/isohunt.png</file>
@ -13,4 +14,4 @@ @@ -13,4 +14,4 @@
<file>search_engine/engines/torrentreactor.png</file>
<file>search_engine/engines/torrentreactor.py</file>
</qresource>
</RCC>
</RCC>

8
src/searchEngine.cpp

@ -303,6 +303,14 @@ void SearchEngine::updateNova() { @@ -303,6 +303,14 @@ void SearchEngine::updateNova() {
}
QFile::copy(":/search_engine/novaprinter.py", filePath);
}
QFile(misc::qBittorrentPath()+"search_engine"+QDir::separator()+"helpers.py").setPermissions(perm);
filePath = misc::qBittorrentPath()+"search_engine"+QDir::separator()+"helpers.py";
if(misc::getPluginVersion(":/search_engine/helpers.py") > misc::getPluginVersion(filePath)) {
if(QFile::exists(filePath)){
QFile::remove(filePath);
}
QFile::copy(":/search_engine/helpers.py", filePath);
}
QString destDir = misc::qBittorrentPath()+"search_engine"+QDir::separator()+"engines"+QDir::separator();
QDir shipped_subDir(":/search_engine/engines/");
QStringList files = shipped_subDir.entryList();

16
src/search_engine/engines/btjunkie.py

@ -1,4 +1,4 @@ @@ -1,4 +1,4 @@
#VERSION: 2.0
#VERSION: 2.1
#AUTHORS: Christophe Dumez (chris@qbittorrent.org)
# Redistribution and use in source and binary forms, with or without
@ -27,8 +27,8 @@ @@ -27,8 +27,8 @@
from novaprinter import prettyPrinter
from helpers import retrieve_url
import sgmllib
import urllib
import re
class btjunkie(object):
@ -72,11 +72,6 @@ class btjunkie(object): @@ -72,11 +72,6 @@ class btjunkie(object):
if not self.current_item.has_key('leech'):
self.current_item['leech'] = ''
self.current_item['leech']+= data.strip()
def start_font(self, attr):
if isinstance(self.th_counter,int):
if self.th_counter == 0:
self.current_item['name'] += ' '
def start_th(self,attr):
if isinstance(self.th_counter,int):
@ -99,7 +94,12 @@ class btjunkie(object): @@ -99,7 +94,12 @@ class btjunkie(object):
while True and i<11:
results = []
parser = self.SimpleSGMLParser(results, self.url)
dat = urllib.urlopen(self.url+'/search?q=%s&o=52&p=%d'%(what,i)).read()
dat = retrieve_url(self.url+'/search?q=%s&o=52&p=%d'%(what,i))
# Remove <font> tags from page
p = re.compile( '<[/]?font.*?>')
dat = p.sub('', dat)
#print dat
#return
results_re = re.compile('(?s)class="tab_results">.*')
for match in results_re.finditer(dat):
res_tab = match.group(0)

11
src/search_engine/engines/mininova.py

@ -1,4 +1,4 @@ @@ -1,4 +1,4 @@
#VERSION: 1.13
#VERSION: 1.2
#AUTHORS: Fabien Devaux (fab@gnux.info)
# Redistribution and use in source and binary forms, with or without
@ -26,7 +26,7 @@ @@ -26,7 +26,7 @@
# POSSIBILITY OF SUCH DAMAGE.
from novaprinter import prettyPrinter
import urllib
from helpers import retrieve_url
from xml.dom import minidom
import re
@ -64,12 +64,15 @@ class mininova(object): @@ -64,12 +64,15 @@ class mininova(object):
return ''.join([ get_text(n) for n in txt.childNodes])
page = 1
while True and page<11:
file = open('/home/chris/mytest.txt', 'w')
file.write(self.url+'/search/%s/seeds/%d'%(what, page))
file.close()
res = 0
dat = urllib.urlopen(self.url+'/search/%s/seeds/%d'%(what, page)).read().decode('utf-8', 'replace')
dat = retrieve_url(self.url+'/search/%s/seeds/%d'%(what, page))
dat = re.sub("<a href=\"http://www.boardreader.com/index.php.*\"", "<a href=\"plop\"", dat)
dat = re.sub("<=", "&lt;=", dat)
dat = re.sub("&\s", "&amp; ", dat)
x = minidom.parseString(dat.encode('utf-8', 'replace'))
x = minidom.parseString(dat)
table = x.getElementsByTagName('table').item(0)
if not table: return
for tr in table.getElementsByTagName('tr'):

4
src/search_engine/engines/versions.txt

@ -1,5 +1,5 @@ @@ -1,5 +1,5 @@
isohunt: 1.1
torrentreactor: 1.02
btjunkie: 2.0
mininova: 1.13
btjunkie: 2.1
mininova: 1.2
piratebay: 1.04

59
src/search_engine/helpers.py

@ -0,0 +1,59 @@ @@ -0,0 +1,59 @@
#VERSION: 1.0
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the author nor the names of its contributors may be
# used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import re, htmlentitydefs
import urllib2
def htmlentitydecode(s):
# First convert alpha entities (such as &eacute;)
# (Inspired from http://mail.python.org/pipermail/python-list/2007-June/443813.html)
def entity2char(m):
entity = m.group(1)
if entity in htmlentitydefs.name2codepoint:
return unichr(htmlentitydefs.name2codepoint[entity])
return u" " # Unknown entity: We replace with a space.
t = re.sub(u'&(%s);' % u'|'.join(htmlentitydefs.name2codepoint), entity2char, s)
# Then convert numerical entities (such as &#233;)
t = re.sub(u'&#(\d+);', lambda x: unichr(int(x.group(1))), t)
# Then convert hexa entities (such as &#x00E9;)
return re.sub(u'&#x(\w+);', lambda x: unichr(int(x.group(1),16)), t)
def retrieve_url(url):
""" Return the content of the url page as a string """
req = urllib2.Request(url)
response = urllib2.urlopen(req)
dat = response.read()
info = response.info()
charset = 'utf-8'
try:
ignore, charset = info['Content-Type'].split('charset=')
except:
pass
dat = dat.decode(charset)
dat = htmlentitydecode(dat)
return dat.encode('utf-8', 'replace')

8
src/search_engine/nova2.py

@ -98,18 +98,18 @@ if __name__ == '__main__': @@ -98,18 +98,18 @@ if __name__ == '__main__':
engines_list = supported_engines
what = '+'.join(sys.argv[2:])
threads = []
for engine in engines_list:
try:
if 1:
#try:
if THREADED:
exec "l = EngineLauncher(%s(), what)" % engine
threads.append(l)
l.start()
else:
engine().search(what)
except:
pass
#except:
# pass
if THREADED:
for t in threads:
t.join()

8
src/search_engine/novaprinter.py

@ -1,4 +1,4 @@ @@ -1,4 +1,4 @@
#VERSION: 1.11
#VERSION: 1.2
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -26,7 +26,9 @@ @@ -26,7 +26,9 @@
def prettyPrinter(dictionnary):
dictionnary['size'] = anySizeToBytes(dictionnary['size'])
print "%(link)s|%(name)s|%(size)s|%(seeds)s|%(leech)s|%(engine_url)s" % dictionnary
if isinstance(dictionnary['name'], unicode):
dictionnary['name'] = dictionnary['name'].encode('utf-8')
print dictionnary['link'],'|',dictionnary['name'],'|',dictionnary['size'],'|',dictionnary['seeds'],'|',dictionnary['leech'],'|',dictionnary['engine_url']
def anySizeToBytes(size_string):
"""
@ -42,7 +44,7 @@ def anySizeToBytes(size_string): @@ -42,7 +44,7 @@ def anySizeToBytes(size_string):
size = size[:-len(unit)]
except:
return -1
if len(size) == 0:
if len(size) == 0:
return -1
size = float(size)
short_unit = unit.upper()[0]

Loading…
Cancel
Save