Browse Source

- Rewrote btjunkie parser (it is now a lot faster and less cpu extensive)

adaptive-webui-19844
Christophe Dumez 16 years ago
parent
commit
288300d264
  1. 111
      src/search_engine/engines/btjunkie.py
  2. 2
      src/search_engine/engines/versions.txt

111
src/search_engine/engines/btjunkie.py

@ -1,5 +1,5 @@
#VERSION: 1.13 #VERSION: 2.0
#AUTHORS: Fabien Devaux (fab@gnux.info) #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
# Redistribution and use in source and binary forms, with or without # Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met: # modification, are permitted provided that the following conditions are met:
@ -25,37 +25,88 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE. # POSSIBILITY OF SUCH DAMAGE.
from novaprinter import prettyPrinter from novaprinter import prettyPrinter
import sgmllib
import urllib import urllib
import re import re
class btjunkie(object): class btjunkie(object):
url = 'http://btjunkie.org' url = 'http://btjunkie.org'
name = 'btjunkie' name = 'btjunkie'
def __init__(self):
self.results = []
self.parser = self.SimpleSGMLParser(self.results, self.url)
class SimpleSGMLParser(sgmllib.SGMLParser):
def __init__(self, results, url, *args):
sgmllib.SGMLParser.__init__(self)
self.url = url
self.th_counter = None
self.current_item = None
self.results = results
def start_a(self, attr):
params = dict(attr)
#print params
if params.has_key('href') and params['href'].startswith("http://dl.btjunkie.org/torrent"):
self.current_item = {}
self.th_counter = 0
self.current_item['link']=params['href'].strip()
def handle_data(self, data):
if self.th_counter == 0:
if not self.current_item.has_key('name'):
self.current_item['name'] = ''
self.current_item['name']+= data.strip()
elif self.th_counter == 3:
if not self.current_item.has_key('size'):
self.current_item['size'] = ''
self.current_item['size']+= data.strip()
elif self.th_counter == 5:
if not self.current_item.has_key('seeds'):
self.current_item['seeds'] = ''
self.current_item['seeds']+= data.strip()
elif self.th_counter == 6:
if not self.current_item.has_key('leech'):
self.current_item['leech'] = ''
self.current_item['leech']+= data.strip()
def start_font(self, attr):
if isinstance(self.th_counter,int):
if self.th_counter == 0:
self.current_item['name'] += ' '
def start_th(self,attr):
if isinstance(self.th_counter,int):
self.th_counter += 1
if self.th_counter > 6:
self.th_counter = None
# Display item
if self.current_item:
self.current_item['engine_url'] = self.url
if not self.current_item['seeds'].isdigit():
self.current_item['seeds'] = 0
if not self.current_item['leech'].isdigit():
self.current_item['leech'] = 0
prettyPrinter(self.current_item)
self.results.append('a')
def search(self, what): def search(self, what):
i = 1 ret = []
while True and i<11: i = 1
res = 0 while True and i<11:
dat = urllib.urlopen(self.url+'/search?q=%s&o=52&p=%d'%(what,i)).read().decode('utf8', 'replace') results = []
# I know it's not very readable, but the SGML parser feels in pain parser = self.SimpleSGMLParser(results, self.url)
section_re = re.compile('(?s)href="http://dl.btjunkie.org/torrent/.*?</tr><tr') dat = urllib.urlopen(self.url+'/search?q=%s&o=52&p=%d'%(what,i)).read()
torrent_re = re.compile('(?s)href="(?P<link>.*?[^"]+).*?' results_re = re.compile('(?s)class="tab_results">.*')
'class="BlckUnd">(?P<name>.*?)</a>.*?' for match in results_re.finditer(dat):
'>(?P<size>\d+MB)</font>.*?' res_tab = match.group(0)
'>.*</font>.*' parser.feed(res_tab)
'>(?P<seeds>\d+)</font>.*?' parser.close()
'>(?P<leech>\d+)</font>') break
for match in section_re.finditer(dat): if len(results) <= 0:
txt = match.group(0) break
m = torrent_re.search(txt) i += 1
if m:
torrent_infos = m.groupdict()
torrent_infos['name'] = re.sub('<.*?>', '', torrent_infos['name'])
torrent_infos['engine_url'] = self.url
#torrent_infos['link'] = self.url+torrent_infos['link']
prettyPrinter(torrent_infos)
res = res + 1
if res == 0:
break
i = i + 1

2
src/search_engine/engines/versions.txt

@ -1,5 +1,5 @@
isohunt: 1.1 isohunt: 1.1
torrentreactor: 1.02 torrentreactor: 1.02
btjunkie: 1.13 btjunkie: 2.0
mininova: 1.13 mininova: 1.13
piratebay: 1.04 piratebay: 1.04

Loading…
Cancel
Save