d4708
/
qBittorrent
mirror of https://github.com/d47081/qBittorrent.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

[search engine] engines update

adaptive-webui-19844
DoumanAsh 10 years ago
parent
bef8106d0f
commit
d6d0f422f5
10 changed files with 660 additions and 507 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 193
      src/searchengine/nova/engines/extratorrent.py
  2. 6
      src/searchengine/nova/engines/legittorrents.py
  3. 182
      src/searchengine/nova/engines/mininova.py
  4. 34
      src/searchengine/nova/engines/torrentreactor.py
  5. 9
      src/searchengine/nova/engines/versions.txt
  6. 191
      src/searchengine/nova3/engines/extratorrent.py
  7. 6
      src/searchengine/nova3/engines/legittorrents.py
  8. 180
      src/searchengine/nova3/engines/mininova.py
  9. 25
      src/searchengine/nova3/engines/torrentreactor.py
  10. 9
      src/searchengine/nova3/engines/versions.txt

193
src/searchengine/nova/engines/extratorrent.py
Unescape View File

@ -1,4 +1,4 @@
#VERSION: 1.2 #VERSION: 2.0
#AUTHORS: Christophe Dumez (chris@qbittorrent.org) #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
# Redistribution and use in source and binary forms, with or without # Redistribution and use in source and binary forms, with or without
@ -25,92 +25,135 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE. # POSSIBILITY OF SUCH DAMAGE.
from HTMLParser import HTMLParser
from httplib import HTTPConnection as http
#qBt
from novaprinter import prettyPrinter from novaprinter import prettyPrinter
from helpers import retrieve_url, download_file from helpers import download_file
import sgmllib
import re
class extratorrent(object): class extratorrent(object):
""" Search engine class """
url = 'http://extratorrent.cc' url = 'http://extratorrent.cc'
name = 'extratorrent' name = 'ExtraTorrent'
supported_categories = {'all': '', 'movies': '4', 'tv': '8', 'music': '5', 'games': '3', 'anime': '1', 'software': '7', 'books': '2', 'pictures': '6'} supported_categories = {'all' : '0',
'movies' : '4',
def __init__(self): 'tv' : '8',
self.results = [] 'music' : '5',
self.parser = self.SimpleSGMLParser(self.results, self.url) 'games' : '3',
'anime' : '1',
'software' : '7',
'books' : '2',
'pictures' : '6'}
def download_torrent(self, info): def download_torrent(self, info):
print download_file(info) """ Downloader """
print(download_file(info))
class SimpleSGMLParser(sgmllib.SGMLParser): class MyHtmlParseWithBlackJack(HTMLParser):
def __init__(self, results, url, *args): """ Parser class """
sgmllib.SGMLParser.__init__(self) def __init__(self, list_searches, url):
HTMLParser.__init__(self)
self.url = url self.url = url
self.td_counter = None self.list_searches = list_searches
self.current_item = None self.current_item = None
self.start_name = False self.cur_item_name = None
self.results = results self.pending_size = False
self.next_queries = True
def start_a(self, attr): self.pending_next_queries = False
params = dict(attr)
#print params def handle_starttag(self, tag, attrs):
if params.has_key('href') and params['href'].startswith("/torrent_download/"): if self.current_item:
self.current_item = {} if tag == "a":
self.td_counter = 0 params = dict(attrs)
self.start_name = False link = params['href']
torrent_id = '/'.join(params['href'].split('/')[2:])
self.current_item['link']=self.url+'/download/'+torrent_id if not link.startswith("/torrent"):
elif params.has_key('href') and params['href'].startswith("/torrent/") and params['href'].endswith(".html"): return
self.current_item['desc_link'] = self.url + params['href'].strip()
self.start_name = True if link[8] == "/":
#description
self.current_item["desc_link"] = "".join((self.url, link))
#remove view at the beginning
self.current_item["name"] = params["title"][5:]
self.pending_size = True
elif link[8] == "_":
#download link
link = link.replace("torrent_", "", 1)
self.current_item["link"] = "".join((self.url, link))
elif tag == "td":
if self.pending_size:
self.cur_item_name = "size"
self.current_item["size"] = ""
self.pending_size = False
for attr in attrs:
if attr[0] == "class":
if attr[1][0] == "s":
self.cur_item_name = "seeds"
self.current_item["seeds"] = ""
elif attr[1][0] == "l":
self.cur_item_name = "leech"
self.current_item["leech"] = ""
break
elif tag == "tr":
for attr in attrs:
if attr[0] == "class" and attr[1].startswith("tl"):
self.current_item = dict()
self.current_item["engine_url"] = self.url
break
elif self.pending_next_queries:
if tag == "a":
params = dict(attrs)
self.list_searches.append(params['href'])
if params["title"] == "10":
self.pending_next_queries = False
else:
self.pending_next_queries = False
elif self.next_queries:
if tag == "b" and ("class", "pager_no_link") in attrs:
self.next_queries = False
self.pending_next_queries = True
def handle_data(self, data): def handle_data(self, data):
if self.td_counter == 2: if self.cur_item_name:
if not self.current_item.has_key('name') and self.start_name: temp = self.current_item[self.cur_item_name]
self.current_item['name'] = data.strip() self.current_item[self.cur_item_name] = " ".join((temp, data))
elif self.td_counter == 3: #Due to utf-8 we need to handle data two times if there is space
if not self.current_item.has_key('size'): if not self.cur_item_name == "size":
self.current_item['size'] = '' self.cur_item_name = None
self.current_item['size']+= data.replace(" ", " ").strip()
elif self.td_counter == 4: def handle_endtag(self, tag):
if not self.current_item.has_key('seeds'):
self.current_item['seeds'] = ''
self.current_item['seeds']+= data.strip()
elif self.td_counter == 5:
if not self.current_item.has_key('leech'):
self.current_item['leech'] = ''
self.current_item['leech']+= data.strip()
def start_td(self,attr):
if isinstance(self.td_counter,int):
self.td_counter += 1
if self.td_counter > 5:
self.td_counter = None
# Display item
if self.current_item: if self.current_item:
self.current_item['engine_url'] = self.url if tag == "tr":
if not self.current_item['seeds'].isdigit():
self.current_item['seeds'] = 0
if not self.current_item['leech'].isdigit():
self.current_item['leech'] = 0
prettyPrinter(self.current_item) prettyPrinter(self.current_item)
self.results.append('a') self.current_item = None
def search(self, what, cat='all'): def search(self, what, cat="all"):
ret = [] """ Performs search """
i = 1 connection = http("extratorrent.cc")
while True and i<11:
results = [] query = "".join(("/search/?new=1&search=", what, "&s_cat=", self.supported_categories[cat]))
parser = self.SimpleSGMLParser(results, self.url)
dat = retrieve_url(self.url+'/advanced_search/?with=%s&s_cat=%s&page=%d'%(what, self.supported_categories[cat], i)) connection.request("GET", query)
results_re = re.compile('(?s)<table class="tl"><thead>.*') response = connection.getresponse()
for match in results_re.finditer(dat): if response.status != 200:
res_tab = match.group(0) return
parser.feed(res_tab)
list_searches = []
parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
parser.feed(response.read().decode('utf-8'))
parser.close()
for search_query in list_searches:
connection.request("GET", search_query)
response = connection.getresponse()
parser.feed(response.read().decode('utf-8'))
parser.close() parser.close()
break
if len(results) <= 0:
break
i += 1
connection.close()
return

6
src/searchengine/nova/engines/legittorrents.py
Unescape View File

@ -1,4 +1,4 @@
#VERSION: 1.02 #VERSION: 1.03
#AUTHORS: Christophe Dumez (chris@qbittorrent.org) #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
# Redistribution and use in source and binary forms, with or without # Redistribution and use in source and binary forms, with or without
@ -36,10 +36,6 @@ class legittorrents(object):
name = 'legittorrents' name = 'legittorrents'
supported_categories = {'all': '', 'movies': '1', 'tv': '13', 'music': '2', 'games': '3', 'anime': '5', 'books': '6'} supported_categories = {'all': '', 'movies': '1', 'tv': '13', 'music': '2', 'games': '3', 'anime': '5', 'books': '6'}
def __init__(self):
self.results = []
self.parser = self.SimpleSGMLParser(self.results, self.url)
def download_torrent(self, info): def download_torrent(self, info):
print download_file(info) print download_file(info)

182
src/searchengine/nova/engines/mininova.py
Unescape View File

@ -1,4 +1,4 @@
#VERSION: 1.51 #VERSION: 2.00
#AUTHORS: Christophe Dumez (chris@qbittorrent.org) #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
#CONTRIBUTORS: Diego de las Heras (diegodelasheras@gmail.com) #CONTRIBUTORS: Diego de las Heras (diegodelasheras@gmail.com)
@ -26,90 +26,124 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE. # POSSIBILITY OF SUCH DAMAGE.
from HTMLParser import HTMLParser
from httplib import HTTPConnection as http
from novaprinter import prettyPrinter from novaprinter import prettyPrinter
from helpers import retrieve_url, download_file from helpers import download_file
import sgmllib
import re
class mininova(object): class mininova(object):
# Mandatory properties """ Search engine class """
url = 'http://www.mininova.org' url = 'http://www.mininova.org'
name = 'Mininova' name = 'Mininova'
supported_categories = {'all': '0', 'movies': '4', 'tv': '8', 'music': '5', 'games': '3', 'anime': '1', 'software': '7', 'pictures': '6', 'books': '2'} supported_categories = {'all' : '0',
'movies' : '4',
def __init__(self): 'tv' : '8',
self.results = [] 'music' : '5',
self.parser = self.SimpleSGMLParser(self.results, self.url) 'games' : '3',
'anime' : '1',
'software' : '7',
'pictures' : '6',
'books' : '2'}
def download_torrent(self, info): def download_torrent(self, info):
print download_file(info) print(download_file(info))
class SimpleSGMLParser(sgmllib.SGMLParser): class MyHtmlParseWithBlackJack(HTMLParser):
def __init__(self, results, url, *args): """ Parser class """
sgmllib.SGMLParser.__init__(self) def __init__(self, list_searches, url):
HTMLParser.__init__(self)
self.list_searches = list_searches
self.url = url self.url = url
self.td_counter = None self.table_results = False
self.current_item = None self.current_item = None
self.results = results self.cur_item_name = None
self.next_queries = True
def start_a(self, attr):
params = dict(attr)
#print params
if params.has_key('href'):
if params['href'].startswith("/get/"):
self.current_item = {}
self.td_counter = 0
self.current_item['link']=self.url+params['href'].strip()
elif params['href'].startswith("/tor/") and self.current_item is not None:
self.current_item['desc_link']=self.url+params['href'].strip()
def handle_data(self, data): def handle_starttag_tr(self, _):
if self.td_counter == 0: """ Handler of tr start tag """
if not self.current_item.has_key('name'): self.current_item = dict()
self.current_item['name'] = ''
self.current_item['name']+= data def handle_starttag_a(self, attrs):
elif self.td_counter == 1: """ Handler of a start tag """
if not self.current_item.has_key('size'): params = dict(attrs)
self.current_item['size'] = '' link = params["href"]
self.current_item['size']+= data.strip()
elif self.td_counter == 2: if link.startswith("/get/"):
if not self.current_item.has_key('seeds'): #download link
self.current_item['seeds'] = '' self.current_item["link"] = "".join((self.url, link))
self.current_item['seeds']+= data.strip() elif link.startswith("/tor/"):
elif self.td_counter == 3: #description
if not self.current_item.has_key('leech'): self.current_item["desc_link"] = "".join((self.url, link))
self.current_item['leech'] = '' self.cur_item_name = "name"
self.current_item['leech']+= data.strip() self.current_item["name"] = ""
elif self.next_queries and link.startswith("/search"):
def start_td(self,attr): if params["title"].startswith("Page"):
if isinstance(self.td_counter,int): self.list_searches.append(link)
self.td_counter += 1
if self.td_counter > 4: def handle_starttag_td(self, attrs):
self.td_counter = None """ Handler of td start tag """
# Display item if ("align", "right") in attrs:
if self.current_item: if not "size" in self.current_item.keys():
self.current_item['engine_url'] = self.url self.cur_item_name = "size"
if not self.current_item['seeds'].isdigit(): self.current_item["size"] = ""
self.current_item['seeds'] = 0
if not self.current_item['leech'].isdigit(): def handle_starttag_span(self, attrs):
self.current_item['leech'] = 0 """ Handler of span start tag """
if ("class", "g") in attrs:
self.cur_item_name = "seeds"
self.current_item["seeds"] = ""
elif ("class", "b") in attrs:
self.cur_item_name = "leech"
self.current_item["leech"] = ""
def handle_starttag(self, tag, attrs):
""" Parser's start tag handler """
if self.table_results:
dispatcher = getattr(self, "_".join(("handle_starttag", tag)), None)
if dispatcher:
dispatcher(attrs)
elif tag == "table":
self.table_results = ("class", "maintable") in attrs
def handle_endtag(self, tag):
""" Parser's end tag handler """
if tag == "tr" and self.current_item:
self.current_item["engine_url"] = self.url
prettyPrinter(self.current_item) prettyPrinter(self.current_item)
self.results.append('a') self.current_item = None
elif self.cur_item_name:
def search(self, what, cat='all'): if tag == "a" or tag == "span":
ret = [] self.cur_item_name = None
i = 1
while True and i<11: def handle_data(self, data):
results = [] """ Parser's data handler """
parser = self.SimpleSGMLParser(results, self.url) if self.cur_item_name:
dat = retrieve_url(self.url+'/search/%s/%s/seeds/%d'%(what, self.supported_categories[cat], i)) temp = self.current_item[self.cur_item_name]
results_re = re.compile('(?s)<h1>Search results for.*') self.current_item[self.cur_item_name] = " ".join((temp, data))
for match in results_re.finditer(dat):
res_tab = match.group(0) def search(self, what, cat="all"):
parser.feed(res_tab) """ Performs search """
connection = http("www.mininova.org")
query = "/".join(("/search", what, self.supported_categories[cat], "seeds"))
connection.request("GET", query)
response = connection.getresponse()
if response.status != 200:
return
list_searches = []
parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
parser.feed(response.read().decode('utf-8'))
parser.close()
parser.next_queries = False
for search_query in list_searches:
connection.request("GET", search_query)
response = connection.getresponse()
parser.feed(response.read().decode('utf-8'))
parser.close() parser.close()
break
if len(results) <= 0:
break
i += 1
connection.close()
return

34
src/searchengine/nova/engines/torrentreactor.py
Unescape View File

@ -1,4 +1,4 @@
#VERSION: 1.33 #VERSION: 1.35
#AUTHORS: Gekko Dam Beer (gekko04@users.sourceforge.net) #AUTHORS: Gekko Dam Beer (gekko04@users.sourceforge.net)
#CONTRIBUTORS: Christophe Dumez (chris@qbittorrent.org) #CONTRIBUTORS: Christophe Dumez (chris@qbittorrent.org)
# Bruno Barbieri (brunorex@gmail.com) # Bruno Barbieri (brunorex@gmail.com)
@ -28,10 +28,10 @@
# POSSIBILITY OF SUCH DAMAGE. # POSSIBILITY OF SUCH DAMAGE.
from novaprinter import prettyPrinter from novaprinter import prettyPrinter
from helpers import retrieve_url, download_file from helpers import download_file
from urllib2 import HTTPError
from HTMLParser import HTMLParser
import urllib import urllib
from HTMLParser import HTMLParser
from httplib import HTTPConnection as http
import re import re
class torrentreactor(object): class torrentreactor(object):
@ -40,7 +40,7 @@ class torrentreactor(object):
supported_categories = {'all': '', 'movies': '5', 'tv': '8', 'music': '6', 'games': '3', 'anime': '1', 'software': '2'} supported_categories = {'all': '', 'movies': '5', 'tv': '8', 'music': '6', 'games': '3', 'anime': '1', 'software': '2'}
def download_torrent(self, info): def download_torrent(self, info):
print download_file(info) print(download_file(info))
class SimpleHTMLParser(HTMLParser): class SimpleHTMLParser(HTMLParser):
def __init__(self, results, url, *args): def __init__(self, results, url, *args):
@ -68,15 +68,15 @@ class torrentreactor(object):
def handle_data(self, data): def handle_data(self, data):
if self.td_counter == 1: if self.td_counter == 1:
if not self.current_item.has_key('size'): if 'size' not in self.current_item:
self.current_item['size'] = '' self.current_item['size'] = ''
self.current_item['size']+= data.strip() self.current_item['size']+= data.strip()
elif self.td_counter == 2: elif self.td_counter == 2:
if not self.current_item.has_key('seeds'): if 'seeds' not in self.current_item:
self.current_item['seeds'] = '' self.current_item['seeds'] = ''
self.current_item['seeds']+= data.strip() self.current_item['seeds']+= data.strip()
elif self.td_counter == 3: elif self.td_counter == 3:
if not self.current_item.has_key('leech'): if 'leech' not in self.current_item:
self.current_item['leech'] = '' self.current_item['leech'] = ''
self.current_item['leech']+= data.strip() self.current_item['leech']+= data.strip()
@ -96,24 +96,26 @@ class torrentreactor(object):
self.has_results = True self.has_results = True
self.results.append('a') self.results.append('a')
def __init__(self):
self.results = []
self.parser = self.SimpleHTMLParser(self.results, self.url)
def search(self, what, cat='all'): def search(self, what, cat='all'):
i = 0 i = 0
dat = '' dat = ''
connection = http("www.torrentreactor.net")
while True and i<11: while True and i<11:
results = [] results = []
parser = self.SimpleHTMLParser(results, self.url) parser = self.SimpleHTMLParser(results, self.url)
query = '/torrents-search/%s/%d?sort=seeders.desc&type=all&period=none&categories=%s'%(what, (i*35), self.supported_categories[cat])
try: connection.request("GET", query)
dat = retrieve_url(self.url+'/torrent-search/%s/%s?sort=seeders.desc&type=all&period=none&categories=%s'%(what, (i*35), self.supported_categories[cat])) response = connection.getresponse()
except HTTPError: if response.status != 200:
break break
dat = response.read().decode('utf-8')
parser.feed(dat) parser.feed(dat)
parser.close() parser.close()
if len(results) <= 0: if len(results) <= 0:
break break
i += 1 i += 1
connection.close()

9
src/searchengine/nova/engines/versions.txt
Unescape View File

@ -1,8 +1,9 @@
torrentreactor: 1.33
mininova: 1.51
piratebay: 2.11
extratorrent: 1.2 extratorrent: 1.2
torrentreactor: 1.35
mininova: 2.00
piratebay: 2.11
extratorrent: 2.0
kickasstorrents: 1.26 kickasstorrents: 1.26
btdigg: 1.24 btdigg: 1.24
legittorrents: 1.02
torrentz: 2.13 torrentz: 2.13
legittorrents: 1.03

191
src/searchengine/nova3/engines/extratorrent.py
Unescape View File

@ -1,4 +1,4 @@
#VERSION: 1.2 #VERSION: 2.0
#AUTHORS: Christophe Dumez (chris@qbittorrent.org) #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
# Redistribution and use in source and binary forms, with or without # Redistribution and use in source and binary forms, with or without
@ -25,92 +25,135 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE. # POSSIBILITY OF SUCH DAMAGE.
from html.parser import HTMLParser
from http.client import HTTPConnection as http
#qBt
from novaprinter import prettyPrinter from novaprinter import prettyPrinter
from helpers import retrieve_url, download_file from helpers import download_file
import sgmllib3
import re
class extratorrent(object): class extratorrent(object):
""" Search engine class """
url = 'http://extratorrent.cc' url = 'http://extratorrent.cc'
name = 'extratorrent' name = 'ExtraTorrent'
supported_categories = {'all': '', 'movies': '4', 'tv': '8', 'music': '5', 'games': '3', 'anime': '1', 'software': '7', 'books': '2', 'pictures': '6'} supported_categories = {'all' : '0',
'movies' : '4',
def __init__(self): 'tv' : '8',
self.results = [] 'music' : '5',
self.parser = self.SimpleSGMLParser(self.results, self.url) 'games' : '3',
'anime' : '1',
'software' : '7',
'books' : '2',
'pictures' : '6'}
def download_torrent(self, info): def download_torrent(self, info):
""" Downloader """
print(download_file(info)) print(download_file(info))
class SimpleSGMLParser(sgmllib3.SGMLParser): class MyHtmlParseWithBlackJack(HTMLParser):
def __init__(self, results, url, *args): """ Parser class """
sgmllib3.SGMLParser.__init__(self) def __init__(self, list_searches, url):
HTMLParser.__init__(self)
self.url = url self.url = url
self.td_counter = None self.list_searches = list_searches
self.current_item = None self.current_item = None
self.start_name = False self.cur_item_name = None
self.results = results self.pending_size = False
self.next_queries = True
def start_a(self, attr): self.pending_next_queries = False
params = dict(attr)
#print params def handle_starttag(self, tag, attrs):
if 'href' in params and params['href'].startswith("/torrent_download/"): if self.current_item:
self.current_item = {} if tag == "a":
self.td_counter = 0 params = dict(attrs)
self.start_name = False link = params['href']
torrent_id = '/'.join(params['href'].split('/')[2:])
self.current_item['link']=self.url+'/download/'+torrent_id if not link.startswith("/torrent"):
elif 'href' in params and params['href'].startswith("/torrent/") and params['href'].endswith(".html"): return
self.current_item['desc_link'] = self.url + params['href'].strip()
self.start_name = True if link[8] == "/":
#description
self.current_item["desc_link"] = "".join((self.url, link))
#remove view at the beginning
self.current_item["name"] = params["title"][5:]
self.pending_size = True
elif link[8] == "_":
#download link
link = link.replace("torrent_", "", 1)
self.current_item["link"] = "".join((self.url, link))
elif tag == "td":
if self.pending_size:
self.cur_item_name = "size"
self.current_item["size"] = ""
self.pending_size = False
for attr in attrs:
if attr[0] == "class":
if attr[1][0] == "s":
self.cur_item_name = "seeds"
self.current_item["seeds"] = ""
elif attr[1][0] == "l":
self.cur_item_name = "leech"
self.current_item["leech"] = ""
break
elif tag == "tr":
for attr in attrs:
if attr[0] == "class" and attr[1].startswith("tl"):
self.current_item = dict()
self.current_item["engine_url"] = self.url
break
elif self.pending_next_queries:
if tag == "a":
params = dict(attrs)
self.list_searches.append(params['href'])
if params["title"] == "10":
self.pending_next_queries = False
else:
self.pending_next_queries = False
elif self.next_queries:
if tag == "b" and ("class", "pager_no_link") in attrs:
self.next_queries = False
self.pending_next_queries = True
def handle_data(self, data): def handle_data(self, data):
if self.td_counter == 2: if self.cur_item_name:
if 'name' not in self.current_item and self.start_name: temp = self.current_item[self.cur_item_name]
self.current_item['name'] = data.strip() self.current_item[self.cur_item_name] = " ".join((temp, data))
elif self.td_counter == 3: #Due to utf-8 we need to handle data two times if there is space
if 'size' not in self.current_item: if not self.cur_item_name == "size":
self.current_item['size'] = '' self.cur_item_name = None
self.current_item['size']+= data.replace("&nbsp;", " ").strip()
elif self.td_counter == 4: def handle_endtag(self, tag):
if 'seeds' not in self.current_item:
self.current_item['seeds'] = ''
self.current_item['seeds']+= data.strip()
elif self.td_counter == 5:
if 'leech' not in self.current_item:
self.current_item['leech'] = ''
self.current_item['leech']+= data.strip()
def start_td(self,attr):
if isinstance(self.td_counter,int):
self.td_counter += 1
if self.td_counter > 5:
self.td_counter = None
# Display item
if self.current_item: if self.current_item:
self.current_item['engine_url'] = self.url if tag == "tr":
if not self.current_item['seeds'].isdigit():
self.current_item['seeds'] = 0
if not self.current_item['leech'].isdigit():
self.current_item['leech'] = 0
prettyPrinter(self.current_item) prettyPrinter(self.current_item)
self.results.append('a') self.current_item = None
def search(self, what, cat='all'): def search(self, what, cat="all"):
ret = [] """ Performs search """
i = 1 connection = http("extratorrent.cc")
while True and i<11:
results = [] query = "".join(("/search/?new=1&search=", what, "&s_cat=", self.supported_categories[cat]))
parser = self.SimpleSGMLParser(results, self.url)
dat = retrieve_url(self.url+'/advanced_search/?with=%s&s_cat=%s&page=%d'%(what, self.supported_categories[cat], i)) connection.request("GET", query)
results_re = re.compile('(?s)<table class="tl"><thead>.*') response = connection.getresponse()
for match in results_re.finditer(dat): if response.status != 200:
res_tab = match.group(0) return
parser.feed(res_tab)
list_searches = []
parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
parser.feed(response.read().decode('utf-8'))
parser.close()
for search_query in list_searches:
connection.request("GET", search_query)
response = connection.getresponse()
parser.feed(response.read().decode('utf-8'))
parser.close() parser.close()
break
if len(results) <= 0:
break
i += 1
connection.close()
return

6
src/searchengine/nova3/engines/legittorrents.py
Unescape View File

@ -1,4 +1,4 @@
#VERSION: 1.03 #VERSION: 1.04
#AUTHORS: Christophe Dumez (chris@qbittorrent.org) #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
# Redistribution and use in source and binary forms, with or without # Redistribution and use in source and binary forms, with or without
@ -36,10 +36,6 @@ class legittorrents(object):
name = 'legittorrents' name = 'legittorrents'
supported_categories = {'all': '', 'movies': '1', 'tv': '13', 'music': '2', 'games': '3', 'anime': '5', 'books': '6'} supported_categories = {'all': '', 'movies': '1', 'tv': '13', 'music': '2', 'games': '3', 'anime': '5', 'books': '6'}
def __init__(self):
self.results = []
self.parser = self.SimpleSGMLParser(self.results, self.url)
def download_torrent(self, info): def download_torrent(self, info):
print(download_file(info)) print(download_file(info))

180
src/searchengine/nova3/engines/mininova.py
Unescape View File

@ -1,4 +1,4 @@
#VERSION: 1.51 #VERSION: 2.00
#AUTHORS: Christophe Dumez (chris@qbittorrent.org) #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
#CONTRIBUTORS: Diego de las Heras (diegodelasheras@gmail.com) #CONTRIBUTORS: Diego de las Heras (diegodelasheras@gmail.com)
@ -26,90 +26,124 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE. # POSSIBILITY OF SUCH DAMAGE.
from html.parser import HTMLParser
from http.client import HTTPConnection as http
from novaprinter import prettyPrinter from novaprinter import prettyPrinter
from helpers import retrieve_url, download_file from helpers import download_file
import sgmllib3
import re
class mininova(object): class mininova(object):
# Mandatory properties """ Search engine class """
url = 'http://www.mininova.org' url = 'http://www.mininova.org'
name = 'Mininova' name = 'Mininova'
supported_categories = {'all': '0', 'movies': '4', 'tv': '8', 'music': '5', 'games': '3', 'anime': '1', 'software': '7', 'pictures': '6', 'books': '2'} supported_categories = {'all' : '0',
'movies' : '4',
def __init__(self): 'tv' : '8',
self.results = [] 'music' : '5',
self.parser = self.SimpleSGMLParser(self.results, self.url) 'games' : '3',
'anime' : '1',
'software' : '7',
'pictures' : '6',
'books' : '2'}
def download_torrent(self, info): def download_torrent(self, info):
print(download_file(info)) print(download_file(info))
class SimpleSGMLParser(sgmllib3.SGMLParser): class MyHtmlParseWithBlackJack(HTMLParser):
def __init__(self, results, url, *args): """ Parser class """
sgmllib3.SGMLParser.__init__(self) def __init__(self, list_searches, url):
HTMLParser.__init__(self)
self.list_searches = list_searches
self.url = url self.url = url
self.td_counter = None self.table_results = False
self.current_item = None self.current_item = None
self.results = results self.cur_item_name = None
self.next_queries = True
def start_a(self, attr):
params = dict(attr)
#print params
if 'href' in params:
if params['href'].startswith("/get/"):
self.current_item = {}
self.td_counter = 0
self.current_item['link']=self.url+params['href'].strip()
elif params['href'].startswith("/tor/") and self.current_item is not None:
self.current_item['desc_link']=self.url+params['href'].strip()
def handle_data(self, data): def handle_starttag_tr(self, _):
if self.td_counter == 0: """ Handler of tr start tag """
if 'name' not in self.current_item: self.current_item = dict()
self.current_item['name'] = ''
self.current_item['name']+= data def handle_starttag_a(self, attrs):
elif self.td_counter == 1: """ Handler of a start tag """
if 'size' not in self.current_item: params = dict(attrs)
self.current_item['size'] = '' link = params["href"]
self.current_item['size']+= data.strip()
elif self.td_counter == 2: if link.startswith("/get/"):
if 'seeds' not in self.current_item: #download link
self.current_item['seeds'] = '' self.current_item["link"] = "".join((self.url, link))
self.current_item['seeds']+= data.strip() elif link.startswith("/tor/"):
elif self.td_counter == 3: #description
if 'leech' not in self.current_item: self.current_item["desc_link"] = "".join((self.url, link))
self.current_item['leech'] = '' self.cur_item_name = "name"
self.current_item['leech']+= data.strip() self.current_item["name"] = ""
elif self.next_queries and link.startswith("/search"):
def start_td(self,attr): if params["title"].startswith("Page"):
if isinstance(self.td_counter,int): self.list_searches.append(link)
self.td_counter += 1
if self.td_counter > 4: def handle_starttag_td(self, attrs):
self.td_counter = None """ Handler of td start tag """
# Display item if ("align", "right") in attrs:
if self.current_item: if not "size" in self.current_item.keys():
self.current_item['engine_url'] = self.url self.cur_item_name = "size"
if not self.current_item['seeds'].isdigit(): self.current_item["size"] = ""
self.current_item['seeds'] = 0
if not self.current_item['leech'].isdigit(): def handle_starttag_span(self, attrs):
self.current_item['leech'] = 0 """ Handler of span start tag """
if ("class", "g") in attrs:
self.cur_item_name = "seeds"
self.current_item["seeds"] = ""
elif ("class", "b") in attrs:
self.cur_item_name = "leech"
self.current_item["leech"] = ""
def handle_starttag(self, tag, attrs):
""" Parser's start tag handler """
if self.table_results:
dispatcher = getattr(self, "_".join(("handle_starttag", tag)), None)
if dispatcher:
dispatcher(attrs)
elif tag == "table":
self.table_results = ("class", "maintable") in attrs
def handle_endtag(self, tag):
""" Parser's end tag handler """
if tag == "tr" and self.current_item:
self.current_item["engine_url"] = self.url
prettyPrinter(self.current_item) prettyPrinter(self.current_item)
self.results.append('a') self.current_item = None
elif self.cur_item_name:
def search(self, what, cat='all'): if tag == "a" or tag == "span":
ret = [] self.cur_item_name = None
i = 1
while True and i<11: def handle_data(self, data):
results = [] """ Parser's data handler """
parser = self.SimpleSGMLParser(results, self.url) if self.cur_item_name:
dat = retrieve_url(self.url+'/search/%s/%s/seeds/%d'%(what, self.supported_categories[cat], i)) temp = self.current_item[self.cur_item_name]
results_re = re.compile('(?s)<h1>Search results for.*') self.current_item[self.cur_item_name] = " ".join((temp, data))
for match in results_re.finditer(dat):
res_tab = match.group(0) def search(self, what, cat="all"):
parser.feed(res_tab) """ Performs search """
connection = http("www.mininova.org")
query = "/".join(("/search", what, self.supported_categories[cat], "seeds"))
connection.request("GET", query)
response = connection.getresponse()
if response.status != 200:
return
list_searches = []
parser = self.MyHtmlParseWithBlackJack(list_searches, self.url)
parser.feed(response.read().decode('utf-8'))
parser.close()
parser.next_queries = False
for search_query in list_searches:
connection.request("GET", search_query)
response = connection.getresponse()
parser.feed(response.read().decode('utf-8'))
parser.close() parser.close()
break
if len(results) <= 0:
break
i += 1
connection.close()
return

25
src/searchengine/nova3/engines/torrentreactor.py
Unescape View File

@ -1,4 +1,4 @@
#VERSION: 1.33 #VERSION: 1.35
#AUTHORS: Gekko Dam Beer (gekko04@users.sourceforge.net) #AUTHORS: Gekko Dam Beer (gekko04@users.sourceforge.net)
#CONTRIBUTORS: Christophe Dumez (chris@qbittorrent.org) #CONTRIBUTORS: Christophe Dumez (chris@qbittorrent.org)
# Bruno Barbieri (brunorex@gmail.com) # Bruno Barbieri (brunorex@gmail.com)
@ -28,9 +28,10 @@
# POSSIBILITY OF SUCH DAMAGE. # POSSIBILITY OF SUCH DAMAGE.
from novaprinter import prettyPrinter from novaprinter import prettyPrinter
from helpers import retrieve_url, download_file from helpers import download_file
from urllib import error, parse from urllib import parse
from html.parser import HTMLParser from html.parser import HTMLParser
from http.client import HTTPConnection as http
import re import re
class torrentreactor(object): class torrentreactor(object):
@ -95,24 +96,26 @@ class torrentreactor(object):
self.has_results = True self.has_results = True
self.results.append('a') self.results.append('a')
def __init__(self):
self.results = []
self.parser = self.SimpleHTMLParser(self.results, self.url)
def search(self, what, cat='all'): def search(self, what, cat='all'):
i = 0 i = 0
dat = '' dat = ''
connection = http("www.torrentreactor.net")
while True and i<11: while True and i<11:
results = [] results = []
parser = self.SimpleHTMLParser(results, self.url) parser = self.SimpleHTMLParser(results, self.url)
query = '/torrents-search/%s/%d?sort=seeders.desc&type=all&period=none&categories=%s'%(what, (i*35), self.supported_categories[cat])
try: connection.request("GET", query)
dat = retrieve_url(self.url+'/torrent-search/%s/%s?sort=seeders.desc&type=all&period=none&categories=%s'%(what, (i*35), self.supported_categories[cat])) response = connection.getresponse()
except error.HTTPError: if response.status != 200:
break break
dat = response.read().decode('utf-8')
parser.feed(dat) parser.feed(dat)
parser.close() parser.close()
if len(results) <= 0: if len(results) <= 0:
break break
i += 1 i += 1
connection.close()

9
src/searchengine/nova3/engines/versions.txt
Unescape View File

@ -1,8 +1,9 @@
torrentreactor: 1.33
mininova: 1.51
piratebay: 2.11
extratorrent: 1.2 extratorrent: 1.2
torrentreactor: 1.35
mininova: 2.00
piratebay: 2.11
extratorrent: 2.0
kickasstorrents: 1.26 kickasstorrents: 1.26
btdigg: 1.23 btdigg: 1.23
legittorrents: 1.03
torrentz: 2.13 torrentz: 2.13
legittorrents: 1.04

Write Preview
Loading…
Cancel
Save