From 91ca2fc9996115864b0be45de7f5b09e766b5efa Mon Sep 17 00:00:00 2001 From: imDMG Date: Sun, 20 Sep 2020 23:07:37 +0500 Subject: [PATCH] minor refactoring, bump versions --- .gitignore | 9 ++--- kinozal.py | 77 +++++++++++++++++++++----------------- nnmclub.py | 81 +++++++++++++++++++++------------------- proxychecker.py | 63 ++++++++++++++++++++++++++----- rutor.py | 69 +++++++++++++++++----------------- rutracker.py | 98 +++++++++++++++++++++++++++++-------------------- 6 files changed, 240 insertions(+), 157 deletions(-) diff --git a/.gitignore b/.gitignore index ff436be..f9270ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ -/tests/ -/venv/ +/tests /kinozal.cookie /kinozal.cookie.bak /kinozal.ico @@ -10,11 +9,11 @@ /nnmclub.ico /nnmclub.json /nnmclub.json.bak +/proxylist.txt +/rutor.ico +/rutor.json /rutracker.cookie /rutracker.cookie.bak /rutracker.ico /rutracker.json /rutracker.json.bak -/rutor.ico -/rutor.json -/proxylist.txt diff --git a/kinozal.py b/kinozal.py index 5120683..63cc766 100644 --- a/kinozal.py +++ b/kinozal.py @@ -1,9 +1,10 @@ -# VERSION: 2.3 +# VERSION: 2.4 # AUTHORS: imDMG [imdmgg@gmail.com] # Kinozal.tv search engine plugin for qBittorrent import base64 +import gzip import json import logging import os @@ -24,7 +25,7 @@ from novaprinter import prettyPrinter # default config config = { - "version": 2, + "version": 3, "torrentDate": True, "username": "USERNAME", "password": "PASSWORD", @@ -46,12 +47,13 @@ def rng(t): return range(1, -(-t // 50)) -PATTERNS = (r'Найдено\s+?(\d+)\s+?раздач', - r'nam">(.*?).+?s\'>.+?s\'>' - r'(.*?)<.+?sl_s\'>(\d+)<.+?sl_p\'>(\d+)<.+?s\'>(.*?)', - '%sbrowse.php?s=%s&c=%s', "%s&page=%s") +RE_TORRENTS = re.compile( + r'nam">(.+?).+?s\'>.+?s\'>(.+?)<.+?' + r'sl_s\'>(\d+?)<.+?sl_p\'>(\d+?)<.+?s\'>(.+?)', re.S) +RE_RESULTS = re.compile(r'Найдено\s+?(\d+?)\s+?раздач', re.S) +PATTERNS = ('%sbrowse.php?s=%s&c=%s', "%s&page=%s") -FILENAME = __file__[__file__.rfind('/') + 1:-3] +FILENAME = os.path.basename(__file__)[:-3] FILE_J, FILE_C = [path_to(FILENAME + fe) for fe in ['.json', '.cookie']] # base64 encoded image @@ -81,9 +83,10 @@ ICON = ("AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAQAQAAAAAAAAAAA" # setup logging logging.basicConfig( format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s", - datefmt="%m-%d %H:%M") + datefmt="%m-%d %H:%M", + level=logging.DEBUG) + logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) try: # try to load user data from file @@ -111,6 +114,7 @@ class kinozal: name = 'Kinozal' url = 'http://kinozal.tv/' url_dl = url.replace("//", "//dl.") + url_login = url + 'takelogin.php' supported_categories = {'all': '0', 'movies': '1002', 'tv': '1001', @@ -135,8 +139,7 @@ class kinozal: self.error = "Proxy enabled, but not set!" # change user-agent - self.session.addheaders.pop() - self.session.addheaders.append(('User-Agent', config['ua'])) + self.session.addheaders = [('User-Agent', config['ua'])] # load local cookies mcj = MozillaCookieJar() @@ -157,18 +160,18 @@ class kinozal: def search(self, what, cat='all'): if self.error: self.pretty_error(what) - return - query = PATTERNS[2] % (self.url, what.replace(" ", "+"), + return None + query = PATTERNS[0] % (self.url, what.replace(" ", "+"), self.supported_categories[cat]) # make first request (maybe it enough) t0, total = time.time(), self.searching(query, True) if self.error: self.pretty_error(what) - return + return None # do async requests if total > 50: - qrs = [PATTERNS[3] % (query, x) for x in rng(total)] + qrs = [PATTERNS[1] % (query, x) for x in rng(total)] with ThreadPoolExecutor(len(qrs)) as executor: executor.map(self.searching, qrs, timeout=30) @@ -181,19 +184,21 @@ class kinozal: url = f"{self.url}get_srv_details.php?" \ f"action=2&id={url.split('=')[1]}" - res = self._catch_error_request(url) + response = self._catch_error_request(url) if self.error: self.pretty_error(url) - return + return None if config.get("magnet"): - path = 'magnet:?xt=urn:btih:' + res.read().decode()[18:58] + if response.startswith(b"\x1f\x8b\x08"): + response = gzip.decompress(response) + path = 'magnet:?xt=urn:btih:' + response.decode()[18:58] else: # Create a torrent file file, path = tempfile.mkstemp('.torrent') with os.fdopen(file, "wb") as fd: # Write it to a file - fd.write(res.read()) + fd.write(response) # return magnet link / file path logger.debug(path + " " + url) @@ -201,7 +206,7 @@ class kinozal: def login(self, mcj): if self.error: - return + return None self.session.add_handler(HTTPCookieProcessor(mcj)) form_data = {"username": config['username'], @@ -212,9 +217,9 @@ class kinozal: {k: v.encode('cp1251') for k, v in form_data.items()}).encode() logger.debug(f"Login. Data after: {data_encoded}") - self._catch_error_request(self.url + 'takelogin.php', data_encoded) + self._catch_error_request(self.url_login, data_encoded) if self.error: - return + return None logger.debug(f"That we have: {[cookie for cookie in mcj]}") if 'uid' in [cookie.name for cookie in mcj]: mcj.save(FILE_C, ignore_discard=True, ignore_expires=True) @@ -227,13 +232,20 @@ class kinozal: response = self._catch_error_request(query) if not response: return None - page = response.read().decode('cp1251') + if response.startswith(b"\x1f\x8b\x08"): + response = gzip.decompress(response) + page, torrents_found = response.decode('cp1251'), -1 + if first: + # firstly we check if there is a result + torrents_found = int(RE_RESULTS.search(page)[1]) + if not torrents_found: + return 0 self.draw(page) - return int(re.search(PATTERNS[0], page)[1]) if first else -1 + return torrents_found def draw(self, html: str): - torrents = re.findall(PATTERNS[1], html, re.S) + torrents = RE_TORRENTS.findall(html) _part = partial(time.strftime, "%y.%m.%d") # yeah this is yesterday yesterday = _part(time.localtime(time.time() - 86400)) @@ -256,23 +268,24 @@ class kinozal: "engine_url": self.url, "desc_link": self.url + tor[0], "name": torrent_date + unescape(tor[1]), - "link": self.url_dl + "download.php?id=" + tor[0].split("=")[1], + "link": f"{self.url_dl}download.php?id={tor[0].split('=')[-1]}", "size": tor[2].translate(tor[2].maketrans(table)), "seeds": tor[3], "leech": tor[4] }) del torrents - def _catch_error_request(self, url='', data=None, retrieve=False): + def _catch_error_request(self, url=None, data=None, repeated=False): url = url or self.url try: - response = self.session.open(url, data, 5) - # checking that tracker is'nt blocked - if not response.geturl().startswith((self.url, self.url_dl)): + with self.session.open(url, data, 5) as r: + # checking that tracker isn't blocked + if r.url.startswith((self.url, self.url_dl)): + return r.read() raise URLError(f"{self.url} is blocked. Try another proxy.") except (socket.error, socket.timeout) as err: - if not retrieve: + if not repeated: return self._catch_error_request(url, data, True) logger.error(err) self.error = f"{self.url} is not response! Maybe it is blocked." @@ -283,8 +296,6 @@ class kinozal: self.error = err.reason if hasattr(err, 'code'): self.error = f"Request to {url} failed with status: {err.code}" - else: - return response return None diff --git a/nnmclub.py b/nnmclub.py index d933c9f..a76601f 100644 --- a/nnmclub.py +++ b/nnmclub.py @@ -1,4 +1,4 @@ -# VERSION: 2.4 +# VERSION: 2.5 # AUTHORS: imDMG [imdmgg@gmail.com] # NoNaMe-Club search engine plugin for qBittorrent @@ -44,12 +44,13 @@ def rng(t): return range(50, -(-t // 50) * 50, 50) -PATTERNS = (r'(\d{1,3})\s\(max:', - r'd\stopic.+?href="(.+?)".+?(.+?).+?href="(d.+?)".+?/u>\s' - r'(.+?)<.+?b>(\d+)(\d+)<.+?(\d+)', - '%stracker.php?nm=%s&%s', "%s&start=%s", r'code"\svalue="(.+?)"') +RE_TORRENTS = re.compile( + r'topictitle"\shref="(.+?)">(.+?).+?href="(d.+?)".+?(\d+?).+?' + r'(\d+).+?(\d+).+?(\d+)', re.S) +RE_RESULTS = re.compile(r'TP_VER">(?:Результатов\sпоиска:\s(\d{1,3}))?\s', re.S) +PATTERNS = ('%stracker.php?nm=%s&%s', "%s&start=%s", r'code"\svalue="(.+?)"') -FILENAME = __file__[__file__.rfind('/') + 1:-3] +FILENAME = os.path.basename(__file__)[:-3] FILE_J, FILE_C = [path_to(FILENAME + fe) for fe in ['.json', '.cookie']] # base64 encoded image @@ -79,9 +80,10 @@ ICON = ("AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAA" # setup logging logging.basicConfig( format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s", - datefmt="%m-%d %H:%M") + datefmt="%m-%d %H:%M", + level=logging.DEBUG) + logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) try: # try to load user data from file @@ -109,6 +111,7 @@ class nnmclub: name = 'NoNaMe-Club' url = 'https://nnmclub.to/forum/' url_dl = 'https://nnm-club.ws/' + url_login = url + 'login.php' supported_categories = {'all': '-1', 'movies': '14', 'tv': '27', @@ -133,8 +136,7 @@ class nnmclub: self.error = "Proxy enabled, but not set!" # change user-agent - self.session.addheaders.pop() - self.session.addheaders.append(('User-Agent', config['ua'])) + self.session.addheaders = [('User-Agent', config['ua'])] # load local cookies mcj = MozillaCookieJar() @@ -155,19 +157,19 @@ class nnmclub: def search(self, what, cat='all'): if self.error: self.pretty_error(what) - return + return None c = self.supported_categories[cat] - query = PATTERNS[2] % (self.url, what.replace(" ", "+"), + query = PATTERNS[0] % (self.url, what.replace(" ", "+"), "f=-1" if c == "-1" else "c=" + c) # make first request (maybe it enough) t0, total = time.time(), self.searching(query, True) if self.error: self.pretty_error(what) - return + return None # do async requests if total > 50: - qrs = [PATTERNS[3] % (query, x) for x in rng(total)] + qrs = [PATTERNS[1] % (query, x) for x in rng(total)] with ThreadPoolExecutor(len(qrs)) as executor: executor.map(self.searching, qrs, timeout=30) @@ -179,13 +181,13 @@ class nnmclub: response = self._catch_error_request(url) if self.error: self.pretty_error(url) - return + return None # Create a torrent file file, path = tempfile.mkstemp('.torrent') with os.fdopen(file, "wb") as fd: # Write it to a file - fd.write(response.read()) + fd.write(response) # return file path logger.debug(path + " " + url) @@ -193,17 +195,17 @@ class nnmclub: def login(self, mcj): if self.error: - return + return None # if we wanna use https we mast add ssl=enable_ssl to cookie mcj.set_cookie(Cookie(0, "ssl", "enable_ssl", None, False, ".nnmclub.to", True, False, "/", True, False, None, False, None, None, {})) self.session.add_handler(HTTPCookieProcessor(mcj)) - response = self._catch_error_request(self.url + 'login.php') + response = self._catch_error_request(self.url_login) if not response: return None - code = re.search(PATTERNS[4], response.read().decode('cp1251'))[1] + code = re.search(PATTERNS[4], response.decode('cp1251'))[1] form_data = {"username": config['username'], "password": config['password'], "autologin": "on", @@ -213,9 +215,9 @@ class nnmclub: data_encoded = urlencode( {k: v.encode('cp1251') for k, v in form_data.items()}).encode() - self._catch_error_request(self.url + 'login.php', data_encoded) + self._catch_error_request(self.url_login, data_encoded) if self.error: - return + return None logger.debug(f"That we have: {[cookie for cookie in mcj]}") if 'phpbb2mysql_4_sid' in [cookie.name for cookie in mcj]: mcj.save(FILE_C, ignore_discard=True, ignore_expires=True) @@ -225,7 +227,7 @@ class nnmclub: logger.warning(self.error) def draw(self, html: str): - torrents = re.findall(PATTERNS[1], html, re.S) + torrents = RE_TORRENTS.findall(html) for tor in torrents: torrent_date = "" @@ -238,7 +240,7 @@ class nnmclub: "desc_link": self.url + tor[0], "name": torrent_date + unescape(tor[1]), "link": self.url + tor[2], - "size": tor[3].replace(',', '.'), + "size": tor[3], "seeds": tor[4], "leech": tor[5] }) @@ -248,26 +250,33 @@ class nnmclub: response = self._catch_error_request(query) if not response: return None - page = response.read().decode('cp1251') - if first and page.find(f'Выход [ {config["username"]} ]') == -1: - logger.debug("Looks like we lost session id, lets login") - self.login(MozillaCookieJar()) - if self.error: - return None + page, torrents_found = response.decode('cp1251'), -1 + if first: + # check login status + if f'Выход [ {config["username"]} ]' not in page: + logger.debug("Looks like we lost session id, lets login") + self.login(MozillaCookieJar()) + if self.error: + return None + # firstly we check if there is a result + torrents_found = int(RE_RESULTS.search(page)[1] or 0) + if not torrents_found: + return 0 self.draw(page) - return int(re.search(PATTERNS[0], page)[1]) if first else -1 + return torrents_found - def _catch_error_request(self, url='', data=None, retrieve=False): + def _catch_error_request(self, url=None, data=None, repeated=False): url = url or self.url try: - response = self.session.open(url, data, 5) - # checking that tracker is'nt blocked - if not response.geturl().startswith((self.url, self.url_dl)): + with self.session.open(url, data, 5) as r: + # checking that tracker isn't blocked + if r.url.startswith((self.url, self.url_dl)): + return r.read() raise URLError(f"{self.url} is blocked. Try another proxy.") except (socket.error, socket.timeout) as err: - if not retrieve: + if not repeated: return self._catch_error_request(url, data, True) logger.error(err) self.error = f"{self.url} is not response! Maybe it is blocked." @@ -278,8 +287,6 @@ class nnmclub: self.error = err.reason if hasattr(err, 'code'): self.error = f"Request to {url} failed with status: {err.code}" - else: - return response return None diff --git a/proxychecker.py b/proxychecker.py index f4443d2..b10987f 100644 --- a/proxychecker.py +++ b/proxychecker.py @@ -1,31 +1,76 @@ +import asyncio +import functools +import time +import threading from concurrent.futures.thread import ThreadPoolExecutor from urllib.request import build_opener, ProxyHandler -HOST = "http://kinozal.tv/" +HOST = "https://rutracker.org/forum/" SCHEME = HOST[:HOST.rfind("://")] -PROXY_FILE = "proxylist.txt" # one address per line +PROXY_FILE = "proxylist1.txt" # one address per line -def print_good_proxy(proxy): +async def is_good_proxy_aio(loop, proxy: str) -> bool: try: opener = build_opener(ProxyHandler({f"{SCHEME}": proxy})) opener.addheaders = [("User-agent", "Mozilla/5.0")] - req = opener.open(HOST, timeout=30) - if not req.geturl().startswith(HOST): + _part = functools.partial(opener.open, HOST, timeout=3) + res = await loop.run_in_executor(None, _part) + if not res.geturl().startswith(HOST): raise Exception() + print(proxy) + return True except Exception as e: - return e + # print(e) + return False + + +async def run_aio(proxies): + loop = asyncio.get_event_loop() + for proxy in proxies: + await is_good_proxy_aio(loop, proxy) + + +def run_thread(proxies): + tasks = [] + for proxy in proxies: + task = threading.Thread(target=is_good_proxy, args=(proxy,)) + task.start() + tasks.append(task) + for t in tasks: + t.join() + + +def run_pool(proxies): + with ThreadPoolExecutor(len(proxies)) as executor: + executor.map(is_good_proxy, proxies, timeout=30) - print(proxy) + +def is_good_proxy(proxy): + try: + opener = build_opener(ProxyHandler({f"{SCHEME}": proxy})) + opener.addheaders = [("User-agent", "Mozilla/5.0")] + req = opener.open(HOST, timeout=3) + if not req.geturl().startswith(HOST): + raise Exception() + except Exception as e: + # print(e) + return False + else: + print(proxy) + return True def main(): + t0 = time.time() with open(PROXY_FILE) as f: proxy_list = [x.rstrip() for x in f] print("Working proxies:") - with ThreadPoolExecutor(len(proxy_list)) as executor: - executor.map(print_good_proxy, proxy_list, timeout=30) + # run_thread(proxy_list) + run_pool(proxy_list) + # asyncio.run(run_aio(proxy_list)) + print(time.time() - t0) if __name__ == '__main__': diff --git a/rutor.py b/rutor.py index cd34935..7eb8686 100644 --- a/rutor.py +++ b/rutor.py @@ -1,4 +1,4 @@ -# VERSION: 1.0 +# VERSION: 1.1 # AUTHORS: imDMG [imdmgg@gmail.com] # Rutor.org search engine plugin for qBittorrent @@ -44,12 +44,13 @@ def rng(t): return range(1, -(-t // 100)) -PATTERNS = (r'(\d+)\s\(max. 2000\)(.+?)(.+?)<\/a.+?' - r'right">([\d\.]+ \w+)(.+?)<.+?red">(.+?)<', - '%ssearch/%i/%i/000/0/%s') +RE_TORRENTS = re.compile( + r'(?:gai|tum)">(.+?)(.+?)([.\d]+ \w+)(.+?)(.+?)\sРезультатов\sпоиска\s(\d{1,4})\s', re.S) +PATTERNS = ('%ssearch/%i/%i/000/0/%s',) -FILENAME = __file__[__file__.rfind('/') + 1:-3] +FILENAME = os.path.basename(__file__)[:-3] FILE_J, FILE_C = [path_to(FILENAME + fe) for fe in ['.json', '.cookie']] # base64 encoded image @@ -75,9 +76,10 @@ ICON = ("AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAA" # setup logging logging.basicConfig( format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s", - datefmt="%m-%d %H:%M") + datefmt="%m-%d %H:%M", + level=logging.DEBUG) + logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) try: # try to load user data from file @@ -98,7 +100,7 @@ except OSError as e: class rutor: name = 'Rutor' url = 'http://rutor.info/' - url_dl = url.replace("//", "//d.") + url_dl = url.replace("//", "//d.") + "download/" supported_categories = {'all': 0, 'movies': 1, 'tv': 6, @@ -125,21 +127,20 @@ class rutor: self.error = "Proxy enabled, but not set!" # change user-agent - self.session.addheaders.pop() - self.session.addheaders.append(('User-Agent', config['ua'])) + self.session.addheaders = [('User-Agent', config['ua'])] def search(self, what, cat='all'): if self.error: self.pretty_error(what) - return - query = PATTERNS[2] % (self.url, 0, self.supported_categories[cat], + return None + query = PATTERNS[0] % (self.url, 0, self.supported_categories[cat], what.replace(" ", "+")) # make first request (maybe it enough) t0, total = time.time(), self.searching(query, True) if self.error: self.pretty_error(what) - return + return None # do async requests if total > 100: query = query.replace('h/0', 'h/%i') @@ -155,13 +156,13 @@ class rutor: response = self._catch_error_request(url) if self.error: self.pretty_error(url) - return + return None # Create a torrent file file, path = tempfile.mkstemp('.torrent') with os.fdopen(file, "wb") as fd: # Write it to a file - fd.write(response.read()) + fd.write(response) # return file path logger.debug(path + " " + url) @@ -171,23 +172,26 @@ class rutor: response = self._catch_error_request(query) if not response: return None - page = response.read().decode() + page, torrents_found = response.decode(), -1 + if first: + # firstly we check if there is a result + torrents_found = int(RE_RESULTS.search(page)[1]) + if not torrents_found: + return 0 self.draw(page) - return int(re.search(PATTERNS[0], page)[1]) if first else -1 + return torrents_found def draw(self, html: str): - torrents = re.findall(PATTERNS[1], html, re.S) + torrents = RE_TORRENTS.findall(html) for tor in torrents: torrent_date = "" if config['torrentDate']: # replace names month - table = {'Янв': '01', 'Фев': '02', 'Мар': '03', 'Апр': '04', - 'Май': '05', 'Июн': '06', 'Июл': '07', 'Авг': '08', - 'Сен': '09', 'Окт': '10', 'Ноя': '11', 'Дек': '12'} - td = tor[0].split(" ") - td[1] = table[td[1]] - ct = " ".join(td) + months = ('Янв', 'Фев', 'Мар', 'Апр', 'Май', 'Июн', + 'Июл', 'Авг', 'Сен', 'Окт', 'Ноя', 'Дек') + ct = [unescape(tor[0].replace(m, f"{i:02d}")) + for i, m in enumerate(months, 1) if m in tor[0]][0] ct = time.strftime("%y.%m.%d", time.strptime(ct, "%d %m %y")) torrent_date = f'[{ct}] ' @@ -195,23 +199,24 @@ class rutor: "engine_url": self.url, "desc_link": self.url + tor[1], "name": torrent_date + unescape(tor[3]), - "link": self.url_dl + "download/" + tor[2], + "link": self.url_dl + tor[2], "size": unescape(tor[4]), "seeds": unescape(tor[5]), "leech": unescape(tor[6]) }) del torrents - def _catch_error_request(self, url='', data=None, retrieve=False): + def _catch_error_request(self, url=None, data=None, repeated=False): url = url or self.url try: - response = self.session.open(url, data, 5) - # checking that tracker is'nt blocked - if not response.geturl().startswith((self.url, self.url_dl)): + with self.session.open(url, data, 5) as r: + # checking that tracker isn't blocked + if r.url.startswith((self.url, self.url_dl)): + return r.read() raise URLError(f"{self.url} is blocked. Try another proxy.") except (socket.error, socket.timeout) as err: - if not retrieve: + if not repeated: return self._catch_error_request(url, data, True) logger.error(err) self.error = f"{self.url} is not response! Maybe it is blocked." @@ -222,8 +227,6 @@ class rutor: self.error = err.reason if hasattr(err, 'code'): self.error = f"Request to {url} failed with status: {err.code}" - else: - return response return None diff --git a/rutracker.py b/rutracker.py index bdfc699..f4c5195 100644 --- a/rutracker.py +++ b/rutracker.py @@ -1,4 +1,4 @@ -# VERSION: 1.2 +# VERSION: 1.3 # AUTHORS: imDMG [imdmgg@gmail.com] # rutracker.org search engine plugin for qBittorrent @@ -44,12 +44,14 @@ def rng(t): return range(50, -(-t // 50) * 50, 50) -PATTERNS = (r'(\d{1,3})\s(.+?)(.+?)\s&.+?data-ts_text="(.+?)">.+?Личи">(\d+)(.+?)' + r'.+?data-ts_text="([-0-9]+?)">.+?Личи">(\d+?)', + re.S) +RE_RESULTS = re.compile(r'Результатов\sпоиска:\s(\d{1,3})\s 50: - qrs = [PATTERNS[3] % (query, x) for x in rng(total)] + qrs = [PATTERNS[1] % (query, x) for x in rng(total)] with ThreadPoolExecutor(len(qrs)) as executor: executor.map(self.searching, qrs, timeout=30) @@ -172,13 +176,13 @@ class rutracker: response = self._catch_error_request(url) if self.error: self.pretty_error(url) - return + return None # Create a torrent file file, path = tempfile.mkstemp('.torrent') with os.fdopen(file, "wb") as fd: # Write it to a file - fd.write(response.read()) + fd.write(response) # return file path logger.debug(path + " " + url) @@ -186,24 +190,24 @@ class rutracker: def login(self, mcj): if self.error: - return - # if we wanna use https we mast add ssl=enable_ssl to cookie - mcj.set_cookie(Cookie(0, 'ssl', "enable_ssl", None, False, - '.rutracker.org', True, False, '/', True, - False, None, 'ParserCookie', None, None, None)) + return None + # if we wanna use https we mast add bb_ssl=1 to cookie + mcj.set_cookie(Cookie(0, "bb_ssl", "1", None, False, ".rutracker.org", + True, True, "/forum/", True, True, + None, False, None, None, {})) self.session.add_handler(HTTPCookieProcessor(mcj)) form_data = {"login_username": config['username'], "login_password": config['password'], - "login": "вход"} + "login": "Вход"} logger.debug(f"Login. Data before: {form_data}") # so we first encode vals to cp1251 then do default decode whole string data_encoded = urlencode( {k: v.encode('cp1251') for k, v in form_data.items()}).encode() logger.debug(f"Login. Data after: {data_encoded}") - self._catch_error_request(self.url + 'login.php', data_encoded) + self._catch_error_request(self.url_login, data_encoded) if self.error: - return + return None logger.debug(f"That we have: {[cookie for cookie in mcj]}") if 'bb_session' in [cookie.name for cookie in mcj]: mcj.save(FILE_C, ignore_discard=True, ignore_expires=True) @@ -216,38 +220,54 @@ class rutracker: response = self._catch_error_request(query) if not response: return None - page = response.read().decode('cp1251') + page, torrents_found = response.decode('cp1251'), -1 + if first: + if "log-out-icon" not in page: + logger.debug("Looks like we lost session id, lets login") + self.login(MozillaCookieJar()) + if self.error: + return None + # retry request because guests cant search + response = self._catch_error_request(query) + if not response: + return None + page = response.decode('cp1251') + # firstly we check if there is a result + torrents_found = int(RE_RESULTS.search(page)[1]) + if not torrents_found: + return 0 self.draw(page) - return int(re.search(PATTERNS[0], page)[1]) if first else -1 + return torrents_found def draw(self, html: str): - torrents = re.findall(PATTERNS[1], html, re.S) + torrents = RE_TORRENTS.findall(html) for tor in torrents: - local = time.strftime("%y.%m.%d", time.localtime(int(tor[6]))) + local = time.strftime("%y.%m.%d", time.localtime(int(tor[5]))) torrent_date = f"[{local}] " if config['torrentDate'] else "" prettyPrinter({ "engine_url": self.url, - "desc_link": self.url + tor[0], + "desc_link": self.url + "viewtopic.php?t=" + tor[0], "name": torrent_date + unescape(tor[1]), - "link": self.url + tor[2], - "size": unescape(tor[3]), - "seeds": tor[4] if tor[4].isdigit() else '0', - "leech": tor[5] + "link": self.url_dl + tor[0], + "size": tor[2], + "seeds": max(0, int(tor[3])), + "leech": tor[4] }) del torrents - def _catch_error_request(self, url='', data=None, retrieve=False): + def _catch_error_request(self, url=None, data=None, repeated=False): url = url or self.url try: - response = self.session.open(url, data, 5) - # checking that tracker is'nt blocked - if not response.geturl().startswith(self.url): + with self.session.open(url, data, 5) as r: + # checking that tracker isn't blocked + if r.url.startswith((self.url, self.url_dl)): + return r.read() raise URLError(f"{self.url} is blocked. Try another proxy.") except (socket.error, socket.timeout) as err: - if not retrieve: + if not repeated: return self._catch_error_request(url, data, True) logger.error(err) self.error = f"{self.url} is not response! Maybe it is blocked." @@ -258,8 +278,6 @@ class rutracker: self.error = err.reason if hasattr(err, 'code'): self.error = f"Request to {url} failed with status: {err.code}" - else: - return response return None