Mike Tzou
5 years ago
committed by
GitHub
12 changed files with 16 additions and 1247 deletions
@ -1,371 +0,0 @@ |
|||||||
# Copyright (c) 2011 The Chromium Authors. All rights reserved. |
|
||||||
# Use of this source code is governed by a BSD-style license that can be |
|
||||||
# found in the LICENSE file. |
|
||||||
|
|
||||||
"""Collection of functions and classes to fix various encoding problems on |
|
||||||
multiple platforms with python. |
|
||||||
""" |
|
||||||
|
|
||||||
import codecs |
|
||||||
import locale |
|
||||||
import os |
|
||||||
import sys |
|
||||||
|
|
||||||
|
|
||||||
# Prevents initializing multiple times. |
|
||||||
_SYS_ARGV_PROCESSED = False |
|
||||||
|
|
||||||
|
|
||||||
def complain(message): |
|
||||||
"""If any exception occurs in this file, we'll probably try to print it |
|
||||||
on stderr, which makes for frustrating debugging if stderr is directed |
|
||||||
to our wrapper. So be paranoid about catching errors and reporting them |
|
||||||
to sys.__stderr__, so that the user has a higher chance to see them. |
|
||||||
""" |
|
||||||
print >> sys.__stderr__, ( |
|
||||||
isinstance(message, str) and message or repr(message)) |
|
||||||
|
|
||||||
|
|
||||||
def fix_default_encoding(): |
|
||||||
"""Forces utf8 solidly on all platforms. |
|
||||||
|
|
||||||
By default python execution environment is lazy and defaults to ascii |
|
||||||
encoding. |
|
||||||
|
|
||||||
http://uucode.com/blog/2007/03/23/shut-up-you-dummy-7-bit-python/ |
|
||||||
""" |
|
||||||
if sys.getdefaultencoding() == 'utf-8': |
|
||||||
return False |
|
||||||
|
|
||||||
# Regenerate setdefaultencoding. |
|
||||||
reload(sys) |
|
||||||
# Module 'sys' has no 'setdefaultencoding' member |
|
||||||
# pylint: disable=E1101 |
|
||||||
sys.setdefaultencoding('utf-8') |
|
||||||
for attr in dir(locale): |
|
||||||
if attr[0:3] != 'LC_': |
|
||||||
continue |
|
||||||
aref = getattr(locale, attr) |
|
||||||
try: |
|
||||||
locale.setlocale(aref, '') |
|
||||||
except locale.Error: |
|
||||||
continue |
|
||||||
try: |
|
||||||
lang = locale.getlocale(aref)[0] |
|
||||||
except (TypeError, ValueError): |
|
||||||
continue |
|
||||||
if lang: |
|
||||||
try: |
|
||||||
locale.setlocale(aref, (lang, 'UTF-8')) |
|
||||||
except locale.Error: |
|
||||||
os.environ[attr] = lang + '.UTF-8' |
|
||||||
try: |
|
||||||
locale.setlocale(locale.LC_ALL, '') |
|
||||||
except locale.Error: |
|
||||||
pass |
|
||||||
return True |
|
||||||
|
|
||||||
|
|
||||||
############################### |
|
||||||
# Windows specific |
|
||||||
|
|
||||||
|
|
||||||
def fix_win_sys_argv(encoding): |
|
||||||
"""Converts sys.argv to 'encoding' encoded string. |
|
||||||
|
|
||||||
utf-8 is recommended. |
|
||||||
|
|
||||||
Works around <http://bugs.python.org/issue2128>. |
|
||||||
""" |
|
||||||
global _SYS_ARGV_PROCESSED |
|
||||||
if _SYS_ARGV_PROCESSED: |
|
||||||
return False |
|
||||||
|
|
||||||
# These types are available on linux but not Mac. |
|
||||||
# pylint: disable=E0611,F0401 |
|
||||||
from ctypes import byref, c_int, POINTER, windll, WINFUNCTYPE |
|
||||||
from ctypes.wintypes import LPCWSTR, LPWSTR |
|
||||||
|
|
||||||
# <http://msdn.microsoft.com/en-us/library/ms683156.aspx> |
|
||||||
GetCommandLineW = WINFUNCTYPE(LPWSTR)(('GetCommandLineW', windll.kernel32)) |
|
||||||
# <http://msdn.microsoft.com/en-us/library/bb776391.aspx> |
|
||||||
CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( |
|
||||||
('CommandLineToArgvW', windll.shell32)) |
|
||||||
|
|
||||||
argc = c_int(0) |
|
||||||
argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) |
|
||||||
argv = [ |
|
||||||
argv_unicode[i].encode(encoding, 'replace') |
|
||||||
for i in xrange(0, argc.value)] |
|
||||||
|
|
||||||
if not hasattr(sys, 'frozen'): |
|
||||||
# If this is an executable produced by py2exe or bbfreeze, then it |
|
||||||
# will have been invoked directly. Otherwise, unicode_argv[0] is the |
|
||||||
# Python interpreter, so skip that. |
|
||||||
argv = argv[1:] |
|
||||||
|
|
||||||
# Also skip option arguments to the Python interpreter. |
|
||||||
while len(argv) > 0: |
|
||||||
arg = argv[0] |
|
||||||
if not arg.startswith(u'-') or arg == u'-': |
|
||||||
break |
|
||||||
argv = argv[1:] |
|
||||||
if arg == u'-m': |
|
||||||
# sys.argv[0] should really be the absolute path of the |
|
||||||
# module source, but never mind. |
|
||||||
break |
|
||||||
if arg == u'-c': |
|
||||||
argv[0] = u'-c' |
|
||||||
break |
|
||||||
sys.argv = argv |
|
||||||
_SYS_ARGV_PROCESSED = True |
|
||||||
return True |
|
||||||
|
|
||||||
|
|
||||||
def fix_win_codec(): |
|
||||||
"""Works around <http://bugs.python.org/issue6058>.""" |
|
||||||
# <http://msdn.microsoft.com/en-us/library/dd317756.aspx> |
|
||||||
try: |
|
||||||
codecs.lookup('cp65001') |
|
||||||
return False |
|
||||||
except LookupError: |
|
||||||
codecs.register( |
|
||||||
lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) |
|
||||||
return True |
|
||||||
|
|
||||||
|
|
||||||
class WinUnicodeOutputBase(object): |
|
||||||
"""Base class to adapt sys.stdout or sys.stderr to behave correctly on |
|
||||||
Windows. |
|
||||||
|
|
||||||
Setting encoding to utf-8 is recommended. |
|
||||||
""" |
|
||||||
def __init__(self, fileno, name, encoding): |
|
||||||
# Corresponding file handle. |
|
||||||
self._fileno = fileno |
|
||||||
self.encoding = encoding |
|
||||||
self.name = name |
|
||||||
|
|
||||||
self.closed = False |
|
||||||
self.softspace = False |
|
||||||
self.mode = 'w' |
|
||||||
|
|
||||||
@staticmethod |
|
||||||
def isatty(): |
|
||||||
return False |
|
||||||
|
|
||||||
def close(self): |
|
||||||
# Don't really close the handle, that would only cause problems. |
|
||||||
self.closed = True |
|
||||||
|
|
||||||
def fileno(self): |
|
||||||
return self._fileno |
|
||||||
|
|
||||||
def flush(self): |
|
||||||
raise NotImplementedError() |
|
||||||
|
|
||||||
def write(self, text): |
|
||||||
raise NotImplementedError() |
|
||||||
|
|
||||||
def writelines(self, lines): |
|
||||||
try: |
|
||||||
for line in lines: |
|
||||||
self.write(line) |
|
||||||
except Exception, e: |
|
||||||
complain('%s.writelines: %r' % (self.name, e)) |
|
||||||
raise |
|
||||||
|
|
||||||
|
|
||||||
class WinUnicodeConsoleOutput(WinUnicodeOutputBase): |
|
||||||
"""Output adapter to a Windows Console. |
|
||||||
|
|
||||||
Understands how to use the win32 console API. |
|
||||||
""" |
|
||||||
def __init__(self, console_handle, fileno, stream_name, encoding): |
|
||||||
super(WinUnicodeConsoleOutput, self).__init__( |
|
||||||
fileno, '<Unicode console %s>' % stream_name, encoding) |
|
||||||
# Handle to use for WriteConsoleW |
|
||||||
self._console_handle = console_handle |
|
||||||
|
|
||||||
# Loads the necessary function. |
|
||||||
# These types are available on linux but not Mac. |
|
||||||
# pylint: disable=E0611,F0401 |
|
||||||
from ctypes import byref, GetLastError, POINTER, windll, WINFUNCTYPE |
|
||||||
from ctypes.wintypes import BOOL, DWORD, HANDLE, LPWSTR |
|
||||||
from ctypes.wintypes import LPVOID # pylint: disable=E0611 |
|
||||||
|
|
||||||
self._DWORD = DWORD |
|
||||||
self._byref = byref |
|
||||||
|
|
||||||
# <http://msdn.microsoft.com/en-us/library/ms687401.aspx> |
|
||||||
self._WriteConsoleW = WINFUNCTYPE( |
|
||||||
BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)( |
|
||||||
('WriteConsoleW', windll.kernel32)) |
|
||||||
self._GetLastError = GetLastError |
|
||||||
|
|
||||||
def flush(self): |
|
||||||
# No need to flush the console since it's immediate. |
|
||||||
pass |
|
||||||
|
|
||||||
def write(self, text): |
|
||||||
try: |
|
||||||
if not isinstance(text, unicode): |
|
||||||
# Convert to unicode. |
|
||||||
text = str(text).decode(self.encoding, 'replace') |
|
||||||
remaining = len(text) |
|
||||||
while remaining > 0: |
|
||||||
n = self._DWORD(0) |
|
||||||
# There is a shorter-than-documented limitation on the length of the |
|
||||||
# string passed to WriteConsoleW. See |
|
||||||
# <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>. |
|
||||||
retval = self._WriteConsoleW( |
|
||||||
self._console_handle, text, |
|
||||||
min(remaining, 10000), |
|
||||||
self._byref(n), None) |
|
||||||
if retval == 0 or n.value == 0: |
|
||||||
raise IOError( |
|
||||||
'WriteConsoleW returned %r, n.value = %r, last error = %r' % ( |
|
||||||
retval, n.value, self._GetLastError())) |
|
||||||
remaining -= n.value |
|
||||||
if not remaining: |
|
||||||
break |
|
||||||
text = text[n.value:] |
|
||||||
except Exception, e: |
|
||||||
complain('%s.write: %r' % (self.name, e)) |
|
||||||
raise |
|
||||||
|
|
||||||
|
|
||||||
class WinUnicodeOutput(WinUnicodeOutputBase): |
|
||||||
"""Output adaptor to a file output on Windows. |
|
||||||
|
|
||||||
If the standard FileWrite function is used, it will be encoded in the current |
|
||||||
code page. WriteConsoleW() permits writing any character. |
|
||||||
""" |
|
||||||
def __init__(self, stream, fileno, encoding): |
|
||||||
super(WinUnicodeOutput, self).__init__( |
|
||||||
fileno, '<Unicode redirected %s>' % stream.name, encoding) |
|
||||||
# Output stream |
|
||||||
self._stream = stream |
|
||||||
|
|
||||||
# Flush right now. |
|
||||||
self.flush() |
|
||||||
|
|
||||||
def flush(self): |
|
||||||
try: |
|
||||||
self._stream.flush() |
|
||||||
except Exception, e: |
|
||||||
complain('%s.flush: %r from %r' % (self.name, e, self._stream)) |
|
||||||
raise |
|
||||||
|
|
||||||
def write(self, text): |
|
||||||
try: |
|
||||||
if isinstance(text, unicode): |
|
||||||
# Replace characters that cannot be printed instead of failing. |
|
||||||
text = text.encode(self.encoding, 'replace') |
|
||||||
self._stream.write(text) |
|
||||||
except Exception, e: |
|
||||||
complain('%s.write: %r' % (self.name, e)) |
|
||||||
raise |
|
||||||
|
|
||||||
|
|
||||||
def win_handle_is_a_console(handle): |
|
||||||
"""Returns True if a Windows file handle is a handle to a console.""" |
|
||||||
# These types are available on linux but not Mac. |
|
||||||
# pylint: disable=E0611,F0401 |
|
||||||
from ctypes import byref, POINTER, windll, WINFUNCTYPE |
|
||||||
from ctypes.wintypes import BOOL, DWORD, HANDLE |
|
||||||
|
|
||||||
FILE_TYPE_CHAR = 0x0002 |
|
||||||
FILE_TYPE_REMOTE = 0x8000 |
|
||||||
INVALID_HANDLE_VALUE = DWORD(-1).value |
|
||||||
|
|
||||||
# <http://msdn.microsoft.com/en-us/library/ms683167.aspx> |
|
||||||
GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( |
|
||||||
('GetConsoleMode', windll.kernel32)) |
|
||||||
# <http://msdn.microsoft.com/en-us/library/aa364960.aspx> |
|
||||||
GetFileType = WINFUNCTYPE(DWORD, DWORD)(('GetFileType', windll.kernel32)) |
|
||||||
|
|
||||||
# GetStdHandle returns INVALID_HANDLE_VALUE, NULL, or a valid handle. |
|
||||||
if handle == INVALID_HANDLE_VALUE or handle is None: |
|
||||||
return False |
|
||||||
return ( |
|
||||||
(GetFileType(handle) & ~FILE_TYPE_REMOTE) == FILE_TYPE_CHAR and |
|
||||||
GetConsoleMode(handle, byref(DWORD()))) |
|
||||||
|
|
||||||
|
|
||||||
def win_get_unicode_stream(stream, excepted_fileno, output_handle, encoding): |
|
||||||
"""Returns a unicode-compatible stream. |
|
||||||
|
|
||||||
This function will return a direct-Console writing object only if: |
|
||||||
- the file number is the expected console file number |
|
||||||
- the handle the expected file handle |
|
||||||
- the 'real' handle is in fact a handle to a console. |
|
||||||
""" |
|
||||||
old_fileno = getattr(stream, 'fileno', lambda: None)() |
|
||||||
if old_fileno == excepted_fileno: |
|
||||||
# These types are available on linux but not Mac. |
|
||||||
# pylint: disable=E0611,F0401 |
|
||||||
from ctypes import windll, WINFUNCTYPE |
|
||||||
from ctypes.wintypes import DWORD, HANDLE |
|
||||||
|
|
||||||
# <http://msdn.microsoft.com/en-us/library/ms683231.aspx> |
|
||||||
GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(('GetStdHandle', windll.kernel32)) |
|
||||||
|
|
||||||
real_output_handle = GetStdHandle(DWORD(output_handle)) |
|
||||||
if win_handle_is_a_console(real_output_handle): |
|
||||||
# It's a console. |
|
||||||
return WinUnicodeConsoleOutput( |
|
||||||
real_output_handle, old_fileno, stream.name, encoding) |
|
||||||
|
|
||||||
# It's something else. Create an auto-encoding stream. |
|
||||||
return WinUnicodeOutput(stream, old_fileno, encoding) |
|
||||||
|
|
||||||
|
|
||||||
def fix_win_console(encoding): |
|
||||||
"""Makes Unicode console output work independently of the current code page. |
|
||||||
|
|
||||||
This also fixes <http://bugs.python.org/issue1602>. |
|
||||||
Credit to Michael Kaplan |
|
||||||
<http://blogs.msdn.com/b/michkap/archive/2010/04/07/9989346.aspx> and |
|
||||||
TZOmegaTZIOY |
|
||||||
<http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>. |
|
||||||
""" |
|
||||||
if (isinstance(sys.stdout, WinUnicodeOutputBase) or |
|
||||||
isinstance(sys.stderr, WinUnicodeOutputBase)): |
|
||||||
return False |
|
||||||
|
|
||||||
try: |
|
||||||
# SetConsoleCP and SetConsoleOutputCP could be used to change the code page |
|
||||||
# but it's not really useful since the code here is using WriteConsoleW(). |
|
||||||
# Also, changing the code page is 'permanent' to the console and needs to be |
|
||||||
# reverted manually. |
|
||||||
# In practice one needs to set the console font to a TTF font to be able to |
|
||||||
# see all the characters but it failed for me in practice. In any case, it |
|
||||||
# won't throw any exception when printing, which is the important part. |
|
||||||
# -11 and -12 are defined in stdio.h |
|
||||||
sys.stdout = win_get_unicode_stream(sys.stdout, 1, -11, encoding) |
|
||||||
sys.stderr = win_get_unicode_stream(sys.stderr, 2, -12, encoding) |
|
||||||
# TODO(maruel): Do sys.stdin with ReadConsoleW(). Albeit the limitation is |
|
||||||
# "It doesn't appear to be possible to read Unicode characters in UTF-8 |
|
||||||
# mode" and this appears to be a limitation of cmd.exe. |
|
||||||
except Exception, e: |
|
||||||
complain('exception %r while fixing up sys.stdout and sys.stderr' % e) |
|
||||||
return True |
|
||||||
|
|
||||||
|
|
||||||
def fix_encoding(): |
|
||||||
"""Fixes various encoding problems on all platforms. |
|
||||||
|
|
||||||
Should be called at the very beginning of the process. |
|
||||||
""" |
|
||||||
ret = True |
|
||||||
if sys.platform == 'win32': |
|
||||||
ret &= fix_win_codec() |
|
||||||
|
|
||||||
ret &= fix_default_encoding() |
|
||||||
|
|
||||||
if sys.platform == 'win32': |
|
||||||
encoding = sys.getdefaultencoding() |
|
||||||
ret &= fix_win_sys_argv(encoding) |
|
||||||
ret &= fix_win_console(encoding) |
|
||||||
return ret |
|
||||||
@ -1,122 +0,0 @@ |
|||||||
#VERSION: 1.44 |
|
||||||
|
|
||||||
# Author: |
|
||||||
# Christophe DUMEZ (chris@qbittorrent.org) |
|
||||||
# Contributors: |
|
||||||
# Diego de las Heras (ngosang@hotmail.es) |
|
||||||
|
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright notice, |
|
||||||
# this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer in the |
|
||||||
# documentation and/or other materials provided with the distribution. |
|
||||||
# * Neither the name of the author nor the names of its contributors may be |
|
||||||
# used to endorse or promote products derived from this software without |
|
||||||
# specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
|
||||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
|
||||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
|
||||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
|
||||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
|
||||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
|
||||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
|
||||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
|
||||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
|
||||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
||||||
# POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
import gzip |
|
||||||
import htmlentitydefs |
|
||||||
import os |
|
||||||
import re |
|
||||||
import socket |
|
||||||
import socks |
|
||||||
import StringIO |
|
||||||
import tempfile |
|
||||||
import urllib2 |
|
||||||
|
|
||||||
# Some sites blocks default python User-agent |
|
||||||
user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0' |
|
||||||
headers = {'User-Agent': user_agent} |
|
||||||
# SOCKS5 Proxy support |
|
||||||
if ("sock_proxy" in os.environ) and (len(os.environ["sock_proxy"].strip()) > 0): |
|
||||||
proxy_str = os.environ["sock_proxy"].strip() |
|
||||||
m = re.match(r"^(?:(?P<username>[^:]+):(?P<password>[^@]+)@)?(?P<host>[^:]+):(?P<port>\w+)$", |
|
||||||
proxy_str) |
|
||||||
if m is not None: |
|
||||||
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, m.group('host'), |
|
||||||
int(m.group('port')), True, m.group('username'), m.group('password')) |
|
||||||
socket.socket = socks.socksocket |
|
||||||
|
|
||||||
|
|
||||||
def htmlentitydecode(s): |
|
||||||
# First convert alpha entities (such as é) |
|
||||||
# (Inspired from http://mail.python.org/pipermail/python-list/2007-June/443813.html) |
|
||||||
def entity2char(m): |
|
||||||
entity = m.group(1) |
|
||||||
if entity in htmlentitydefs.name2codepoint: |
|
||||||
return unichr(htmlentitydefs.name2codepoint[entity]) |
|
||||||
return u" " # Unknown entity: We replace with a space. |
|
||||||
t = re.sub(u'&(%s);' % u'|'.join(htmlentitydefs.name2codepoint), entity2char, s) |
|
||||||
|
|
||||||
# Then convert numerical entities (such as é) |
|
||||||
t = re.sub(r'&#(\d+);', lambda x: unichr(int(x.group(1))), t) |
|
||||||
|
|
||||||
# Then convert hexa entities (such as é) |
|
||||||
return re.sub(r'&#x(\w+);', lambda x: unichr(int(x.group(1), 16)), t) |
|
||||||
|
|
||||||
|
|
||||||
def retrieve_url(url): |
|
||||||
""" Return the content of the url page as a string """ |
|
||||||
req = urllib2.Request(url, headers=headers) |
|
||||||
try: |
|
||||||
response = urllib2.urlopen(req) |
|
||||||
except urllib2.URLError as errno: |
|
||||||
print(" ".join(("Connection error:", str(errno.reason)))) |
|
||||||
return "" |
|
||||||
dat = response.read() |
|
||||||
# Check if it is gzipped |
|
||||||
if dat[:2] == '\037\213': |
|
||||||
# Data is gzip encoded, decode it |
|
||||||
compressedstream = StringIO.StringIO(dat) |
|
||||||
gzipper = gzip.GzipFile(fileobj=compressedstream) |
|
||||||
extracted_data = gzipper.read() |
|
||||||
dat = extracted_data |
|
||||||
info = response.info() |
|
||||||
charset = 'utf-8' |
|
||||||
try: |
|
||||||
ignore, charset = info['Content-Type'].split('charset=') |
|
||||||
except Exception: |
|
||||||
pass |
|
||||||
dat = dat.decode(charset, 'replace') |
|
||||||
dat = htmlentitydecode(dat) |
|
||||||
return dat |
|
||||||
|
|
||||||
|
|
||||||
def download_file(url, referer=None): |
|
||||||
""" Download file at url and write it to a file, return the path to the file and the url """ |
|
||||||
file, path = tempfile.mkstemp() |
|
||||||
file = os.fdopen(file, "w") |
|
||||||
# Download url |
|
||||||
req = urllib2.Request(url, headers=headers) |
|
||||||
if referer is not None: |
|
||||||
req.add_header('referer', referer) |
|
||||||
response = urllib2.urlopen(req) |
|
||||||
dat = response.read() |
|
||||||
# Check if it is gzipped |
|
||||||
if dat[:2] == '\037\213': |
|
||||||
# Data is gzip encoded, decode it |
|
||||||
compressedstream = StringIO.StringIO(dat) |
|
||||||
gzipper = gzip.GzipFile(fileobj=compressedstream) |
|
||||||
extracted_data = gzipper.read() |
|
||||||
dat = extracted_data |
|
||||||
|
|
||||||
# Write it to a file |
|
||||||
file.write(dat) |
|
||||||
file.close() |
|
||||||
# return file path |
|
||||||
return (path + " " + url) |
|
||||||
@ -1,191 +0,0 @@ |
|||||||
#VERSION: 1.43 |
|
||||||
|
|
||||||
# Author: |
|
||||||
# Fabien Devaux <fab AT gnux DOT info> |
|
||||||
# Contributors: |
|
||||||
# Christophe Dumez <chris@qbittorrent.org> (qbittorrent integration) |
|
||||||
# Thanks to gab #gcu @ irc.freenode.net (multipage support on PirateBay) |
|
||||||
# Thanks to Elias <gekko04@users.sourceforge.net> (torrentreactor and isohunt search engines) |
|
||||||
# |
|
||||||
# Licence: BSD |
|
||||||
|
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright notice, |
|
||||||
# this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer in the |
|
||||||
# documentation and/or other materials provided with the distribution. |
|
||||||
# * Neither the name of the author nor the names of its contributors may be |
|
||||||
# used to endorse or promote products derived from this software without |
|
||||||
# specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
|
||||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
|
||||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
|
||||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
|
||||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
|
||||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
|
||||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
|
||||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
|
||||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
|
||||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
||||||
# POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
import urllib |
|
||||||
from os import path |
|
||||||
from glob import glob |
|
||||||
from sys import argv |
|
||||||
from multiprocessing import Pool, cpu_count |
|
||||||
from fix_encoding import fix_encoding |
|
||||||
|
|
||||||
THREADED = True |
|
||||||
try: |
|
||||||
MAX_THREADS = cpu_count() |
|
||||||
except NotImplementedError: |
|
||||||
MAX_THREADS = 1 |
|
||||||
|
|
||||||
CATEGORIES = {'all', 'movies', 'tv', 'music', 'games', 'anime', 'software', 'pictures', 'books'} |
|
||||||
|
|
||||||
################################################################################ |
|
||||||
# Every engine should have a "search" method taking |
|
||||||
# a space-free string as parameter (ex. "family+guy") |
|
||||||
# it should call prettyPrinter() with a dict as parameter. |
|
||||||
# The keys in the dict must be: link,name,size,seeds,leech,engine_url |
|
||||||
# As a convention, try to list results by decreasing number of seeds or similar |
|
||||||
################################################################################ |
|
||||||
|
|
||||||
|
|
||||||
def initialize_engines(): |
|
||||||
""" Import available engines |
|
||||||
|
|
||||||
Return list of available engines |
|
||||||
""" |
|
||||||
supported_engines = [] |
|
||||||
|
|
||||||
engines = glob(path.join(path.dirname(__file__), 'engines', '*.py')) |
|
||||||
for engine in engines: |
|
||||||
engi = path.basename(engine).split('.')[0].strip() |
|
||||||
if len(engi) == 0 or engi.startswith('_'): |
|
||||||
continue |
|
||||||
try: |
|
||||||
# import engines.[engine] |
|
||||||
engine_module = __import__(".".join(("engines", engi))) |
|
||||||
# get low-level module |
|
||||||
engine_module = getattr(engine_module, engi) |
|
||||||
# bind class name |
|
||||||
globals()[engi] = getattr(engine_module, engi) |
|
||||||
supported_engines.append(engi) |
|
||||||
except Exception: |
|
||||||
pass |
|
||||||
|
|
||||||
return supported_engines |
|
||||||
|
|
||||||
|
|
||||||
def engines_to_xml(supported_engines): |
|
||||||
""" Generates xml for supported engines """ |
|
||||||
tab = " " * 4 |
|
||||||
|
|
||||||
for short_name in supported_engines: |
|
||||||
search_engine = globals()[short_name]() |
|
||||||
|
|
||||||
supported_categories = "" |
|
||||||
if hasattr(search_engine, "supported_categories"): |
|
||||||
supported_categories = " ".join((key |
|
||||||
for key in search_engine.supported_categories.keys() |
|
||||||
if key != "all")) |
|
||||||
|
|
||||||
yield "".join((tab, "<", short_name, ">\n", |
|
||||||
tab, tab, "<name>", search_engine.name, "</name>\n", |
|
||||||
tab, tab, "<url>", search_engine.url, "</url>\n", |
|
||||||
tab, tab, "<categories>", supported_categories, "</categories>\n", |
|
||||||
tab, "</", short_name, ">\n")) |
|
||||||
|
|
||||||
|
|
||||||
def displayCapabilities(supported_engines): |
|
||||||
""" |
|
||||||
Display capabilities in XML format |
|
||||||
<capabilities> |
|
||||||
<engine_short_name> |
|
||||||
<name>long name</name> |
|
||||||
<url>http://example.com</url> |
|
||||||
<categories>movies music games</categories> |
|
||||||
</engine_short_name> |
|
||||||
</capabilities> |
|
||||||
""" |
|
||||||
xml = "".join(("<capabilities>\n", |
|
||||||
"".join(engines_to_xml(supported_engines)), |
|
||||||
"</capabilities>")) |
|
||||||
print(xml) |
|
||||||
|
|
||||||
|
|
||||||
def run_search(engine_list): |
|
||||||
""" Run search in engine |
|
||||||
|
|
||||||
@param engine_list List with engine, query and category |
|
||||||
|
|
||||||
@retval False if any exceptions occurred |
|
||||||
@retval True otherwise |
|
||||||
""" |
|
||||||
engine, what, cat = engine_list |
|
||||||
try: |
|
||||||
engine = engine() |
|
||||||
# avoid exceptions due to invalid category |
|
||||||
if hasattr(engine, 'supported_categories'): |
|
||||||
if cat in engine.supported_categories: |
|
||||||
engine.search(what, cat) |
|
||||||
else: |
|
||||||
engine.search(what) |
|
||||||
return True |
|
||||||
except Exception: |
|
||||||
return False |
|
||||||
|
|
||||||
|
|
||||||
def main(args): |
|
||||||
fix_encoding() |
|
||||||
supported_engines = initialize_engines() |
|
||||||
|
|
||||||
if not args: |
|
||||||
raise SystemExit("./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n" |
|
||||||
"available engines: %s" % (','.join(supported_engines))) |
|
||||||
|
|
||||||
elif args[0] == "--capabilities": |
|
||||||
displayCapabilities(supported_engines) |
|
||||||
return |
|
||||||
|
|
||||||
elif len(args) < 3: |
|
||||||
raise SystemExit("./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n" |
|
||||||
"available engines: %s" % (','.join(supported_engines))) |
|
||||||
|
|
||||||
# get only unique engines with set |
|
||||||
engines_list = set(e.lower() for e in args[0].strip().split(',')) |
|
||||||
|
|
||||||
if 'all' in engines_list: |
|
||||||
engines_list = supported_engines |
|
||||||
else: |
|
||||||
# discard un-supported engines |
|
||||||
engines_list = [engine for engine in engines_list |
|
||||||
if engine in supported_engines] |
|
||||||
|
|
||||||
if not engines_list: |
|
||||||
# engine list is empty. Nothing to do here |
|
||||||
return |
|
||||||
|
|
||||||
cat = args[1].lower() |
|
||||||
|
|
||||||
if cat not in CATEGORIES: |
|
||||||
raise SystemExit(" - ".join(('Invalid category', cat))) |
|
||||||
|
|
||||||
what = urllib.quote(' '.join(args[2:])) |
|
||||||
|
|
||||||
if THREADED: |
|
||||||
# child process spawning is controlled min(number of searches, number of cpu) |
|
||||||
pool = Pool(min(len(engines_list), MAX_THREADS)) |
|
||||||
pool.map(run_search, ([globals()[engine], what, cat] for engine in engines_list)) |
|
||||||
else: |
|
||||||
map(run_search, ([globals()[engine], what, cat] for engine in engines_list)) |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
main(argv[1:]) |
|
||||||
@ -1,63 +0,0 @@ |
|||||||
#VERSION: 1.22 |
|
||||||
|
|
||||||
# Author: |
|
||||||
# Christophe DUMEZ (chris@qbittorrent.org) |
|
||||||
|
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright notice, |
|
||||||
# this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer in the |
|
||||||
# documentation and/or other materials provided with the distribution. |
|
||||||
# * Neither the name of the author nor the names of its contributors may be |
|
||||||
# used to endorse or promote products derived from this software without |
|
||||||
# specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
|
||||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
|
||||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
|
||||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
|
||||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
|
||||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
|
||||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
|
||||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
|
||||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
|
||||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
||||||
# POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
import sys |
|
||||||
import os |
|
||||||
import glob |
|
||||||
from helpers import download_file |
|
||||||
|
|
||||||
supported_engines = dict() |
|
||||||
|
|
||||||
engines = glob.glob(os.path.join(os.path.dirname(__file__), 'engines', '*.py')) |
|
||||||
for engine in engines: |
|
||||||
e = engine.split(os.sep)[-1][:-3] |
|
||||||
if len(e.strip()) == 0: |
|
||||||
continue |
|
||||||
if e.startswith('_'): |
|
||||||
continue |
|
||||||
try: |
|
||||||
exec("from engines.%s import %s" % (e, e)) |
|
||||||
exec("engine_url = %s.url" % e) |
|
||||||
supported_engines[engine_url] = e |
|
||||||
except Exception: |
|
||||||
pass |
|
||||||
|
|
||||||
if __name__ == '__main__': |
|
||||||
if len(sys.argv) < 3: |
|
||||||
raise SystemExit('./nova2dl.py engine_url download_parameter') |
|
||||||
engine_url = sys.argv[1].strip() |
|
||||||
download_param = sys.argv[2].strip() |
|
||||||
if engine_url not in list(supported_engines.keys()): |
|
||||||
raise SystemExit('./nova2dl.py: this engine_url was not recognized') |
|
||||||
exec("engine = %s()" % supported_engines[engine_url]) |
|
||||||
if hasattr(engine, 'download_torrent'): |
|
||||||
engine.download_torrent(download_param) |
|
||||||
else: |
|
||||||
print(download_file(download_param)) |
|
||||||
sys.exit(0) |
|
||||||
@ -1,73 +0,0 @@ |
|||||||
#VERSION: 1.47 |
|
||||||
|
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright notice, |
|
||||||
# this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer in the |
|
||||||
# documentation and/or other materials provided with the distribution. |
|
||||||
# * Neither the name of the author nor the names of its contributors may be |
|
||||||
# used to endorse or promote products derived from this software without |
|
||||||
# specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
|
||||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
|
||||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
|
||||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
|
||||||
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
|
||||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
|
||||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
|
||||||
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
|
||||||
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
|
||||||
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
||||||
# POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
import codecs |
|
||||||
import sys |
|
||||||
from io import open |
|
||||||
|
|
||||||
# Force UTF-8 printing |
|
||||||
sys.stdout = codecs.getwriter('utf-8')(sys.stdout) |
|
||||||
|
|
||||||
|
|
||||||
def prettyPrinter(dictionary): |
|
||||||
dictionary['size'] = anySizeToBytes(dictionary['size']) |
|
||||||
outtext = "|".join((dictionary["link"], dictionary["name"].replace("|", " "), |
|
||||||
str(dictionary["size"]), str(dictionary["seeds"]), |
|
||||||
str(dictionary["leech"]), dictionary["engine_url"])) |
|
||||||
if 'desc_link' in dictionary: |
|
||||||
outtext = "|".join((outtext, dictionary["desc_link"])) |
|
||||||
|
|
||||||
with open(1, 'w', encoding='utf-8', closefd=False) as utf8_stdout: |
|
||||||
utf8_stdout.write(unicode("".join((outtext, "\n")))) |
|
||||||
|
|
||||||
|
|
||||||
def anySizeToBytes(size_string): |
|
||||||
""" |
|
||||||
Convert a string like '1 KB' to '1024' (bytes) |
|
||||||
""" |
|
||||||
# separate integer from unit |
|
||||||
try: |
|
||||||
size, unit = size_string.split() |
|
||||||
except Exception: |
|
||||||
try: |
|
||||||
size = size_string.strip() |
|
||||||
unit = ''.join([c for c in size if c.isalpha()]) |
|
||||||
if len(unit) > 0: |
|
||||||
size = size[:-len(unit)] |
|
||||||
except Exception: |
|
||||||
return -1 |
|
||||||
if len(size) == 0: |
|
||||||
return -1 |
|
||||||
size = float(size) |
|
||||||
if len(unit) == 0: |
|
||||||
return int(size) |
|
||||||
short_unit = unit.upper()[0] |
|
||||||
|
|
||||||
# convert |
|
||||||
units_dict = {'T': 40, 'G': 30, 'M': 20, 'K': 10} |
|
||||||
if short_unit in units_dict: |
|
||||||
size = size * 2**units_dict[short_unit] |
|
||||||
return int(size) |
|
||||||
@ -1,391 +0,0 @@ |
|||||||
"""SocksiPy - Python SOCKS module. |
|
||||||
Version 1.01 |
|
||||||
|
|
||||||
Copyright 2006 Dan-Haim. All rights reserved. |
|
||||||
Various fixes by Christophe DUMEZ <chris@qbittorrent.org> - 2010 |
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without modification, |
|
||||||
are permitted provided that the following conditions are met: |
|
||||||
1. Redistributions of source code must retain the above copyright notice, this |
|
||||||
list of conditions and the following disclaimer. |
|
||||||
2. Redistributions in binary form must reproduce the above copyright notice, |
|
||||||
this list of conditions and the following disclaimer in the documentation |
|
||||||
and/or other materials provided with the distribution. |
|
||||||
3. Neither the name of Dan Haim nor the names of his contributors may be used |
|
||||||
to endorse or promote products derived from this software without specific |
|
||||||
prior written permission. |
|
||||||
|
|
||||||
THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED |
|
||||||
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
|
||||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
|
||||||
EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
|
||||||
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA |
|
||||||
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
|
||||||
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT |
|
||||||
OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. |
|
||||||
|
|
||||||
|
|
||||||
This module provides a standard socket-like interface for Python |
|
||||||
for tunneling connections through SOCKS proxies. |
|
||||||
|
|
||||||
""" |
|
||||||
|
|
||||||
import socket |
|
||||||
import struct |
|
||||||
|
|
||||||
PROXY_TYPE_SOCKS4 = 1 |
|
||||||
PROXY_TYPE_SOCKS5 = 2 |
|
||||||
PROXY_TYPE_HTTP = 3 |
|
||||||
|
|
||||||
_defaultproxy = None |
|
||||||
_orgsocket = socket.socket |
|
||||||
|
|
||||||
class ProxyError(Exception): |
|
||||||
def __init__(self, value): |
|
||||||
self.value = value |
|
||||||
def __str__(self): |
|
||||||
return repr(self.value) |
|
||||||
|
|
||||||
class GeneralProxyError(ProxyError): |
|
||||||
def __init__(self, value): |
|
||||||
self.value = value |
|
||||||
def __str__(self): |
|
||||||
return repr(self.value) |
|
||||||
|
|
||||||
class Socks5AuthError(ProxyError): |
|
||||||
def __init__(self, value): |
|
||||||
self.value = value |
|
||||||
def __str__(self): |
|
||||||
return repr(self.value) |
|
||||||
|
|
||||||
class Socks5Error(ProxyError): |
|
||||||
def __init__(self, value): |
|
||||||
self.value = value |
|
||||||
def __str__(self): |
|
||||||
return repr(self.value) |
|
||||||
|
|
||||||
class Socks4Error(ProxyError): |
|
||||||
def __init__(self, value): |
|
||||||
self.value = value |
|
||||||
def __str__(self): |
|
||||||
return repr(self.value) |
|
||||||
|
|
||||||
class HTTPError(ProxyError): |
|
||||||
def __init__(self, value): |
|
||||||
self.value = value |
|
||||||
def __str__(self): |
|
||||||
return repr(self.value) |
|
||||||
|
|
||||||
_generalerrors = ("success", |
|
||||||
"invalid data", |
|
||||||
"not connected", |
|
||||||
"not available", |
|
||||||
"bad proxy type", |
|
||||||
"bad input") |
|
||||||
|
|
||||||
_socks5errors = ("succeeded", |
|
||||||
"general SOCKS server failure", |
|
||||||
"connection not allowed by ruleset", |
|
||||||
"Network unreachable", |
|
||||||
"Host unreachable", |
|
||||||
"Connection refused", |
|
||||||
"TTL expired", |
|
||||||
"Command not supported", |
|
||||||
"Address type not supported", |
|
||||||
"Unknown error") |
|
||||||
|
|
||||||
_socks5autherrors = ("succeeded", |
|
||||||
"authentication is required", |
|
||||||
"all offered authentication methods were rejected", |
|
||||||
"unknown username or invalid password", |
|
||||||
"unknown error") |
|
||||||
|
|
||||||
_socks4errors = ("request granted", |
|
||||||
"request rejected or failed", |
|
||||||
"request rejected because SOCKS server cannot connect to identd on the client", |
|
||||||
"request rejected because the client program and identd report different user-ids", |
|
||||||
"unknown error") |
|
||||||
|
|
||||||
def setdefaultproxy(proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): |
|
||||||
"""setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) |
|
||||||
Sets a default proxy which all further socksocket objects will use, |
|
||||||
unless explicitly changed. |
|
||||||
""" |
|
||||||
global _defaultproxy |
|
||||||
_defaultproxy = (proxytype,addr,port,rdns,username,password) |
|
||||||
|
|
||||||
class socksocket(socket.socket): |
|
||||||
"""socksocket([family[, type[, proto]]]) -> socket object |
|
||||||
|
|
||||||
Open a SOCKS enabled socket. The parameters are the same as |
|
||||||
those of the standard socket init. In order for SOCKS to work, |
|
||||||
you must specify family=AF_INET, type=SOCK_STREAM and proto=0. |
|
||||||
""" |
|
||||||
|
|
||||||
def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): |
|
||||||
_orgsocket.__init__(self,family,type,proto,_sock) |
|
||||||
if _defaultproxy != None: |
|
||||||
self.__proxy = _defaultproxy |
|
||||||
else: |
|
||||||
self.__proxy = (None, None, None, None, None, None) |
|
||||||
self.__proxysockname = None |
|
||||||
self.__proxypeername = None |
|
||||||
|
|
||||||
def __recvall(self, bytes): |
|
||||||
"""__recvall(bytes) -> data |
|
||||||
Receive EXACTLY the number of bytes requested from the socket. |
|
||||||
Blocks until the required number of bytes have been received. |
|
||||||
""" |
|
||||||
data = "" |
|
||||||
while len(data) < bytes: |
|
||||||
d = self.recv(bytes-len(data)) |
|
||||||
if not d: |
|
||||||
raise GeneralProxyError("connection closed unexpectedly") |
|
||||||
data = data + d |
|
||||||
return data |
|
||||||
|
|
||||||
def setproxy(self,proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): |
|
||||||
"""setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) |
|
||||||
Sets the proxy to be used. |
|
||||||
proxytype - The type of the proxy to be used. Three types |
|
||||||
are supported: PROXY_TYPE_SOCKS4 (including socks4a), |
|
||||||
PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP |
|
||||||
addr - The address of the server (IP or DNS). |
|
||||||
port - The port of the server. Defaults to 1080 for SOCKS |
|
||||||
servers and 8080 for HTTP proxy servers. |
|
||||||
rdns - Should DNS queries be preformed on the remote side |
|
||||||
(rather than the local side). The default is True. |
|
||||||
Note: This has no effect with SOCKS4 servers. |
|
||||||
username - Username to authenticate with to the server. |
|
||||||
The default is no authentication. |
|
||||||
password - Password to authenticate with to the server. |
|
||||||
Only relevant when username is also provided. |
|
||||||
""" |
|
||||||
self.__proxy = (proxytype,addr,port,rdns,username,password) |
|
||||||
|
|
||||||
def __negotiatesocks5(self,destaddr,destport): |
|
||||||
"""__negotiatesocks5(self,destaddr,destport) |
|
||||||
Negotiates a connection through a SOCKS5 server. |
|
||||||
""" |
|
||||||
# First we'll send the authentication packages we support. |
|
||||||
if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): |
|
||||||
# The username/password details were supplied to the |
|
||||||
# setproxy method so we support the USERNAME/PASSWORD |
|
||||||
# authentication (in addition to the standard none). |
|
||||||
self.sendall("\x05\x02\x00\x02") |
|
||||||
else: |
|
||||||
# No username/password were entered, therefore we |
|
||||||
# only support connections with no authentication. |
|
||||||
self.sendall("\x05\x01\x00") |
|
||||||
# We'll receive the server's response to determine which |
|
||||||
# method was selected |
|
||||||
chosenauth = self.__recvall(2) |
|
||||||
if chosenauth[0] != "\x05": |
|
||||||
self.close() |
|
||||||
raise GeneralProxyError((1,_generalerrors[1])) |
|
||||||
# Check the chosen authentication method |
|
||||||
if chosenauth[1] == "\x00": |
|
||||||
# No authentication is required |
|
||||||
pass |
|
||||||
elif chosenauth[1] == "\x02": |
|
||||||
# Okay, we need to perform a basic username/password |
|
||||||
# authentication. |
|
||||||
self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5]) |
|
||||||
authstat = self.__recvall(2) |
|
||||||
if authstat[0] != "\x01": |
|
||||||
# Bad response |
|
||||||
self.close() |
|
||||||
raise GeneralProxyError((1,_generalerrors[1])) |
|
||||||
if authstat[1] != "\x00": |
|
||||||
# Authentication failed |
|
||||||
self.close() |
|
||||||
raise Socks5AuthError,((3,_socks5autherrors[3])) |
|
||||||
# Authentication succeeded |
|
||||||
else: |
|
||||||
# Reaching here is always bad |
|
||||||
self.close() |
|
||||||
if chosenauth[1] == "\xFF": |
|
||||||
raise Socks5AuthError((2,_socks5autherrors[2])) |
|
||||||
else: |
|
||||||
raise GeneralProxyError((1,_generalerrors[1])) |
|
||||||
# Now we can request the actual connection |
|
||||||
req = "\x05\x01\x00" |
|
||||||
# If the given destination address is an IP address, we'll |
|
||||||
# use the IPv4 address request even if remote resolving was specified. |
|
||||||
try: |
|
||||||
ipaddr = socket.inet_aton(destaddr) |
|
||||||
req = req + "\x01" + ipaddr |
|
||||||
except socket.error: |
|
||||||
# Well it's not an IP number, so it's probably a DNS name. |
|
||||||
if self.__proxy[3]==True: |
|
||||||
# Resolve remotely |
|
||||||
ipaddr = None |
|
||||||
req = req + "\x03" + chr(len(destaddr)) + destaddr |
|
||||||
else: |
|
||||||
# Resolve locally |
|
||||||
ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) |
|
||||||
req = req + "\x01" + ipaddr |
|
||||||
req = req + struct.pack(">H",destport) |
|
||||||
self.sendall(req) |
|
||||||
# Get the response |
|
||||||
resp = self.__recvall(4) |
|
||||||
if resp[0] != "\x05": |
|
||||||
self.close() |
|
||||||
raise GeneralProxyError((1,_generalerrors[1])) |
|
||||||
elif resp[1] != "\x00": |
|
||||||
# Connection failed |
|
||||||
self.close() |
|
||||||
if ord(resp[1])<=8: |
|
||||||
raise Socks5Error((ord(resp[1]),_generalerrors[ord(resp[1])])) |
|
||||||
else: |
|
||||||
raise Socks5Error((9,_generalerrors[9])) |
|
||||||
# Get the bound address/port |
|
||||||
elif resp[3] == "\x01": |
|
||||||
boundaddr = self.__recvall(4) |
|
||||||
elif resp[3] == "\x03": |
|
||||||
resp = resp + self.recv(1) |
|
||||||
boundaddr = self.__recvall(ord(resp[4])) |
|
||||||
else: |
|
||||||
self.close() |
|
||||||
raise GeneralProxyError((1,_generalerrors[1])) |
|
||||||
boundport = struct.unpack(">H",self.__recvall(2))[0] |
|
||||||
self.__proxysockname = (boundaddr,boundport) |
|
||||||
if ipaddr != None: |
|
||||||
self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) |
|
||||||
else: |
|
||||||
self.__proxypeername = (destaddr,destport) |
|
||||||
|
|
||||||
def getproxysockname(self): |
|
||||||
"""getsockname() -> address info |
|
||||||
Returns the bound IP address and port number at the proxy. |
|
||||||
""" |
|
||||||
return self.__proxysockname |
|
||||||
|
|
||||||
def getproxypeername(self): |
|
||||||
"""getproxypeername() -> address info |
|
||||||
Returns the IP and port number of the proxy. |
|
||||||
""" |
|
||||||
return _orgsocket.getpeername(self) |
|
||||||
|
|
||||||
def getpeername(self): |
|
||||||
"""getpeername() -> address info |
|
||||||
Returns the IP address and port number of the destination |
|
||||||
machine (note: getproxypeername returns the proxy) |
|
||||||
""" |
|
||||||
return self.__proxypeername |
|
||||||
|
|
||||||
def __negotiatesocks4(self,destaddr,destport): |
|
||||||
"""__negotiatesocks4(self,destaddr,destport) |
|
||||||
Negotiates a connection through a SOCKS4 server. |
|
||||||
""" |
|
||||||
# Check if the destination address provided is an IP address |
|
||||||
rmtrslv = False |
|
||||||
try: |
|
||||||
ipaddr = socket.inet_aton(destaddr) |
|
||||||
except socket.error: |
|
||||||
# It's a DNS name. Check where it should be resolved. |
|
||||||
if self.__proxy[3]==True: |
|
||||||
ipaddr = "\x00\x00\x00\x01" |
|
||||||
rmtrslv = True |
|
||||||
else: |
|
||||||
ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) |
|
||||||
# Construct the request packet |
|
||||||
req = "\x04\x01" + struct.pack(">H",destport) + ipaddr |
|
||||||
# The username parameter is considered userid for SOCKS4 |
|
||||||
if self.__proxy[4] != None: |
|
||||||
req = req + self.__proxy[4] |
|
||||||
req = req + "\x00" |
|
||||||
# DNS name if remote resolving is required |
|
||||||
# NOTE: This is actually an extension to the SOCKS4 protocol |
|
||||||
# called SOCKS4A and may not be supported in all cases. |
|
||||||
if rmtrslv==True: |
|
||||||
req = req + destaddr + "\x00" |
|
||||||
self.sendall(req) |
|
||||||
# Get the response from the server |
|
||||||
resp = self.__recvall(8) |
|
||||||
if resp[0] != "\x00": |
|
||||||
# Bad data |
|
||||||
self.close() |
|
||||||
raise GeneralProxyError((1,_generalerrors[1])) |
|
||||||
if resp[1] != "\x5A": |
|
||||||
# Server returned an error |
|
||||||
self.close() |
|
||||||
if ord(resp[1]) in (91,92,93): |
|
||||||
self.close() |
|
||||||
raise Socks4Error((ord(resp[1]),_socks4errors[ord(resp[1])-90])) |
|
||||||
else: |
|
||||||
raise Socks4Error((94,_socks4errors[4])) |
|
||||||
# Get the bound address/port |
|
||||||
self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",resp[2:4])[0]) |
|
||||||
if rmtrslv != None: |
|
||||||
self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) |
|
||||||
else: |
|
||||||
self.__proxypeername = (destaddr,destport) |
|
||||||
|
|
||||||
def __negotiatehttp(self,destaddr,destport): |
|
||||||
"""__negotiatehttp(self,destaddr,destport) |
|
||||||
Negotiates a connection through an HTTP server. |
|
||||||
""" |
|
||||||
# If we need to resolve locally, we do this now |
|
||||||
if self.__proxy[3] == False: |
|
||||||
addr = socket.gethostbyname(destaddr) |
|
||||||
else: |
|
||||||
addr = destaddr |
|
||||||
self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n") |
|
||||||
# We read the response until we get the string "\r\n\r\n" |
|
||||||
resp = self.recv(1) |
|
||||||
while resp.find("\r\n\r\n")==-1: |
|
||||||
resp = resp + self.recv(1) |
|
||||||
# We just need the first line to check if the connection |
|
||||||
# was successful |
|
||||||
statusline = resp.splitlines()[0].split(" ",2) |
|
||||||
if statusline[0] not in ("HTTP/1.0","HTTP/1.1"): |
|
||||||
self.close() |
|
||||||
raise GeneralProxyError((1,_generalerrors[1])) |
|
||||||
try: |
|
||||||
statuscode = int(statusline[1]) |
|
||||||
except ValueError: |
|
||||||
self.close() |
|
||||||
raise GeneralProxyError((1,_generalerrors[1])) |
|
||||||
if statuscode != 200: |
|
||||||
self.close() |
|
||||||
raise HTTPError((statuscode,statusline[2])) |
|
||||||
self.__proxysockname = ("0.0.0.0",0) |
|
||||||
self.__proxypeername = (addr,destport) |
|
||||||
|
|
||||||
def connect(self,destpair): |
|
||||||
"""connect(self,despair) |
|
||||||
Connects to the specified destination through a proxy. |
|
||||||
destpar - A tuple of the IP/DNS address and the port number. |
|
||||||
(identical to socket's connect). |
|
||||||
To select the proxy server use setproxy(). |
|
||||||
""" |
|
||||||
# Do a minimal input check first |
|
||||||
if (type(destpair) in (list,tuple)==False) or (len(destpair)<2) or (type(destpair[0])!=str) or (type(destpair[1])!=int): |
|
||||||
raise GeneralProxyError((5,_generalerrors[5])) |
|
||||||
if self.__proxy[0] == PROXY_TYPE_SOCKS5: |
|
||||||
if self.__proxy[2] != None: |
|
||||||
portnum = self.__proxy[2] |
|
||||||
else: |
|
||||||
portnum = 1080 |
|
||||||
_orgsocket.connect(self,(self.__proxy[1],portnum)) |
|
||||||
self.__negotiatesocks5(destpair[0],destpair[1]) |
|
||||||
elif self.__proxy[0] == PROXY_TYPE_SOCKS4: |
|
||||||
if self.__proxy[2] != None: |
|
||||||
portnum = self.__proxy[2] |
|
||||||
else: |
|
||||||
portnum = 1080 |
|
||||||
_orgsocket.connect(self,(self.__proxy[1],portnum)) |
|
||||||
self.__negotiatesocks4(destpair[0],destpair[1]) |
|
||||||
elif self.__proxy[0] == PROXY_TYPE_HTTP: |
|
||||||
if self.__proxy[2] != None: |
|
||||||
portnum = self.__proxy[2] |
|
||||||
else: |
|
||||||
portnum = 8080 |
|
||||||
_orgsocket.connect(self,(self.__proxy[1],portnum)) |
|
||||||
self.__negotiatehttp(destpair[0],destpair[1]) |
|
||||||
elif self.__proxy[0] == None: |
|
||||||
_orgsocket.connect(self,(destpair[0],destpair[1])) |
|
||||||
else: |
|
||||||
raise GeneralProxyError((4,_generalerrors[4])) |
|
||||||
Loading…
Reference in new issue