sledgehammer999
7 years ago
11 changed files with 1231 additions and 22 deletions
@ -1,10 +1,16 @@ |
|||||||
<RCC> |
<RCC> |
||||||
<qresource prefix="/"> |
<qresource prefix="/"> |
||||||
|
<file>searchengine/nova/fix_encoding.py</file> |
||||||
|
<file>searchengine/nova/helpers.py</file> |
||||||
|
<file>searchengine/nova/nova2.py</file> |
||||||
|
<file>searchengine/nova/novaprinter.py</file> |
||||||
|
<file>searchengine/nova/socks.py</file> |
||||||
<file>searchengine/nova3/helpers.py</file> |
<file>searchengine/nova3/helpers.py</file> |
||||||
<file>searchengine/nova3/nova2.py</file> |
<file>searchengine/nova3/nova2.py</file> |
||||||
<file>searchengine/nova3/novaprinter.py</file> |
<file>searchengine/nova3/novaprinter.py</file> |
||||||
<file>searchengine/nova3/sgmllib3.py</file> |
<file>searchengine/nova3/sgmllib3.py</file> |
||||||
<file>searchengine/nova3/socks.py</file> |
<file>searchengine/nova3/socks.py</file> |
||||||
|
<file>searchengine/nova/nova2dl.py</file> |
||||||
<file>searchengine/nova3/nova2dl.py</file> |
<file>searchengine/nova3/nova2dl.py</file> |
||||||
</qresource> |
</qresource> |
||||||
</RCC> |
</RCC> |
||||||
|
@ -0,0 +1,371 @@ |
|||||||
|
# Copyright (c) 2011 The Chromium Authors. All rights reserved. |
||||||
|
# Use of this source code is governed by a BSD-style license that can be |
||||||
|
# found in the LICENSE file. |
||||||
|
|
||||||
|
"""Collection of functions and classes to fix various encoding problems on |
||||||
|
multiple platforms with python. |
||||||
|
""" |
||||||
|
|
||||||
|
import codecs |
||||||
|
import locale |
||||||
|
import os |
||||||
|
import sys |
||||||
|
|
||||||
|
|
||||||
|
# Prevents initializing multiple times. |
||||||
|
_SYS_ARGV_PROCESSED = False |
||||||
|
|
||||||
|
|
||||||
|
def complain(message): |
||||||
|
"""If any exception occurs in this file, we'll probably try to print it |
||||||
|
on stderr, which makes for frustrating debugging if stderr is directed |
||||||
|
to our wrapper. So be paranoid about catching errors and reporting them |
||||||
|
to sys.__stderr__, so that the user has a higher chance to see them. |
||||||
|
""" |
||||||
|
print >> sys.__stderr__, ( |
||||||
|
isinstance(message, str) and message or repr(message)) |
||||||
|
|
||||||
|
|
||||||
|
def fix_default_encoding(): |
||||||
|
"""Forces utf8 solidly on all platforms. |
||||||
|
|
||||||
|
By default python execution environment is lazy and defaults to ascii |
||||||
|
encoding. |
||||||
|
|
||||||
|
http://uucode.com/blog/2007/03/23/shut-up-you-dummy-7-bit-python/ |
||||||
|
""" |
||||||
|
if sys.getdefaultencoding() == 'utf-8': |
||||||
|
return False |
||||||
|
|
||||||
|
# Regenerate setdefaultencoding. |
||||||
|
reload(sys) |
||||||
|
# Module 'sys' has no 'setdefaultencoding' member |
||||||
|
# pylint: disable=E1101 |
||||||
|
sys.setdefaultencoding('utf-8') |
||||||
|
for attr in dir(locale): |
||||||
|
if attr[0:3] != 'LC_': |
||||||
|
continue |
||||||
|
aref = getattr(locale, attr) |
||||||
|
try: |
||||||
|
locale.setlocale(aref, '') |
||||||
|
except locale.Error: |
||||||
|
continue |
||||||
|
try: |
||||||
|
lang = locale.getlocale(aref)[0] |
||||||
|
except (TypeError, ValueError): |
||||||
|
continue |
||||||
|
if lang: |
||||||
|
try: |
||||||
|
locale.setlocale(aref, (lang, 'UTF-8')) |
||||||
|
except locale.Error: |
||||||
|
os.environ[attr] = lang + '.UTF-8' |
||||||
|
try: |
||||||
|
locale.setlocale(locale.LC_ALL, '') |
||||||
|
except locale.Error: |
||||||
|
pass |
||||||
|
return True |
||||||
|
|
||||||
|
|
||||||
|
############################### |
||||||
|
# Windows specific |
||||||
|
|
||||||
|
|
||||||
|
def fix_win_sys_argv(encoding): |
||||||
|
"""Converts sys.argv to 'encoding' encoded string. |
||||||
|
|
||||||
|
utf-8 is recommended. |
||||||
|
|
||||||
|
Works around <http://bugs.python.org/issue2128>. |
||||||
|
""" |
||||||
|
global _SYS_ARGV_PROCESSED |
||||||
|
if _SYS_ARGV_PROCESSED: |
||||||
|
return False |
||||||
|
|
||||||
|
# These types are available on linux but not Mac. |
||||||
|
# pylint: disable=E0611,F0401 |
||||||
|
from ctypes import byref, c_int, POINTER, windll, WINFUNCTYPE |
||||||
|
from ctypes.wintypes import LPCWSTR, LPWSTR |
||||||
|
|
||||||
|
# <http://msdn.microsoft.com/en-us/library/ms683156.aspx> |
||||||
|
GetCommandLineW = WINFUNCTYPE(LPWSTR)(('GetCommandLineW', windll.kernel32)) |
||||||
|
# <http://msdn.microsoft.com/en-us/library/bb776391.aspx> |
||||||
|
CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( |
||||||
|
('CommandLineToArgvW', windll.shell32)) |
||||||
|
|
||||||
|
argc = c_int(0) |
||||||
|
argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) |
||||||
|
argv = [ |
||||||
|
argv_unicode[i].encode(encoding, 'replace') |
||||||
|
for i in xrange(0, argc.value)] |
||||||
|
|
||||||
|
if not hasattr(sys, 'frozen'): |
||||||
|
# If this is an executable produced by py2exe or bbfreeze, then it |
||||||
|
# will have been invoked directly. Otherwise, unicode_argv[0] is the |
||||||
|
# Python interpreter, so skip that. |
||||||
|
argv = argv[1:] |
||||||
|
|
||||||
|
# Also skip option arguments to the Python interpreter. |
||||||
|
while len(argv) > 0: |
||||||
|
arg = argv[0] |
||||||
|
if not arg.startswith(u'-') or arg == u'-': |
||||||
|
break |
||||||
|
argv = argv[1:] |
||||||
|
if arg == u'-m': |
||||||
|
# sys.argv[0] should really be the absolute path of the |
||||||
|
# module source, but never mind. |
||||||
|
break |
||||||
|
if arg == u'-c': |
||||||
|
argv[0] = u'-c' |
||||||
|
break |
||||||
|
sys.argv = argv |
||||||
|
_SYS_ARGV_PROCESSED = True |
||||||
|
return True |
||||||
|
|
||||||
|
|
||||||
|
def fix_win_codec(): |
||||||
|
"""Works around <http://bugs.python.org/issue6058>.""" |
||||||
|
# <http://msdn.microsoft.com/en-us/library/dd317756.aspx> |
||||||
|
try: |
||||||
|
codecs.lookup('cp65001') |
||||||
|
return False |
||||||
|
except LookupError: |
||||||
|
codecs.register( |
||||||
|
lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) |
||||||
|
return True |
||||||
|
|
||||||
|
|
||||||
|
class WinUnicodeOutputBase(object): |
||||||
|
"""Base class to adapt sys.stdout or sys.stderr to behave correctly on |
||||||
|
Windows. |
||||||
|
|
||||||
|
Setting encoding to utf-8 is recommended. |
||||||
|
""" |
||||||
|
def __init__(self, fileno, name, encoding): |
||||||
|
# Corresponding file handle. |
||||||
|
self._fileno = fileno |
||||||
|
self.encoding = encoding |
||||||
|
self.name = name |
||||||
|
|
||||||
|
self.closed = False |
||||||
|
self.softspace = False |
||||||
|
self.mode = 'w' |
||||||
|
|
||||||
|
@staticmethod |
||||||
|
def isatty(): |
||||||
|
return False |
||||||
|
|
||||||
|
def close(self): |
||||||
|
# Don't really close the handle, that would only cause problems. |
||||||
|
self.closed = True |
||||||
|
|
||||||
|
def fileno(self): |
||||||
|
return self._fileno |
||||||
|
|
||||||
|
def flush(self): |
||||||
|
raise NotImplementedError() |
||||||
|
|
||||||
|
def write(self, text): |
||||||
|
raise NotImplementedError() |
||||||
|
|
||||||
|
def writelines(self, lines): |
||||||
|
try: |
||||||
|
for line in lines: |
||||||
|
self.write(line) |
||||||
|
except Exception, e: |
||||||
|
complain('%s.writelines: %r' % (self.name, e)) |
||||||
|
raise |
||||||
|
|
||||||
|
|
||||||
|
class WinUnicodeConsoleOutput(WinUnicodeOutputBase): |
||||||
|
"""Output adapter to a Windows Console. |
||||||
|
|
||||||
|
Understands how to use the win32 console API. |
||||||
|
""" |
||||||
|
def __init__(self, console_handle, fileno, stream_name, encoding): |
||||||
|
super(WinUnicodeConsoleOutput, self).__init__( |
||||||
|
fileno, '<Unicode console %s>' % stream_name, encoding) |
||||||
|
# Handle to use for WriteConsoleW |
||||||
|
self._console_handle = console_handle |
||||||
|
|
||||||
|
# Loads the necessary function. |
||||||
|
# These types are available on linux but not Mac. |
||||||
|
# pylint: disable=E0611,F0401 |
||||||
|
from ctypes import byref, GetLastError, POINTER, windll, WINFUNCTYPE |
||||||
|
from ctypes.wintypes import BOOL, DWORD, HANDLE, LPWSTR |
||||||
|
from ctypes.wintypes import LPVOID # pylint: disable=E0611 |
||||||
|
|
||||||
|
self._DWORD = DWORD |
||||||
|
self._byref = byref |
||||||
|
|
||||||
|
# <http://msdn.microsoft.com/en-us/library/ms687401.aspx> |
||||||
|
self._WriteConsoleW = WINFUNCTYPE( |
||||||
|
BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)( |
||||||
|
('WriteConsoleW', windll.kernel32)) |
||||||
|
self._GetLastError = GetLastError |
||||||
|
|
||||||
|
def flush(self): |
||||||
|
# No need to flush the console since it's immediate. |
||||||
|
pass |
||||||
|
|
||||||
|
def write(self, text): |
||||||
|
try: |
||||||
|
if not isinstance(text, unicode): |
||||||
|
# Convert to unicode. |
||||||
|
text = str(text).decode(self.encoding, 'replace') |
||||||
|
remaining = len(text) |
||||||
|
while remaining > 0: |
||||||
|
n = self._DWORD(0) |
||||||
|
# There is a shorter-than-documented limitation on the length of the |
||||||
|
# string passed to WriteConsoleW. See |
||||||
|
# <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>. |
||||||
|
retval = self._WriteConsoleW( |
||||||
|
self._console_handle, text, |
||||||
|
min(remaining, 10000), |
||||||
|
self._byref(n), None) |
||||||
|
if retval == 0 or n.value == 0: |
||||||
|
raise IOError( |
||||||
|
'WriteConsoleW returned %r, n.value = %r, last error = %r' % ( |
||||||
|
retval, n.value, self._GetLastError())) |
||||||
|
remaining -= n.value |
||||||
|
if not remaining: |
||||||
|
break |
||||||
|
text = text[n.value:] |
||||||
|
except Exception, e: |
||||||
|
complain('%s.write: %r' % (self.name, e)) |
||||||
|
raise |
||||||
|
|
||||||
|
|
||||||
|
class WinUnicodeOutput(WinUnicodeOutputBase): |
||||||
|
"""Output adaptor to a file output on Windows. |
||||||
|
|
||||||
|
If the standard FileWrite function is used, it will be encoded in the current |
||||||
|
code page. WriteConsoleW() permits writting any character. |
||||||
|
""" |
||||||
|
def __init__(self, stream, fileno, encoding): |
||||||
|
super(WinUnicodeOutput, self).__init__( |
||||||
|
fileno, '<Unicode redirected %s>' % stream.name, encoding) |
||||||
|
# Output stream |
||||||
|
self._stream = stream |
||||||
|
|
||||||
|
# Flush right now. |
||||||
|
self.flush() |
||||||
|
|
||||||
|
def flush(self): |
||||||
|
try: |
||||||
|
self._stream.flush() |
||||||
|
except Exception, e: |
||||||
|
complain('%s.flush: %r from %r' % (self.name, e, self._stream)) |
||||||
|
raise |
||||||
|
|
||||||
|
def write(self, text): |
||||||
|
try: |
||||||
|
if isinstance(text, unicode): |
||||||
|
# Replace characters that cannot be printed instead of failing. |
||||||
|
text = text.encode(self.encoding, 'replace') |
||||||
|
self._stream.write(text) |
||||||
|
except Exception, e: |
||||||
|
complain('%s.write: %r' % (self.name, e)) |
||||||
|
raise |
||||||
|
|
||||||
|
|
||||||
|
def win_handle_is_a_console(handle): |
||||||
|
"""Returns True if a Windows file handle is a handle to a console.""" |
||||||
|
# These types are available on linux but not Mac. |
||||||
|
# pylint: disable=E0611,F0401 |
||||||
|
from ctypes import byref, POINTER, windll, WINFUNCTYPE |
||||||
|
from ctypes.wintypes import BOOL, DWORD, HANDLE |
||||||
|
|
||||||
|
FILE_TYPE_CHAR = 0x0002 |
||||||
|
FILE_TYPE_REMOTE = 0x8000 |
||||||
|
INVALID_HANDLE_VALUE = DWORD(-1).value |
||||||
|
|
||||||
|
# <http://msdn.microsoft.com/en-us/library/ms683167.aspx> |
||||||
|
GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( |
||||||
|
('GetConsoleMode', windll.kernel32)) |
||||||
|
# <http://msdn.microsoft.com/en-us/library/aa364960.aspx> |
||||||
|
GetFileType = WINFUNCTYPE(DWORD, DWORD)(('GetFileType', windll.kernel32)) |
||||||
|
|
||||||
|
# GetStdHandle returns INVALID_HANDLE_VALUE, NULL, or a valid handle. |
||||||
|
if handle == INVALID_HANDLE_VALUE or handle is None: |
||||||
|
return False |
||||||
|
return ( |
||||||
|
(GetFileType(handle) & ~FILE_TYPE_REMOTE) == FILE_TYPE_CHAR and |
||||||
|
GetConsoleMode(handle, byref(DWORD()))) |
||||||
|
|
||||||
|
|
||||||
|
def win_get_unicode_stream(stream, excepted_fileno, output_handle, encoding): |
||||||
|
"""Returns a unicode-compatible stream. |
||||||
|
|
||||||
|
This function will return a direct-Console writing object only if: |
||||||
|
- the file number is the expected console file number |
||||||
|
- the handle the expected file handle |
||||||
|
- the 'real' handle is in fact a handle to a console. |
||||||
|
""" |
||||||
|
old_fileno = getattr(stream, 'fileno', lambda: None)() |
||||||
|
if old_fileno == excepted_fileno: |
||||||
|
# These types are available on linux but not Mac. |
||||||
|
# pylint: disable=E0611,F0401 |
||||||
|
from ctypes import windll, WINFUNCTYPE |
||||||
|
from ctypes.wintypes import DWORD, HANDLE |
||||||
|
|
||||||
|
# <http://msdn.microsoft.com/en-us/library/ms683231.aspx> |
||||||
|
GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(('GetStdHandle', windll.kernel32)) |
||||||
|
|
||||||
|
real_output_handle = GetStdHandle(DWORD(output_handle)) |
||||||
|
if win_handle_is_a_console(real_output_handle): |
||||||
|
# It's a console. |
||||||
|
return WinUnicodeConsoleOutput( |
||||||
|
real_output_handle, old_fileno, stream.name, encoding) |
||||||
|
|
||||||
|
# It's something else. Create an auto-encoding stream. |
||||||
|
return WinUnicodeOutput(stream, old_fileno, encoding) |
||||||
|
|
||||||
|
|
||||||
|
def fix_win_console(encoding): |
||||||
|
"""Makes Unicode console output work independently of the current code page. |
||||||
|
|
||||||
|
This also fixes <http://bugs.python.org/issue1602>. |
||||||
|
Credit to Michael Kaplan |
||||||
|
<http://blogs.msdn.com/b/michkap/archive/2010/04/07/9989346.aspx> and |
||||||
|
TZOmegaTZIOY |
||||||
|
<http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>. |
||||||
|
""" |
||||||
|
if (isinstance(sys.stdout, WinUnicodeOutputBase) or |
||||||
|
isinstance(sys.stderr, WinUnicodeOutputBase)): |
||||||
|
return False |
||||||
|
|
||||||
|
try: |
||||||
|
# SetConsoleCP and SetConsoleOutputCP could be used to change the code page |
||||||
|
# but it's not really useful since the code here is using WriteConsoleW(). |
||||||
|
# Also, changing the code page is 'permanent' to the console and needs to be |
||||||
|
# reverted manually. |
||||||
|
# In practice one needs to set the console font to a TTF font to be able to |
||||||
|
# see all the characters but it failed for me in practice. In any case, it |
||||||
|
# won't throw any exception when printing, which is the important part. |
||||||
|
# -11 and -12 are defined in stdio.h |
||||||
|
sys.stdout = win_get_unicode_stream(sys.stdout, 1, -11, encoding) |
||||||
|
sys.stderr = win_get_unicode_stream(sys.stderr, 2, -12, encoding) |
||||||
|
# TODO(maruel): Do sys.stdin with ReadConsoleW(). Albeit the limitation is |
||||||
|
# "It doesn't appear to be possible to read Unicode characters in UTF-8 |
||||||
|
# mode" and this appears to be a limitation of cmd.exe. |
||||||
|
except Exception, e: |
||||||
|
complain('exception %r while fixing up sys.stdout and sys.stderr' % e) |
||||||
|
return True |
||||||
|
|
||||||
|
|
||||||
|
def fix_encoding(): |
||||||
|
"""Fixes various encoding problems on all platforms. |
||||||
|
|
||||||
|
Should be called at the very begining of the process. |
||||||
|
""" |
||||||
|
ret = True |
||||||
|
if sys.platform == 'win32': |
||||||
|
ret &= fix_win_codec() |
||||||
|
|
||||||
|
ret &= fix_default_encoding() |
||||||
|
|
||||||
|
if sys.platform == 'win32': |
||||||
|
encoding = sys.getdefaultencoding() |
||||||
|
ret &= fix_win_sys_argv(encoding) |
||||||
|
ret &= fix_win_console(encoding) |
||||||
|
return ret |
@ -0,0 +1,115 @@ |
|||||||
|
#VERSION: 1.41 |
||||||
|
|
||||||
|
# Author: |
||||||
|
# Christophe DUMEZ (chris@qbittorrent.org) |
||||||
|
# Contributors: |
||||||
|
# Diego de las Heras (ngosang@hotmail.es) |
||||||
|
|
||||||
|
# Redistribution and use in source and binary forms, with or without |
||||||
|
# modification, are permitted provided that the following conditions are met: |
||||||
|
# |
||||||
|
# * Redistributions of source code must retain the above copyright notice, |
||||||
|
# this list of conditions and the following disclaimer. |
||||||
|
# * Redistributions in binary form must reproduce the above copyright |
||||||
|
# notice, this list of conditions and the following disclaimer in the |
||||||
|
# documentation and/or other materials provided with the distribution. |
||||||
|
# * Neither the name of the author nor the names of its contributors may be |
||||||
|
# used to endorse or promote products derived from this software without |
||||||
|
# specific prior written permission. |
||||||
|
# |
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
||||||
|
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
||||||
|
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
||||||
|
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
||||||
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
||||||
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
||||||
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
||||||
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
||||||
|
# POSSIBILITY OF SUCH DAMAGE. |
||||||
|
|
||||||
|
import re, htmlentitydefs |
||||||
|
import tempfile |
||||||
|
import os |
||||||
|
import StringIO, gzip, urllib2 |
||||||
|
import socket |
||||||
|
import socks |
||||||
|
import re |
||||||
|
|
||||||
|
# Some sites blocks default python User-agent |
||||||
|
user_agent = 'Mozilla/5.0 (X11; Linux i686; rv:38.0) Gecko/20100101 Firefox/38.0' |
||||||
|
headers = {'User-Agent': user_agent} |
||||||
|
# SOCKS5 Proxy support |
||||||
|
if os.environ.has_key("sock_proxy") and len(os.environ["sock_proxy"].strip()) > 0: |
||||||
|
proxy_str = os.environ["sock_proxy"].strip() |
||||||
|
m=re.match(r"^(?:(?P<username>[^:]+):(?P<password>[^@]+)@)?(?P<host>[^:]+):(?P<port>\w+)$", proxy_str) |
||||||
|
if m is not None: |
||||||
|
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, m.group('host'), int(m.group('port')), True, m.group('username'), m.group('password')) |
||||||
|
socket.socket = socks.socksocket |
||||||
|
|
||||||
|
def htmlentitydecode(s): |
||||||
|
# First convert alpha entities (such as é) |
||||||
|
# (Inspired from http://mail.python.org/pipermail/python-list/2007-June/443813.html) |
||||||
|
def entity2char(m): |
||||||
|
entity = m.group(1) |
||||||
|
if entity in htmlentitydefs.name2codepoint: |
||||||
|
return unichr(htmlentitydefs.name2codepoint[entity]) |
||||||
|
return u" " # Unknown entity: We replace with a space. |
||||||
|
t = re.sub(u'&(%s);' % u'|'.join(htmlentitydefs.name2codepoint), entity2char, s) |
||||||
|
|
||||||
|
# Then convert numerical entities (such as é) |
||||||
|
t = re.sub(u'&#(\d+);', lambda x: unichr(int(x.group(1))), t) |
||||||
|
|
||||||
|
# Then convert hexa entities (such as é) |
||||||
|
return re.sub(u'&#x(\w+);', lambda x: unichr(int(x.group(1),16)), t) |
||||||
|
|
||||||
|
def retrieve_url(url): |
||||||
|
""" Return the content of the url page as a string """ |
||||||
|
req = urllib2.Request(url, headers = headers) |
||||||
|
try: |
||||||
|
response = urllib2.urlopen(req) |
||||||
|
except urllib2.URLError as errno: |
||||||
|
print(" ".join(("Connection error:", str(errno.reason)))) |
||||||
|
return "" |
||||||
|
dat = response.read() |
||||||
|
# Check if it is gzipped |
||||||
|
if dat[:2] == '\037\213': |
||||||
|
# Data is gzip encoded, decode it |
||||||
|
compressedstream = StringIO.StringIO(dat) |
||||||
|
gzipper = gzip.GzipFile(fileobj=compressedstream) |
||||||
|
extracted_data = gzipper.read() |
||||||
|
dat = extracted_data |
||||||
|
info = response.info() |
||||||
|
charset = 'utf-8' |
||||||
|
try: |
||||||
|
ignore, charset = info['Content-Type'].split('charset=') |
||||||
|
except: |
||||||
|
pass |
||||||
|
dat = dat.decode(charset, 'replace') |
||||||
|
dat = htmlentitydecode(dat) |
||||||
|
return dat |
||||||
|
|
||||||
|
def download_file(url, referer=None): |
||||||
|
""" Download file at url and write it to a file, return the path to the file and the url """ |
||||||
|
file, path = tempfile.mkstemp() |
||||||
|
file = os.fdopen(file, "w") |
||||||
|
# Download url |
||||||
|
req = urllib2.Request(url, headers = headers) |
||||||
|
if referer is not None: |
||||||
|
req.add_header('referer', referer) |
||||||
|
response = urllib2.urlopen(req) |
||||||
|
dat = response.read() |
||||||
|
# Check if it is gzipped |
||||||
|
if dat[:2] == '\037\213': |
||||||
|
# Data is gzip encoded, decode it |
||||||
|
compressedstream = StringIO.StringIO(dat) |
||||||
|
gzipper = gzip.GzipFile(fileobj=compressedstream) |
||||||
|
extracted_data = gzipper.read() |
||||||
|
dat = extracted_data |
||||||
|
|
||||||
|
# Write it to a file |
||||||
|
file.write(dat) |
||||||
|
file.close() |
||||||
|
# return file path |
||||||
|
return path+" "+url |
@ -0,0 +1,184 @@ |
|||||||
|
#VERSION: 1.41 |
||||||
|
|
||||||
|
# Author: |
||||||
|
# Fabien Devaux <fab AT gnux DOT info> |
||||||
|
# Contributors: |
||||||
|
# Christophe Dumez <chris@qbittorrent.org> (qbittorrent integration) |
||||||
|
# Thanks to gab #gcu @ irc.freenode.net (multipage support on PirateBay) |
||||||
|
# Thanks to Elias <gekko04@users.sourceforge.net> (torrentreactor and isohunt search engines) |
||||||
|
# |
||||||
|
# Licence: BSD |
||||||
|
|
||||||
|
# Redistribution and use in source and binary forms, with or without |
||||||
|
# modification, are permitted provided that the following conditions are met: |
||||||
|
# |
||||||
|
# * Redistributions of source code must retain the above copyright notice, |
||||||
|
# this list of conditions and the following disclaimer. |
||||||
|
# * Redistributions in binary form must reproduce the above copyright |
||||||
|
# notice, this list of conditions and the following disclaimer in the |
||||||
|
# documentation and/or other materials provided with the distribution. |
||||||
|
# * Neither the name of the author nor the names of its contributors may be |
||||||
|
# used to endorse or promote products derived from this software without |
||||||
|
# specific prior written permission. |
||||||
|
# |
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
||||||
|
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
||||||
|
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
||||||
|
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
||||||
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
||||||
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
||||||
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
||||||
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
||||||
|
# POSSIBILITY OF SUCH DAMAGE. |
||||||
|
|
||||||
|
import urllib |
||||||
|
from os import path |
||||||
|
from glob import glob |
||||||
|
from sys import argv |
||||||
|
from multiprocessing import Pool, cpu_count |
||||||
|
from fix_encoding import fix_encoding |
||||||
|
|
||||||
|
THREADED = True |
||||||
|
try: |
||||||
|
MAX_THREADS = cpu_count() |
||||||
|
except NotImplementedError: |
||||||
|
MAX_THREADS = 1 |
||||||
|
|
||||||
|
CATEGORIES = {'all', 'movies', 'tv', 'music', 'games', 'anime', 'software', 'pictures', 'books'} |
||||||
|
|
||||||
|
################################################################################ |
||||||
|
# Every engine should have a "search" method taking |
||||||
|
# a space-free string as parameter (ex. "family+guy") |
||||||
|
# it should call prettyPrinter() with a dict as parameter. |
||||||
|
# The keys in the dict must be: link,name,size,seeds,leech,engine_url |
||||||
|
# As a convention, try to list results by decreasing number of seeds or similar |
||||||
|
################################################################################ |
||||||
|
|
||||||
|
def initialize_engines(): |
||||||
|
""" Import available engines |
||||||
|
|
||||||
|
Return list of available engines |
||||||
|
""" |
||||||
|
supported_engines = [] |
||||||
|
|
||||||
|
engines = glob(path.join(path.dirname(__file__), 'engines', '*.py')) |
||||||
|
for engine in engines: |
||||||
|
engi = path.basename(engine).split('.')[0].strip() |
||||||
|
if len(engi) == 0 or engi.startswith('_'): |
||||||
|
continue |
||||||
|
try: |
||||||
|
#import engines.[engine] |
||||||
|
engine_module = __import__(".".join(("engines", engi))) |
||||||
|
#get low-level module |
||||||
|
engine_module = getattr(engine_module, engi) |
||||||
|
#bind class name |
||||||
|
globals()[engi] = getattr(engine_module, engi) |
||||||
|
supported_engines.append(engi) |
||||||
|
except: |
||||||
|
pass |
||||||
|
|
||||||
|
return supported_engines |
||||||
|
|
||||||
|
def engines_to_xml(supported_engines): |
||||||
|
""" Generates xml for supported engines """ |
||||||
|
tab = " " * 4 |
||||||
|
|
||||||
|
for short_name in supported_engines: |
||||||
|
search_engine = globals()[short_name]() |
||||||
|
|
||||||
|
supported_categories = "" |
||||||
|
if hasattr(search_engine, "supported_categories"): |
||||||
|
supported_categories = " ".join((key for key in search_engine.supported_categories.keys() |
||||||
|
if key is not "all")) |
||||||
|
|
||||||
|
yield "".join((tab, "<", short_name, ">\n", |
||||||
|
tab, tab, "<name>", search_engine.name, "</name>\n", |
||||||
|
tab, tab, "<url>", search_engine.url, "</url>\n", |
||||||
|
tab, tab, "<categories>", supported_categories, "</categories>\n", |
||||||
|
tab, "</", short_name, ">\n")) |
||||||
|
|
||||||
|
def displayCapabilities(supported_engines): |
||||||
|
""" |
||||||
|
Display capabilities in XML format |
||||||
|
<capabilities> |
||||||
|
<engine_short_name> |
||||||
|
<name>long name</name> |
||||||
|
<url>http://example.com</url> |
||||||
|
<categories>movies music games</categories> |
||||||
|
</engine_short_name> |
||||||
|
</capabilities> |
||||||
|
""" |
||||||
|
xml = "".join(("<capabilities>\n", |
||||||
|
"".join(engines_to_xml(supported_engines)), |
||||||
|
"</capabilities>")) |
||||||
|
print(xml) |
||||||
|
|
||||||
|
def run_search(engine_list): |
||||||
|
""" Run search in engine |
||||||
|
|
||||||
|
@param engine_list List with engine, query and category |
||||||
|
|
||||||
|
@retval False if any exceptions occurred |
||||||
|
@retval True otherwise |
||||||
|
""" |
||||||
|
engine, what, cat = engine_list |
||||||
|
try: |
||||||
|
engine = engine() |
||||||
|
#avoid exceptions due to invalid category |
||||||
|
if hasattr(engine, 'supported_categories'): |
||||||
|
cat = cat if cat in engine.supported_categories else "all" |
||||||
|
engine.search(what, cat) |
||||||
|
else: |
||||||
|
engine.search(what) |
||||||
|
return True |
||||||
|
except: |
||||||
|
return False |
||||||
|
|
||||||
|
def main(args): |
||||||
|
fix_encoding() |
||||||
|
supported_engines = initialize_engines() |
||||||
|
|
||||||
|
if not args: |
||||||
|
raise SystemExit("./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n" |
||||||
|
"available engines: %s" % (','.join(supported_engines))) |
||||||
|
|
||||||
|
elif args[0] == "--capabilities": |
||||||
|
displayCapabilities(supported_engines) |
||||||
|
return |
||||||
|
|
||||||
|
elif len(args) < 3: |
||||||
|
raise SystemExit("./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n" |
||||||
|
"available engines: %s" % (','.join(supported_engines))) |
||||||
|
|
||||||
|
#get only unique engines with set |
||||||
|
engines_list = set(e.lower() for e in args[0].strip().split(',')) |
||||||
|
|
||||||
|
if 'all' in engines_list: |
||||||
|
engines_list = supported_engines |
||||||
|
else: |
||||||
|
#discard un-supported engines |
||||||
|
engines_list = [engine for engine in engines_list |
||||||
|
if engine in supported_engines] |
||||||
|
|
||||||
|
if not engines_list: |
||||||
|
#engine list is empty. Nothing to do here |
||||||
|
return |
||||||
|
|
||||||
|
cat = args[1].lower() |
||||||
|
|
||||||
|
if cat not in CATEGORIES: |
||||||
|
raise SystemExit(" - ".join(('Invalid category', cat))) |
||||||
|
|
||||||
|
what = urllib.quote(' '.join(args[2:])) |
||||||
|
|
||||||
|
if THREADED: |
||||||
|
#child process spawning is controlled min(number of searches, number of cpu) |
||||||
|
pool = Pool(min(len(engines_list), MAX_THREADS)) |
||||||
|
pool.map(run_search, ([globals()[engine], what, cat] for engine in engines_list)) |
||||||
|
else: |
||||||
|
map(run_search, ([globals()[engine], what, cat] for engine in engines_list)) |
||||||
|
|
||||||
|
if __name__ == "__main__": |
||||||
|
main(argv[1:]) |
@ -0,0 +1,61 @@ |
|||||||
|
#VERSION: 1.20 |
||||||
|
|
||||||
|
# Author: |
||||||
|
# Christophe DUMEZ (chris@qbittorrent.org) |
||||||
|
|
||||||
|
# Redistribution and use in source and binary forms, with or without |
||||||
|
# modification, are permitted provided that the following conditions are met: |
||||||
|
# |
||||||
|
# * Redistributions of source code must retain the above copyright notice, |
||||||
|
# this list of conditions and the following disclaimer. |
||||||
|
# * Redistributions in binary form must reproduce the above copyright |
||||||
|
# notice, this list of conditions and the following disclaimer in the |
||||||
|
# documentation and/or other materials provided with the distribution. |
||||||
|
# * Neither the name of the author nor the names of its contributors may be |
||||||
|
# used to endorse or promote products derived from this software without |
||||||
|
# specific prior written permission. |
||||||
|
# |
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
||||||
|
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
||||||
|
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
||||||
|
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
||||||
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
||||||
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
||||||
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
||||||
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
||||||
|
# POSSIBILITY OF SUCH DAMAGE. |
||||||
|
|
||||||
|
import sys |
||||||
|
import os |
||||||
|
import glob |
||||||
|
from helpers import download_file |
||||||
|
|
||||||
|
supported_engines = dict() |
||||||
|
|
||||||
|
engines = glob.glob(os.path.join(os.path.dirname(__file__), 'engines','*.py')) |
||||||
|
for engine in engines: |
||||||
|
e = engine.split(os.sep)[-1][:-3] |
||||||
|
if len(e.strip()) == 0: continue |
||||||
|
if e.startswith('_'): continue |
||||||
|
try: |
||||||
|
exec("from engines.%s import %s"%(e,e)) |
||||||
|
exec("engine_url = %s.url"%e) |
||||||
|
supported_engines[engine_url] = e |
||||||
|
except: |
||||||
|
pass |
||||||
|
|
||||||
|
if __name__ == '__main__': |
||||||
|
if len(sys.argv) < 3: |
||||||
|
raise SystemExit('./nova2dl.py engine_url download_parameter') |
||||||
|
engine_url = sys.argv[1].strip() |
||||||
|
download_param = sys.argv[2].strip() |
||||||
|
if engine_url not in list(supported_engines.keys()): |
||||||
|
raise SystemExit('./nova2dl.py: this engine_url was not recognized') |
||||||
|
exec("engine = %s()"%supported_engines[engine_url]) |
||||||
|
if hasattr(engine, 'download_torrent'): |
||||||
|
engine.download_torrent(download_param) |
||||||
|
else: |
||||||
|
print(download_file(download_param)) |
||||||
|
sys.exit(0) |
@ -0,0 +1,68 @@ |
|||||||
|
#VERSION: 1.45 |
||||||
|
|
||||||
|
# Redistribution and use in source and binary forms, with or without |
||||||
|
# modification, are permitted provided that the following conditions are met: |
||||||
|
# |
||||||
|
# * Redistributions of source code must retain the above copyright notice, |
||||||
|
# this list of conditions and the following disclaimer. |
||||||
|
# * Redistributions in binary form must reproduce the above copyright |
||||||
|
# notice, this list of conditions and the following disclaimer in the |
||||||
|
# documentation and/or other materials provided with the distribution. |
||||||
|
# * Neither the name of the author nor the names of its contributors may be |
||||||
|
# used to endorse or promote products derived from this software without |
||||||
|
# specific prior written permission. |
||||||
|
# |
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
||||||
|
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
||||||
|
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
||||||
|
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
||||||
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
||||||
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
||||||
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
||||||
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
||||||
|
# POSSIBILITY OF SUCH DAMAGE. |
||||||
|
|
||||||
|
import sys, codecs |
||||||
|
from io import open |
||||||
|
|
||||||
|
# Force UTF-8 printing |
||||||
|
sys.stdout = codecs.getwriter('utf-8')(sys.stdout) |
||||||
|
|
||||||
|
def prettyPrinter(dictionary): |
||||||
|
dictionary['size'] = anySizeToBytes(dictionary['size']) |
||||||
|
outtext = "|".join((dictionary["link"], dictionary["name"].replace("|", " "), str(dictionary["size"]), str(dictionary["seeds"]), str(dictionary["leech"]), dictionary["engine_url"])) |
||||||
|
if 'desc_link' in dictionary: |
||||||
|
outtext = "|".join((outtext, dictionary["desc_link"])) |
||||||
|
|
||||||
|
with open(1, 'w', encoding='utf-8', closefd=False) as utf8_stdout: |
||||||
|
utf8_stdout.write(unicode("".join((outtext, "\n")))) |
||||||
|
|
||||||
|
def anySizeToBytes(size_string): |
||||||
|
""" |
||||||
|
Convert a string like '1 KB' to '1024' (bytes) |
||||||
|
""" |
||||||
|
# separate integer from unit |
||||||
|
try: |
||||||
|
size, unit = size_string.split() |
||||||
|
except: |
||||||
|
try: |
||||||
|
size = size_string.strip() |
||||||
|
unit = ''.join([c for c in size if c.isalpha()]) |
||||||
|
if len(unit) > 0: |
||||||
|
size = size[:-len(unit)] |
||||||
|
except: |
||||||
|
return -1 |
||||||
|
if len(size) == 0: |
||||||
|
return -1 |
||||||
|
size = float(size) |
||||||
|
if len(unit) == 0: |
||||||
|
return int(size) |
||||||
|
short_unit = unit.upper()[0] |
||||||
|
|
||||||
|
# convert |
||||||
|
units_dict = {'T': 40, 'G': 30, 'M': 20, 'K': 10} |
||||||
|
if units_dict.has_key(short_unit): |
||||||
|
size = size * 2**units_dict[short_unit] |
||||||
|
return int(size) |
@ -0,0 +1,391 @@ |
|||||||
|
"""SocksiPy - Python SOCKS module. |
||||||
|
Version 1.01 |
||||||
|
|
||||||
|
Copyright 2006 Dan-Haim. All rights reserved. |
||||||
|
Various fixes by Christophe DUMEZ <chris@qbittorrent.org> - 2010 |
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without modification, |
||||||
|
are permitted provided that the following conditions are met: |
||||||
|
1. Redistributions of source code must retain the above copyright notice, this |
||||||
|
list of conditions and the following disclaimer. |
||||||
|
2. Redistributions in binary form must reproduce the above copyright notice, |
||||||
|
this list of conditions and the following disclaimer in the documentation |
||||||
|
and/or other materials provided with the distribution. |
||||||
|
3. Neither the name of Dan Haim nor the names of his contributors may be used |
||||||
|
to endorse or promote products derived from this software without specific |
||||||
|
prior written permission. |
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED |
||||||
|
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
||||||
|
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
||||||
|
EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
||||||
|
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA |
||||||
|
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
||||||
|
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT |
||||||
|
OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. |
||||||
|
|
||||||
|
|
||||||
|
This module provides a standard socket-like interface for Python |
||||||
|
for tunneling connections through SOCKS proxies. |
||||||
|
|
||||||
|
""" |
||||||
|
|
||||||
|
import socket |
||||||
|
import struct |
||||||
|
|
||||||
|
PROXY_TYPE_SOCKS4 = 1 |
||||||
|
PROXY_TYPE_SOCKS5 = 2 |
||||||
|
PROXY_TYPE_HTTP = 3 |
||||||
|
|
||||||
|
_defaultproxy = None |
||||||
|
_orgsocket = socket.socket |
||||||
|
|
||||||
|
class ProxyError(Exception): |
||||||
|
def __init__(self, value): |
||||||
|
self.value = value |
||||||
|
def __str__(self): |
||||||
|
return repr(self.value) |
||||||
|
|
||||||
|
class GeneralProxyError(ProxyError): |
||||||
|
def __init__(self, value): |
||||||
|
self.value = value |
||||||
|
def __str__(self): |
||||||
|
return repr(self.value) |
||||||
|
|
||||||
|
class Socks5AuthError(ProxyError): |
||||||
|
def __init__(self, value): |
||||||
|
self.value = value |
||||||
|
def __str__(self): |
||||||
|
return repr(self.value) |
||||||
|
|
||||||
|
class Socks5Error(ProxyError): |
||||||
|
def __init__(self, value): |
||||||
|
self.value = value |
||||||
|
def __str__(self): |
||||||
|
return repr(self.value) |
||||||
|
|
||||||
|
class Socks4Error(ProxyError): |
||||||
|
def __init__(self, value): |
||||||
|
self.value = value |
||||||
|
def __str__(self): |
||||||
|
return repr(self.value) |
||||||
|
|
||||||
|
class HTTPError(ProxyError): |
||||||
|
def __init__(self, value): |
||||||
|
self.value = value |
||||||
|
def __str__(self): |
||||||
|
return repr(self.value) |
||||||
|
|
||||||
|
_generalerrors = ("success", |
||||||
|
"invalid data", |
||||||
|
"not connected", |
||||||
|
"not available", |
||||||
|
"bad proxy type", |
||||||
|
"bad input") |
||||||
|
|
||||||
|
_socks5errors = ("succeeded", |
||||||
|
"general SOCKS server failure", |
||||||
|
"connection not allowed by ruleset", |
||||||
|
"Network unreachable", |
||||||
|
"Host unreachable", |
||||||
|
"Connection refused", |
||||||
|
"TTL expired", |
||||||
|
"Command not supported", |
||||||
|
"Address type not supported", |
||||||
|
"Unknown error") |
||||||
|
|
||||||
|
_socks5autherrors = ("succeeded", |
||||||
|
"authentication is required", |
||||||
|
"all offered authentication methods were rejected", |
||||||
|
"unknown username or invalid password", |
||||||
|
"unknown error") |
||||||
|
|
||||||
|
_socks4errors = ("request granted", |
||||||
|
"request rejected or failed", |
||||||
|
"request rejected because SOCKS server cannot connect to identd on the client", |
||||||
|
"request rejected because the client program and identd report different user-ids", |
||||||
|
"unknown error") |
||||||
|
|
||||||
|
def setdefaultproxy(proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): |
||||||
|
"""setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) |
||||||
|
Sets a default proxy which all further socksocket objects will use, |
||||||
|
unless explicitly changed. |
||||||
|
""" |
||||||
|
global _defaultproxy |
||||||
|
_defaultproxy = (proxytype,addr,port,rdns,username,password) |
||||||
|
|
||||||
|
class socksocket(socket.socket): |
||||||
|
"""socksocket([family[, type[, proto]]]) -> socket object |
||||||
|
|
||||||
|
Open a SOCKS enabled socket. The parameters are the same as |
||||||
|
those of the standard socket init. In order for SOCKS to work, |
||||||
|
you must specify family=AF_INET, type=SOCK_STREAM and proto=0. |
||||||
|
""" |
||||||
|
|
||||||
|
def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): |
||||||
|
_orgsocket.__init__(self,family,type,proto,_sock) |
||||||
|
if _defaultproxy != None: |
||||||
|
self.__proxy = _defaultproxy |
||||||
|
else: |
||||||
|
self.__proxy = (None, None, None, None, None, None) |
||||||
|
self.__proxysockname = None |
||||||
|
self.__proxypeername = None |
||||||
|
|
||||||
|
def __recvall(self, bytes): |
||||||
|
"""__recvall(bytes) -> data |
||||||
|
Receive EXACTLY the number of bytes requested from the socket. |
||||||
|
Blocks until the required number of bytes have been received. |
||||||
|
""" |
||||||
|
data = "" |
||||||
|
while len(data) < bytes: |
||||||
|
d = self.recv(bytes-len(data)) |
||||||
|
if not d: |
||||||
|
raise GeneralProxyError("connection closed unexpectedly") |
||||||
|
data = data + d |
||||||
|
return data |
||||||
|
|
||||||
|
def setproxy(self,proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): |
||||||
|
"""setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) |
||||||
|
Sets the proxy to be used. |
||||||
|
proxytype - The type of the proxy to be used. Three types |
||||||
|
are supported: PROXY_TYPE_SOCKS4 (including socks4a), |
||||||
|
PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP |
||||||
|
addr - The address of the server (IP or DNS). |
||||||
|
port - The port of the server. Defaults to 1080 for SOCKS |
||||||
|
servers and 8080 for HTTP proxy servers. |
||||||
|
rdns - Should DNS queries be preformed on the remote side |
||||||
|
(rather than the local side). The default is True. |
||||||
|
Note: This has no effect with SOCKS4 servers. |
||||||
|
username - Username to authenticate with to the server. |
||||||
|
The default is no authentication. |
||||||
|
password - Password to authenticate with to the server. |
||||||
|
Only relevant when username is also provided. |
||||||
|
""" |
||||||
|
self.__proxy = (proxytype,addr,port,rdns,username,password) |
||||||
|
|
||||||
|
def __negotiatesocks5(self,destaddr,destport): |
||||||
|
"""__negotiatesocks5(self,destaddr,destport) |
||||||
|
Negotiates a connection through a SOCKS5 server. |
||||||
|
""" |
||||||
|
# First we'll send the authentication packages we support. |
||||||
|
if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): |
||||||
|
# The username/password details were supplied to the |
||||||
|
# setproxy method so we support the USERNAME/PASSWORD |
||||||
|
# authentication (in addition to the standard none). |
||||||
|
self.sendall("\x05\x02\x00\x02") |
||||||
|
else: |
||||||
|
# No username/password were entered, therefore we |
||||||
|
# only support connections with no authentication. |
||||||
|
self.sendall("\x05\x01\x00") |
||||||
|
# We'll receive the server's response to determine which |
||||||
|
# method was selected |
||||||
|
chosenauth = self.__recvall(2) |
||||||
|
if chosenauth[0] != "\x05": |
||||||
|
self.close() |
||||||
|
raise GeneralProxyError((1,_generalerrors[1])) |
||||||
|
# Check the chosen authentication method |
||||||
|
if chosenauth[1] == "\x00": |
||||||
|
# No authentication is required |
||||||
|
pass |
||||||
|
elif chosenauth[1] == "\x02": |
||||||
|
# Okay, we need to perform a basic username/password |
||||||
|
# authentication. |
||||||
|
self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5]) |
||||||
|
authstat = self.__recvall(2) |
||||||
|
if authstat[0] != "\x01": |
||||||
|
# Bad response |
||||||
|
self.close() |
||||||
|
raise GeneralProxyError((1,_generalerrors[1])) |
||||||
|
if authstat[1] != "\x00": |
||||||
|
# Authentication failed |
||||||
|
self.close() |
||||||
|
raise Socks5AuthError,((3,_socks5autherrors[3])) |
||||||
|
# Authentication succeeded |
||||||
|
else: |
||||||
|
# Reaching here is always bad |
||||||
|
self.close() |
||||||
|
if chosenauth[1] == "\xFF": |
||||||
|
raise Socks5AuthError((2,_socks5autherrors[2])) |
||||||
|
else: |
||||||
|
raise GeneralProxyError((1,_generalerrors[1])) |
||||||
|
# Now we can request the actual connection |
||||||
|
req = "\x05\x01\x00" |
||||||
|
# If the given destination address is an IP address, we'll |
||||||
|
# use the IPv4 address request even if remote resolving was specified. |
||||||
|
try: |
||||||
|
ipaddr = socket.inet_aton(destaddr) |
||||||
|
req = req + "\x01" + ipaddr |
||||||
|
except socket.error: |
||||||
|
# Well it's not an IP number, so it's probably a DNS name. |
||||||
|
if self.__proxy[3]==True: |
||||||
|
# Resolve remotely |
||||||
|
ipaddr = None |
||||||
|
req = req + "\x03" + chr(len(destaddr)) + destaddr |
||||||
|
else: |
||||||
|
# Resolve locally |
||||||
|
ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) |
||||||
|
req = req + "\x01" + ipaddr |
||||||
|
req = req + struct.pack(">H",destport) |
||||||
|
self.sendall(req) |
||||||
|
# Get the response |
||||||
|
resp = self.__recvall(4) |
||||||
|
if resp[0] != "\x05": |
||||||
|
self.close() |
||||||
|
raise GeneralProxyError((1,_generalerrors[1])) |
||||||
|
elif resp[1] != "\x00": |
||||||
|
# Connection failed |
||||||
|
self.close() |
||||||
|
if ord(resp[1])<=8: |
||||||
|
raise Socks5Error((ord(resp[1]),_generalerrors[ord(resp[1])])) |
||||||
|
else: |
||||||
|
raise Socks5Error((9,_generalerrors[9])) |
||||||
|
# Get the bound address/port |
||||||
|
elif resp[3] == "\x01": |
||||||
|
boundaddr = self.__recvall(4) |
||||||
|
elif resp[3] == "\x03": |
||||||
|
resp = resp + self.recv(1) |
||||||
|
boundaddr = self.__recvall(ord(resp[4])) |
||||||
|
else: |
||||||
|
self.close() |
||||||
|
raise GeneralProxyError((1,_generalerrors[1])) |
||||||
|
boundport = struct.unpack(">H",self.__recvall(2))[0] |
||||||
|
self.__proxysockname = (boundaddr,boundport) |
||||||
|
if ipaddr != None: |
||||||
|
self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) |
||||||
|
else: |
||||||
|
self.__proxypeername = (destaddr,destport) |
||||||
|
|
||||||
|
def getproxysockname(self): |
||||||
|
"""getsockname() -> address info |
||||||
|
Returns the bound IP address and port number at the proxy. |
||||||
|
""" |
||||||
|
return self.__proxysockname |
||||||
|
|
||||||
|
def getproxypeername(self): |
||||||
|
"""getproxypeername() -> address info |
||||||
|
Returns the IP and port number of the proxy. |
||||||
|
""" |
||||||
|
return _orgsocket.getpeername(self) |
||||||
|
|
||||||
|
def getpeername(self): |
||||||
|
"""getpeername() -> address info |
||||||
|
Returns the IP address and port number of the destination |
||||||
|
machine (note: getproxypeername returns the proxy) |
||||||
|
""" |
||||||
|
return self.__proxypeername |
||||||
|
|
||||||
|
def __negotiatesocks4(self,destaddr,destport): |
||||||
|
"""__negotiatesocks4(self,destaddr,destport) |
||||||
|
Negotiates a connection through a SOCKS4 server. |
||||||
|
""" |
||||||
|
# Check if the destination address provided is an IP address |
||||||
|
rmtrslv = False |
||||||
|
try: |
||||||
|
ipaddr = socket.inet_aton(destaddr) |
||||||
|
except socket.error: |
||||||
|
# It's a DNS name. Check where it should be resolved. |
||||||
|
if self.__proxy[3]==True: |
||||||
|
ipaddr = "\x00\x00\x00\x01" |
||||||
|
rmtrslv = True |
||||||
|
else: |
||||||
|
ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) |
||||||
|
# Construct the request packet |
||||||
|
req = "\x04\x01" + struct.pack(">H",destport) + ipaddr |
||||||
|
# The username parameter is considered userid for SOCKS4 |
||||||
|
if self.__proxy[4] != None: |
||||||
|
req = req + self.__proxy[4] |
||||||
|
req = req + "\x00" |
||||||
|
# DNS name if remote resolving is required |
||||||
|
# NOTE: This is actually an extension to the SOCKS4 protocol |
||||||
|
# called SOCKS4A and may not be supported in all cases. |
||||||
|
if rmtrslv==True: |
||||||
|
req = req + destaddr + "\x00" |
||||||
|
self.sendall(req) |
||||||
|
# Get the response from the server |
||||||
|
resp = self.__recvall(8) |
||||||
|
if resp[0] != "\x00": |
||||||
|
# Bad data |
||||||
|
self.close() |
||||||
|
raise GeneralProxyError((1,_generalerrors[1])) |
||||||
|
if resp[1] != "\x5A": |
||||||
|
# Server returned an error |
||||||
|
self.close() |
||||||
|
if ord(resp[1]) in (91,92,93): |
||||||
|
self.close() |
||||||
|
raise Socks4Error((ord(resp[1]),_socks4errors[ord(resp[1])-90])) |
||||||
|
else: |
||||||
|
raise Socks4Error((94,_socks4errors[4])) |
||||||
|
# Get the bound address/port |
||||||
|
self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",resp[2:4])[0]) |
||||||
|
if rmtrslv != None: |
||||||
|
self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) |
||||||
|
else: |
||||||
|
self.__proxypeername = (destaddr,destport) |
||||||
|
|
||||||
|
def __negotiatehttp(self,destaddr,destport): |
||||||
|
"""__negotiatehttp(self,destaddr,destport) |
||||||
|
Negotiates a connection through an HTTP server. |
||||||
|
""" |
||||||
|
# If we need to resolve locally, we do this now |
||||||
|
if self.__proxy[3] == False: |
||||||
|
addr = socket.gethostbyname(destaddr) |
||||||
|
else: |
||||||
|
addr = destaddr |
||||||
|
self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n") |
||||||
|
# We read the response until we get the string "\r\n\r\n" |
||||||
|
resp = self.recv(1) |
||||||
|
while resp.find("\r\n\r\n")==-1: |
||||||
|
resp = resp + self.recv(1) |
||||||
|
# We just need the first line to check if the connection |
||||||
|
# was successful |
||||||
|
statusline = resp.splitlines()[0].split(" ",2) |
||||||
|
if statusline[0] not in ("HTTP/1.0","HTTP/1.1"): |
||||||
|
self.close() |
||||||
|
raise GeneralProxyError((1,_generalerrors[1])) |
||||||
|
try: |
||||||
|
statuscode = int(statusline[1]) |
||||||
|
except ValueError: |
||||||
|
self.close() |
||||||
|
raise GeneralProxyError((1,_generalerrors[1])) |
||||||
|
if statuscode != 200: |
||||||
|
self.close() |
||||||
|
raise HTTPError((statuscode,statusline[2])) |
||||||
|
self.__proxysockname = ("0.0.0.0",0) |
||||||
|
self.__proxypeername = (addr,destport) |
||||||
|
|
||||||
|
def connect(self,destpair): |
||||||
|
"""connect(self,despair) |
||||||
|
Connects to the specified destination through a proxy. |
||||||
|
destpar - A tuple of the IP/DNS address and the port number. |
||||||
|
(identical to socket's connect). |
||||||
|
To select the proxy server use setproxy(). |
||||||
|
""" |
||||||
|
# Do a minimal input check first |
||||||
|
if (type(destpair) in (list,tuple)==False) or (len(destpair)<2) or (type(destpair[0])!=str) or (type(destpair[1])!=int): |
||||||
|
raise GeneralProxyError((5,_generalerrors[5])) |
||||||
|
if self.__proxy[0] == PROXY_TYPE_SOCKS5: |
||||||
|
if self.__proxy[2] != None: |
||||||
|
portnum = self.__proxy[2] |
||||||
|
else: |
||||||
|
portnum = 1080 |
||||||
|
_orgsocket.connect(self,(self.__proxy[1],portnum)) |
||||||
|
self.__negotiatesocks5(destpair[0],destpair[1]) |
||||||
|
elif self.__proxy[0] == PROXY_TYPE_SOCKS4: |
||||||
|
if self.__proxy[2] != None: |
||||||
|
portnum = self.__proxy[2] |
||||||
|
else: |
||||||
|
portnum = 1080 |
||||||
|
_orgsocket.connect(self,(self.__proxy[1],portnum)) |
||||||
|
self.__negotiatesocks4(destpair[0],destpair[1]) |
||||||
|
elif self.__proxy[0] == PROXY_TYPE_HTTP: |
||||||
|
if self.__proxy[2] != None: |
||||||
|
portnum = self.__proxy[2] |
||||||
|
else: |
||||||
|
portnum = 8080 |
||||||
|
_orgsocket.connect(self,(self.__proxy[1],portnum)) |
||||||
|
self.__negotiatehttp(destpair[0],destpair[1]) |
||||||
|
elif self.__proxy[0] == None: |
||||||
|
_orgsocket.connect(self,(destpair[0],destpair[1])) |
||||||
|
else: |
||||||
|
raise GeneralProxyError((4,_generalerrors[4])) |
Loading…
Reference in new issue