From d12c5cdf55e9c4feed0b4da92b0ac8bb21b3fac1 Mon Sep 17 00:00:00 2001 From: Christophe Dumez Date: Sat, 15 Sep 2012 16:54:09 +0300 Subject: [PATCH] Address encoding issues when using search engine on Windows (closes #29) --- src/icons.qrc | 606 +++++++++++++------------- src/lang.qrc | 64 +-- src/searchengine/nova/fix_encoding.py | 371 ++++++++++++++++ src/searchengine/nova/nova2.py | 7 +- src/searchengine/search.qrc | 64 +-- src/searchengine/searchengine.cpp | 6 + 6 files changed, 751 insertions(+), 367 deletions(-) create mode 100644 src/searchengine/nova/fix_encoding.py diff --git a/src/icons.qrc b/src/icons.qrc index c5a98445d..c0a6f6e2a 100644 --- a/src/icons.qrc +++ b/src/icons.qrc @@ -1,352 +1,352 @@ - Icons/url.png - Icons/loading.png Icons/3-state-checkbox.gif - Icons/sphere.png - Icons/slow_off.png - Icons/sphere2.png + Icons/L.gif + Icons/loading.png Icons/magnet.png Icons/slow.png - Icons/L.gif - Icons/skin/qbittorrent_mono_light.png - Icons/skin/seeding.png - Icons/skin/splash.png - Icons/skin/tabs.gif - Icons/skin/qbittorrent16.png - Icons/skin/connected.png - Icons/skin/qbittorrent22.png - Icons/skin/disconnected.png - Icons/skin/mascot.png - Icons/skin/paused.png - Icons/skin/slider-area.gif - Icons/skin/stalledDL.png - Icons/skin/error.png - Icons/skin/qbittorrent_mono_dark.png - Icons/skin/handle-icon-horizontal.gif - Icons/skin/qbittorrent32.png - Icons/skin/knob.gif - Icons/skin/filterall.png - Icons/skin/uploading.png - Icons/skin/queued.png - Icons/skin/checking.png - Icons/skin/handle-icon.gif - Icons/skin/arrow-right.gif - Icons/skin/filterinactive.png - Icons/skin/firewalled.png - Icons/skin/downloading.png - Icons/skin/toolbox-divider.gif - Icons/skin/stalledUP.png - Icons/skin/filteractive.png - Icons/skin/bg-handle-horizontal.gif - Icons/skin/download.png - Icons/skin/ratio.png - Icons/flags/sm.png - Icons/flags/lt.png - Icons/flags/th.png - Icons/flags/sd.png - Icons/flags/mr.png - Icons/flags/by.png - Icons/flags/sc.png - Icons/flags/mz.png - Icons/flags/tt.png - Icons/flags/kp.png - Icons/flags/re.png - Icons/flags/um.png - Icons/flags/ec.png - Icons/flags/fk.png - Icons/flags/aw.png - Icons/flags/lb.png - Icons/flags/vg.png - Icons/flags/do.png - Icons/flags/ee.png - Icons/flags/ck.png - Icons/flags/gn.png - Icons/flags/cg.png - Icons/flags/ao.png - Icons/flags/dk.png - Icons/flags/ms.png - Icons/flags/sg.png - Icons/flags/bw.png - Icons/flags/br.png - Icons/flags/cn.png - Icons/flags/ua.png + Icons/slow_off.png + Icons/sphere.png + Icons/sphere2.png + Icons/url.png + Icons/flags/ad.png Icons/flags/ae.png - Icons/flags/tg.png - Icons/flags/ge.png - Icons/flags/ne.png - Icons/flags/km.png - Icons/flags/bf.png - Icons/flags/nf.png + Icons/flags/af.png + Icons/flags/ag.png + Icons/flags/ai.png + Icons/flags/al.png + Icons/flags/am.png + Icons/flags/an.png + Icons/flags/ao.png + Icons/flags/ar.png + Icons/flags/as.png + Icons/flags/at.png + Icons/flags/au.png + Icons/flags/aw.png + Icons/flags/ax.png Icons/flags/az.png - Icons/flags/sl.png - Icons/flags/md.png - Icons/flags/ph.png - Icons/flags/mc.png - Icons/flags/mq.png - Icons/flags/ng.png - Icons/flags/vu.png - Icons/flags/et.png + Icons/flags/ba.png + Icons/flags/bb.png + Icons/flags/bd.png + Icons/flags/be.png + Icons/flags/bf.png + Icons/flags/bg.png + Icons/flags/bh.png + Icons/flags/bi.png Icons/flags/bj.png - Icons/flags/nl.png - Icons/flags/ad.png - Icons/flags/tv.png - Icons/flags/fj.png - Icons/flags/to.png - Icons/flags/ru.png - Icons/flags/jp.png - Icons/flags/ma.png - Icons/flags/bz.png - Icons/flags/vn.png - Icons/flags/bt.png - Icons/flags/ht.png - Icons/flags/kw.png - Icons/flags/tr.png - Icons/flags/sy.png - Icons/flags/nr.png - Icons/flags/nz.png - Icons/flags/np.png - Icons/flags/uz.png - Icons/flags/in.png - Icons/flags/ga.png - Icons/flags/sa.png - Icons/flags/tl.png - Icons/flags/ly.png - Icons/flags/lr.png - Icons/flags/cz.png Icons/flags/bm.png - Icons/flags/us.png - Icons/flags/ye.png - Icons/flags/gs.png - Icons/flags/sb.png - Icons/flags/hm.png - Icons/flags/ke.png - Icons/flags/so.png - Icons/flags/ni.png + Icons/flags/bn.png + Icons/flags/bo.png + Icons/flags/br.png + Icons/flags/bs.png + Icons/flags/bt.png + Icons/flags/bv.png + Icons/flags/bw.png + Icons/flags/by.png + Icons/flags/bz.png + Icons/flags/ca.png + Icons/flags/cc.png + Icons/flags/cd.png + Icons/flags/cf.png + Icons/flags/cg.png + Icons/flags/ch.png + Icons/flags/ci.png + Icons/flags/ck.png + Icons/flags/cl.png Icons/flags/cm.png - Icons/flags/li.png - Icons/flags/gp.png + Icons/flags/cn.png Icons/flags/co.png - Icons/flags/kz.png + Icons/flags/cr.png + Icons/flags/cs.png + Icons/flags/cu.png + Icons/flags/cv.png + Icons/flags/cx.png + Icons/flags/cy.png + Icons/flags/cz.png Icons/flags/de.png - Icons/flags/mm.png + Icons/flags/dj.png + Icons/flags/dk.png + Icons/flags/dm.png + Icons/flags/do.png + Icons/flags/dz.png + Icons/flags/ec.png + Icons/flags/ee.png + Icons/flags/eg.png + Icons/flags/eh.png + Icons/flags/er.png + Icons/flags/es.png + Icons/flags/et.png + Icons/flags/fi.png + Icons/flags/fj.png + Icons/flags/fk.png + Icons/flags/fm.png + Icons/flags/fo.png + Icons/flags/fr.png + Icons/flags/ga.png Icons/flags/gb.png - Icons/flags/mx.png - Icons/flags/la.png - Icons/flags/au.png + Icons/flags/gd.png + Icons/flags/ge.png + Icons/flags/gf.png + Icons/flags/gh.png + Icons/flags/gi.png + Icons/flags/gl.png + Icons/flags/gm.png + Icons/flags/gn.png + Icons/flags/gp.png + Icons/flags/gq.png + Icons/flags/gr.png + Icons/flags/gs.png + Icons/flags/gt.png Icons/flags/gu.png Icons/flags/gw.png - Icons/flags/lc.png - Icons/flags/st.png - Icons/flags/nc.png - Icons/flags/ch.png - Icons/flags/cd.png - Icons/flags/mh.png - Icons/flags/pg.png - Icons/flags/bh.png - Icons/flags/kr.png + Icons/flags/gy.png + Icons/flags/hk.png + Icons/flags/hm.png + Icons/flags/hn.png + Icons/flags/hr.png + Icons/flags/ht.png + Icons/flags/hu.png + Icons/flags/id.png + Icons/flags/ie.png + Icons/flags/il.png + Icons/flags/in.png + Icons/flags/io.png + Icons/flags/iq.png + Icons/flags/ir.png Icons/flags/is.png - Icons/flags/gh.png - Icons/flags/mp.png - Icons/flags/ca.png - Icons/flags/bi.png - Icons/flags/yt.png Icons/flags/it.png + Icons/flags/jm.png + Icons/flags/jo.png + Icons/flags/jp.png + Icons/flags/ke.png + Icons/flags/kg.png + Icons/flags/kh.png + Icons/flags/ki.png + Icons/flags/km.png Icons/flags/kn.png - Icons/flags/ve.png - Icons/flags/cu.png - Icons/flags/bs.png - Icons/flags/rs.png - Icons/flags/tw.png - Icons/flags/bd.png - Icons/flags/pr.png + Icons/flags/kp.png + Icons/flags/kr.png + Icons/flags/kw.png + Icons/flags/ky.png + Icons/flags/kz.png + Icons/flags/la.png + Icons/flags/lb.png + Icons/flags/lc.png + Icons/flags/li.png Icons/flags/lk.png - Icons/flags/sn.png - Icons/flags/gr.png - Icons/flags/gm.png - Icons/flags/na.png - Icons/flags/om.png - Icons/flags/tz.png - Icons/flags/as.png - Icons/flags/no.png - Icons/flags/mv.png - Icons/flags/ki.png - Icons/flags/eg.png - Icons/flags/tj.png - Icons/flags/sj.png - Icons/flags/fi.png + Icons/flags/lr.png + Icons/flags/ls.png + Icons/flags/lt.png + Icons/flags/lu.png Icons/flags/lv.png - Icons/flags/cx.png - Icons/flags/ci.png - Icons/flags/vi.png - Icons/flags/be.png - Icons/flags/mw.png - Icons/flags/zw.png - Icons/flags/si.png - Icons/flags/dm.png - Icons/flags/jo.png - Icons/flags/hk.png + Icons/flags/ly.png + Icons/flags/ma.png + Icons/flags/mc.png + Icons/flags/md.png Icons/flags/me.png - Icons/flags/va.png - Icons/flags/pm.png - Icons/flags/qa.png - Icons/flags/cc.png - Icons/flags/kh.png - Icons/flags/id.png - Icons/flags/ai.png - Icons/flags/mn.png - Icons/flags/gd.png - Icons/flags/mt.png - Icons/flags/gq.png - Icons/flags/hr.png - Icons/flags/sv.png - Icons/flags/dz.png - Icons/flags/kg.png - Icons/flags/gy.png - Icons/flags/bv.png - Icons/flags/ag.png - Icons/flags/hn.png - Icons/flags/ml.png - Icons/flags/fo.png - Icons/flags/pw.png Icons/flags/mg.png - Icons/flags/iq.png - Icons/flags/jm.png - Icons/flags/bo.png - Icons/flags/sk.png - Icons/flags/sh.png - Icons/flags/tf.png + Icons/flags/mh.png Icons/flags/mk.png - Icons/flags/my.png - Icons/flags/es.png - Icons/flags/py.png - Icons/flags/ax.png - Icons/flags/ls.png - Icons/flags/tn.png - Icons/flags/ps.png - Icons/flags/bn.png - Icons/flags/dj.png - Icons/flags/cs.png - Icons/flags/pl.png - Icons/flags/cl.png - Icons/flags/pf.png - Icons/flags/gi.png - Icons/flags/bb.png - Icons/flags/ws.png - Icons/flags/cy.png - Icons/flags/pt.png - Icons/flags/ba.png - Icons/flags/gl.png - Icons/flags/ie.png - Icons/flags/fr.png - Icons/flags/tm.png - Icons/flags/er.png - Icons/flags/cr.png - Icons/flags/af.png + Icons/flags/ml.png + Icons/flags/mm.png + Icons/flags/mn.png + Icons/flags/mo.png + Icons/flags/mp.png + Icons/flags/mq.png + Icons/flags/mr.png + Icons/flags/ms.png + Icons/flags/mt.png Icons/flags/mu.png - Icons/flags/lu.png + Icons/flags/mv.png + Icons/flags/mw.png + Icons/flags/mx.png + Icons/flags/my.png + Icons/flags/mz.png + Icons/flags/na.png + Icons/flags/nc.png + Icons/flags/ne.png + Icons/flags/nf.png + Icons/flags/ng.png + Icons/flags/ni.png + Icons/flags/nl.png + Icons/flags/no.png + Icons/flags/np.png + Icons/flags/nr.png + Icons/flags/nu.png + Icons/flags/nz.png + Icons/flags/om.png + Icons/flags/pa.png + Icons/flags/pe.png + Icons/flags/pf.png + Icons/flags/pg.png + Icons/flags/ph.png Icons/flags/pk.png - Icons/flags/am.png - Icons/flags/ar.png - Icons/flags/uy.png - Icons/flags/hu.png + Icons/flags/pl.png + Icons/flags/pm.png Icons/flags/pn.png - Icons/flags/gf.png - Icons/flags/ir.png - Icons/flags/pe.png - Icons/flags/nu.png + Icons/flags/pr.png + Icons/flags/ps.png + Icons/flags/pt.png + Icons/flags/pw.png + Icons/flags/py.png + Icons/flags/qa.png + Icons/flags/re.png Icons/flags/ro.png - Icons/flags/mo.png - Icons/flags/za.png - Icons/flags/ky.png - Icons/flags/zm.png + Icons/flags/rs.png + Icons/flags/ru.png + Icons/flags/rw.png + Icons/flags/sa.png + Icons/flags/sb.png + Icons/flags/sc.png + Icons/flags/sd.png Icons/flags/se.png - Icons/flags/il.png - Icons/flags/fm.png + Icons/flags/sg.png + Icons/flags/sh.png + Icons/flags/si.png + Icons/flags/sj.png + Icons/flags/sk.png + Icons/flags/sl.png + Icons/flags/sm.png + Icons/flags/sn.png + Icons/flags/so.png Icons/flags/sr.png - Icons/flags/pa.png - Icons/flags/cv.png - Icons/flags/tc.png - Icons/flags/an.png + Icons/flags/st.png + Icons/flags/sv.png + Icons/flags/sy.png Icons/flags/sz.png - Icons/flags/io.png - Icons/flags/rw.png - Icons/flags/eh.png - Icons/flags/cf.png - Icons/flags/gt.png - Icons/flags/al.png - Icons/flags/ug.png + Icons/flags/tc.png Icons/flags/td.png - Icons/flags/at.png - Icons/flags/vc.png + Icons/flags/tf.png + Icons/flags/tg.png + Icons/flags/th.png + Icons/flags/tj.png Icons/flags/tk.png - Icons/flags/bg.png + Icons/flags/tl.png + Icons/flags/tm.png + Icons/flags/tn.png + Icons/flags/to.png + Icons/flags/tr.png + Icons/flags/tt.png + Icons/flags/tv.png + Icons/flags/tw.png + Icons/flags/tz.png + Icons/flags/ua.png + Icons/flags/ug.png + Icons/flags/um.png + Icons/flags/us.png + Icons/flags/uy.png + Icons/flags/uz.png + Icons/flags/va.png + Icons/flags/vc.png + Icons/flags/ve.png + Icons/flags/vg.png + Icons/flags/vi.png + Icons/flags/vn.png + Icons/flags/vu.png Icons/flags/wf.png - Icons/oxygen/preferences-system.png - Icons/oxygen/unavailable.png - Icons/oxygen/document-edit-verify.png - Icons/oxygen/list-remove.png + Icons/flags/ws.png + Icons/flags/ye.png + Icons/flags/yt.png + Icons/flags/za.png + Icons/flags/zm.png + Icons/flags/zw.png + Icons/oxygen/application-exit.png + Icons/oxygen/application-rss+xml.png + Icons/oxygen/application-x-mswinurl.png + Icons/oxygen/chronometer.png + Icons/oxygen/dialog-cancel.png + Icons/oxygen/dialog-information.png Icons/oxygen/dialog-warning.png - Icons/oxygen/mail-folder-inbox.png + Icons/oxygen/document-edit-verify.png + Icons/oxygen/document-edit.png + Icons/oxygen/document-encrypt.png + Icons/oxygen/document-import.png + Icons/oxygen/document-new.png + Icons/oxygen/document-properties.png + Icons/oxygen/document-save.png + Icons/oxygen/download.png Icons/oxygen/edit-clear-history.png + Icons/oxygen/edit-clear.png Icons/oxygen/edit-copy.png + Icons/oxygen/edit-cut.png + Icons/oxygen/edit-delete.png + Icons/oxygen/edit-find-user.png + Icons/oxygen/edit-find.png + Icons/oxygen/edit-paste.png + Icons/oxygen/edit-rename.png Icons/oxygen/folder-documents.png - Icons/oxygen/document-edit.png - Icons/oxygen/security-low.png + Icons/oxygen/folder-new.png + Icons/oxygen/folder-remote.png + Icons/oxygen/gear.png + Icons/oxygen/gear32.png + Icons/oxygen/go-down.png + Icons/oxygen/go-up.png + Icons/oxygen/help-about.png + Icons/oxygen/help-contents.png + Icons/oxygen/inode-directory.png Icons/oxygen/insert-link.png - Icons/oxygen/network-wired.png + Icons/oxygen/list-add.png + Icons/oxygen/list-remove.png + Icons/oxygen/mail-folder-inbox.png Icons/oxygen/mail-mark-read.png - Icons/oxygen/go-up.png - Icons/oxygen/application-exit.png - Icons/oxygen/edit-rename.png - Icons/oxygen/edit-cut.png - Icons/oxygen/gear32.png - Icons/oxygen/media-playback-start.png - Icons/oxygen/user-group-delete.png - Icons/oxygen/edit-find-user.png Icons/oxygen/media-playback-pause.png + Icons/oxygen/media-playback-start.png + Icons/oxygen/network-server.png + Icons/oxygen/network-wired.png + Icons/oxygen/object-locked.png + Icons/oxygen/preferences-desktop.png + Icons/oxygen/preferences-other.png + Icons/oxygen/preferences-system-network.png + Icons/oxygen/preferences-system.png + Icons/oxygen/preferences-web-browser-cookies.png + Icons/oxygen/security-high.png + Icons/oxygen/security-low.png + Icons/oxygen/services.png Icons/oxygen/tab-close.png - Icons/oxygen/inode-directory.png + Icons/oxygen/task-attention.png + Icons/oxygen/text-plain.png Icons/oxygen/tools-report-bug.png + Icons/oxygen/unavailable.png + Icons/oxygen/user-group-delete.png + Icons/oxygen/user-group-new.png + Icons/oxygen/view-calendar-journal.png + Icons/oxygen/view-categories.png Icons/oxygen/view-filter.png - Icons/oxygen/services.png Icons/oxygen/view-preview.png Icons/oxygen/view-refresh.png - Icons/oxygen/text-plain.png - Icons/oxygen/edit-delete.png - Icons/oxygen/chronometer.png - Icons/oxygen/dialog-cancel.png - Icons/oxygen/task-attention.png - Icons/oxygen/preferences-system-network.png - Icons/oxygen/document-properties.png - Icons/oxygen/user-group-new.png - Icons/oxygen/security-high.png - Icons/oxygen/network-server.png Icons/oxygen/wallet-open.png - Icons/oxygen/preferences-web-browser-cookies.png - Icons/oxygen/list-add.png - Icons/oxygen/edit-paste.png - Icons/oxygen/folder-remote.png - Icons/oxygen/help-about.png - Icons/oxygen/document-save.png - Icons/oxygen/gear.png - Icons/oxygen/view-categories.png - Icons/oxygen/document-new.png - Icons/oxygen/dialog-information.png - Icons/oxygen/preferences-other.png - Icons/oxygen/object-locked.png - Icons/oxygen/view-calendar-journal.png - Icons/oxygen/application-rss+xml.png - Icons/oxygen/document-encrypt.png - Icons/oxygen/help-contents.png - Icons/oxygen/preferences-desktop.png - Icons/oxygen/application-x-mswinurl.png - Icons/oxygen/go-down.png - Icons/oxygen/document-import.png - Icons/oxygen/download.png - Icons/oxygen/edit-find.png - Icons/oxygen/edit-clear.png Icons/oxygen/webui.png - Icons/oxygen/folder-new.png + Icons/skin/arrow-right.gif + Icons/skin/bg-handle-horizontal.gif + Icons/skin/checking.png + Icons/skin/connected.png + Icons/skin/disconnected.png + Icons/skin/download.png + Icons/skin/downloading.png + Icons/skin/error.png + Icons/skin/filteractive.png + Icons/skin/filterall.png + Icons/skin/filterinactive.png + Icons/skin/firewalled.png + Icons/skin/handle-icon-horizontal.gif + Icons/skin/handle-icon.gif + Icons/skin/knob.gif + Icons/skin/mascot.png + Icons/skin/paused.png + Icons/skin/qbittorrent16.png + Icons/skin/qbittorrent22.png + Icons/skin/qbittorrent32.png + Icons/skin/qbittorrent_mono_dark.png + Icons/skin/qbittorrent_mono_light.png + Icons/skin/queued.png + Icons/skin/ratio.png + Icons/skin/seeding.png + Icons/skin/slider-area.gif + Icons/skin/splash.png + Icons/skin/stalledDL.png + Icons/skin/stalledUP.png + Icons/skin/tabs.gif + Icons/skin/toolbox-divider.gif + Icons/skin/uploading.png \ No newline at end of file diff --git a/src/lang.qrc b/src/lang.qrc index 0da19f611..e631354e5 100644 --- a/src/lang.qrc +++ b/src/lang.qrc @@ -1,41 +1,41 @@ - lang/qbittorrent_nl.qm - lang/qbittorrent_hu.qm - lang/qbittorrent_ru.qm - lang/qbittorrent_zh_TW.qm - lang/qbittorrent_tr.qm - lang/qbittorrent_fi.qm - lang/qbittorrent_sk.qm - lang/qbittorrent_ja.qm - lang/qbittorrent_el.qm + lang/qbittorrent_ar.qm + lang/qbittorrent_be.qm + lang/qbittorrent_bg.qm lang/qbittorrent_ca.qm - lang/qbittorrent_pt.qm - lang/qbittorrent_it.qm - lang/qbittorrent_fr.qm - lang/qbittorrent_uk.qm - lang/qbittorrent_zh.qm - lang/qbittorrent_lt.qm - lang/qbittorrent_ko.qm - lang/qbittorrent_nb.qm - lang/qbittorrent_sv.qm + lang/qbittorrent_cs.qm + lang/qbittorrent_da.qm lang/qbittorrent_de.qm + lang/qbittorrent_el.qm + lang/qbittorrent_en.qm + lang/qbittorrent_es.qm + lang/qbittorrent_eu.qm + lang/qbittorrent_fi.qm + lang/qbittorrent_fr.qm lang/qbittorrent_gl.qm - lang/qbittorrent_sr.qm - lang/qbittorrent_pt_BR.qm - lang/qbittorrent_da.qm - lang/qbittorrent_cs.qm + lang/qbittorrent_he.qm + lang/qbittorrent_hr.qm + lang/qbittorrent_hu.qm lang/qbittorrent_hy.qm + lang/qbittorrent_it.qm + lang/qbittorrent_ja.qm + lang/qbittorrent_ka.qm + lang/qbittorrent_ko.qm + lang/qbittorrent_lt.qm + lang/qbittorrent_nb.qm + lang/qbittorrent_nl.qm lang/qbittorrent_pl.qm - lang/qbittorrent_bg.qm - lang/qbittorrent_ar.qm - lang/qbittorrent_es.qm - lang/qbittorrent_en.qm - lang/qbittorrent_hr.qm + lang/qbittorrent_pt.qm + lang/qbittorrent_pt_BR.qm lang/qbittorrent_ro.qm - lang/qbittorrent_ka.qm - lang/qbittorrent_be.qm - lang/qbittorrent_eu.qm - lang/qbittorrent_he.qm + lang/qbittorrent_ru.qm + lang/qbittorrent_sk.qm + lang/qbittorrent_sr.qm + lang/qbittorrent_sv.qm + lang/qbittorrent_tr.qm + lang/qbittorrent_uk.qm + lang/qbittorrent_zh.qm + lang/qbittorrent_zh_TW.qm - + \ No newline at end of file diff --git a/src/searchengine/nova/fix_encoding.py b/src/searchengine/nova/fix_encoding.py new file mode 100644 index 000000000..61bd742b4 --- /dev/null +++ b/src/searchengine/nova/fix_encoding.py @@ -0,0 +1,371 @@ +# Copyright (c) 2011 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Collection of functions and classes to fix various encoding problems on +multiple platforms with python. +""" + +import codecs +import locale +import os +import sys + + +# Prevents initializing multiple times. +_SYS_ARGV_PROCESSED = False + + +def complain(message): + """If any exception occurs in this file, we'll probably try to print it + on stderr, which makes for frustrating debugging if stderr is directed + to our wrapper. So be paranoid about catching errors and reporting them + to sys.__stderr__, so that the user has a higher chance to see them. + """ + print >> sys.__stderr__, ( + isinstance(message, str) and message or repr(message)) + + +def fix_default_encoding(): + """Forces utf8 solidly on all platforms. + + By default python execution environment is lazy and defaults to ascii + encoding. + + http://uucode.com/blog/2007/03/23/shut-up-you-dummy-7-bit-python/ + """ + if sys.getdefaultencoding() == 'utf-8': + return False + + # Regenerate setdefaultencoding. + reload(sys) + # Module 'sys' has no 'setdefaultencoding' member + # pylint: disable=E1101 + sys.setdefaultencoding('utf-8') + for attr in dir(locale): + if attr[0:3] != 'LC_': + continue + aref = getattr(locale, attr) + try: + locale.setlocale(aref, '') + except locale.Error: + continue + try: + lang = locale.getlocale(aref)[0] + except (TypeError, ValueError): + continue + if lang: + try: + locale.setlocale(aref, (lang, 'UTF-8')) + except locale.Error: + os.environ[attr] = lang + '.UTF-8' + try: + locale.setlocale(locale.LC_ALL, '') + except locale.Error: + pass + return True + + +############################### +# Windows specific + + +def fix_win_sys_argv(encoding): + """Converts sys.argv to 'encoding' encoded string. + + utf-8 is recommended. + + Works around . + """ + global _SYS_ARGV_PROCESSED + if _SYS_ARGV_PROCESSED: + return False + + # These types are available on linux but not Mac. + # pylint: disable=E0611,F0401 + from ctypes import byref, c_int, POINTER, windll, WINFUNCTYPE + from ctypes.wintypes import LPCWSTR, LPWSTR + + # + GetCommandLineW = WINFUNCTYPE(LPWSTR)(('GetCommandLineW', windll.kernel32)) + # + CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( + ('CommandLineToArgvW', windll.shell32)) + + argc = c_int(0) + argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) + argv = [ + argv_unicode[i].encode(encoding, 'replace') + for i in xrange(0, argc.value)] + + if not hasattr(sys, 'frozen'): + # If this is an executable produced by py2exe or bbfreeze, then it + # will have been invoked directly. Otherwise, unicode_argv[0] is the + # Python interpreter, so skip that. + argv = argv[1:] + + # Also skip option arguments to the Python interpreter. + while len(argv) > 0: + arg = argv[0] + if not arg.startswith(u'-') or arg == u'-': + break + argv = argv[1:] + if arg == u'-m': + # sys.argv[0] should really be the absolute path of the + # module source, but never mind. + break + if arg == u'-c': + argv[0] = u'-c' + break + sys.argv = argv + _SYS_ARGV_PROCESSED = True + return True + + +def fix_win_codec(): + """Works around .""" + # + try: + codecs.lookup('cp65001') + return False + except LookupError: + codecs.register( + lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) + return True + + +class WinUnicodeOutputBase(object): + """Base class to adapt sys.stdout or sys.stderr to behave correctly on + Windows. + + Setting encoding to utf-8 is recommended. + """ + def __init__(self, fileno, name, encoding): + # Corresponding file handle. + self._fileno = fileno + self.encoding = encoding + self.name = name + + self.closed = False + self.softspace = False + self.mode = 'w' + + @staticmethod + def isatty(): + return False + + def close(self): + # Don't really close the handle, that would only cause problems. + self.closed = True + + def fileno(self): + return self._fileno + + def flush(self): + raise NotImplementedError() + + def write(self, text): + raise NotImplementedError() + + def writelines(self, lines): + try: + for line in lines: + self.write(line) + except Exception, e: + complain('%s.writelines: %r' % (self.name, e)) + raise + + +class WinUnicodeConsoleOutput(WinUnicodeOutputBase): + """Output adapter to a Windows Console. + + Understands how to use the win32 console API. + """ + def __init__(self, console_handle, fileno, stream_name, encoding): + super(WinUnicodeConsoleOutput, self).__init__( + fileno, '' % stream_name, encoding) + # Handle to use for WriteConsoleW + self._console_handle = console_handle + + # Loads the necessary function. + # These types are available on linux but not Mac. + # pylint: disable=E0611,F0401 + from ctypes import byref, GetLastError, POINTER, windll, WINFUNCTYPE + from ctypes.wintypes import BOOL, DWORD, HANDLE, LPWSTR + from ctypes.wintypes import LPVOID # pylint: disable=E0611 + + self._DWORD = DWORD + self._byref = byref + + # + self._WriteConsoleW = WINFUNCTYPE( + BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)( + ('WriteConsoleW', windll.kernel32)) + self._GetLastError = GetLastError + + def flush(self): + # No need to flush the console since it's immediate. + pass + + def write(self, text): + try: + if not isinstance(text, unicode): + # Convert to unicode. + text = str(text).decode(self.encoding, 'replace') + remaining = len(text) + while remaining > 0: + n = self._DWORD(0) + # There is a shorter-than-documented limitation on the length of the + # string passed to WriteConsoleW. See + # . + retval = self._WriteConsoleW( + self._console_handle, text, + min(remaining, 10000), + self._byref(n), None) + if retval == 0 or n.value == 0: + raise IOError( + 'WriteConsoleW returned %r, n.value = %r, last error = %r' % ( + retval, n.value, self._GetLastError())) + remaining -= n.value + if not remaining: + break + text = text[n.value:] + except Exception, e: + complain('%s.write: %r' % (self.name, e)) + raise + + +class WinUnicodeOutput(WinUnicodeOutputBase): + """Output adaptor to a file output on Windows. + + If the standard FileWrite function is used, it will be encoded in the current + code page. WriteConsoleW() permits writting any character. + """ + def __init__(self, stream, fileno, encoding): + super(WinUnicodeOutput, self).__init__( + fileno, '' % stream.name, encoding) + # Output stream + self._stream = stream + + # Flush right now. + self.flush() + + def flush(self): + try: + self._stream.flush() + except Exception, e: + complain('%s.flush: %r from %r' % (self.name, e, self._stream)) + raise + + def write(self, text): + try: + if isinstance(text, unicode): + # Replace characters that cannot be printed instead of failing. + text = text.encode(self.encoding, 'replace') + self._stream.write(text) + except Exception, e: + complain('%s.write: %r' % (self.name, e)) + raise + + +def win_handle_is_a_console(handle): + """Returns True if a Windows file handle is a handle to a console.""" + # These types are available on linux but not Mac. + # pylint: disable=E0611,F0401 + from ctypes import byref, POINTER, windll, WINFUNCTYPE + from ctypes.wintypes import BOOL, DWORD, HANDLE + + FILE_TYPE_CHAR = 0x0002 + FILE_TYPE_REMOTE = 0x8000 + INVALID_HANDLE_VALUE = DWORD(-1).value + + # + GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( + ('GetConsoleMode', windll.kernel32)) + # + GetFileType = WINFUNCTYPE(DWORD, DWORD)(('GetFileType', windll.kernel32)) + + # GetStdHandle returns INVALID_HANDLE_VALUE, NULL, or a valid handle. + if handle == INVALID_HANDLE_VALUE or handle is None: + return False + return ( + (GetFileType(handle) & ~FILE_TYPE_REMOTE) == FILE_TYPE_CHAR and + GetConsoleMode(handle, byref(DWORD()))) + + +def win_get_unicode_stream(stream, excepted_fileno, output_handle, encoding): + """Returns a unicode-compatible stream. + + This function will return a direct-Console writing object only if: + - the file number is the expected console file number + - the handle the expected file handle + - the 'real' handle is in fact a handle to a console. + """ + old_fileno = getattr(stream, 'fileno', lambda: None)() + if old_fileno == excepted_fileno: + # These types are available on linux but not Mac. + # pylint: disable=E0611,F0401 + from ctypes import windll, WINFUNCTYPE + from ctypes.wintypes import DWORD, HANDLE + + # + GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(('GetStdHandle', windll.kernel32)) + + real_output_handle = GetStdHandle(DWORD(output_handle)) + if win_handle_is_a_console(real_output_handle): + # It's a console. + return WinUnicodeConsoleOutput( + real_output_handle, old_fileno, stream.name, encoding) + + # It's something else. Create an auto-encoding stream. + return WinUnicodeOutput(stream, old_fileno, encoding) + + +def fix_win_console(encoding): + """Makes Unicode console output work independently of the current code page. + + This also fixes . + Credit to Michael Kaplan + and + TZOmegaTZIOY + . + """ + if (isinstance(sys.stdout, WinUnicodeOutputBase) or + isinstance(sys.stderr, WinUnicodeOutputBase)): + return False + + try: + # SetConsoleCP and SetConsoleOutputCP could be used to change the code page + # but it's not really useful since the code here is using WriteConsoleW(). + # Also, changing the code page is 'permanent' to the console and needs to be + # reverted manually. + # In practice one needs to set the console font to a TTF font to be able to + # see all the characters but it failed for me in practice. In any case, it + # won't throw any exception when printing, which is the important part. + # -11 and -12 are defined in stdio.h + sys.stdout = win_get_unicode_stream(sys.stdout, 1, -11, encoding) + sys.stderr = win_get_unicode_stream(sys.stderr, 2, -12, encoding) + # TODO(maruel): Do sys.stdin with ReadConsoleW(). Albeit the limitation is + # "It doesn't appear to be possible to read Unicode characters in UTF-8 + # mode" and this appears to be a limitation of cmd.exe. + except Exception, e: + complain('exception %r while fixing up sys.stdout and sys.stderr' % e) + return True + + +def fix_encoding(): + """Fixes various encoding problems on all platforms. + + Should be called at the very begining of the process. + """ + ret = True + if sys.platform == 'win32': + ret &= fix_win_codec() + + ret &= fix_default_encoding() + + if sys.platform == 'win32': + encoding = sys.getdefaultencoding() + ret &= fix_win_sys_argv(encoding) + ret &= fix_win_console(encoding) + return ret diff --git a/src/searchengine/nova/nova2.py b/src/searchengine/nova/nova2.py index 5d75642bb..2228729a0 100755 --- a/src/searchengine/nova/nova2.py +++ b/src/searchengine/nova/nova2.py @@ -26,7 +26,7 @@ # POSSIBILITY OF SUCH DAMAGE. -#VERSION: 1.23 +#VERSION: 1.31 # Author: # Fabien Devaux @@ -42,6 +42,8 @@ import threading import os import glob +import fix_encoding + THREADED = True CATEGORIES = ('all', 'movies', 'tv', 'music', 'games', 'anime', 'software', 'pictures', 'books') @@ -111,6 +113,9 @@ class EngineLauncher(threading.Thread): self.engine.search(self.what) if __name__ == '__main__': + # Make sure we enforce utf-8 encoding + fix_encoding.fix_encoding() + if len(sys.argv) < 2: raise SystemExit('./nova2.py [all|engine1[,engine2]*] \navailable engines: %s'% (','.join(supported_engines))) diff --git a/src/searchengine/search.qrc b/src/searchengine/search.qrc index 5b8af7f32..7364dc95a 100644 --- a/src/searchengine/search.qrc +++ b/src/searchengine/search.qrc @@ -1,55 +1,57 @@ + nova/fix_encoding.py + nova/helpers.py nova/nova2.py + nova/nova2dl.py nova/novaprinter.py nova/socks.py - nova/nova2dl.py - nova/helpers.py - nova/engines/vertor.png + nova/engines/btdigg.png + nova/engines/btdigg.py + nova/engines/btjunkie.png + nova/engines/btjunkie.py + nova/engines/extratorrent.png + nova/engines/extratorrent.py + nova/engines/isohunt.png + nova/engines/isohunt.py nova/engines/kickasstorrents.png + nova/engines/kickasstorrents.py + nova/engines/limetorrents.py nova/engines/mininova.png nova/engines/mininova.py - nova/engines/torrentdownloads.png - nova/engines/isohunt.png - nova/engines/torrentreactor.py - nova/engines/btjunkie.png - nova/engines/extratorrent.py + nova/engines/piratebay.png nova/engines/piratebay.py + nova/engines/torrentdownloads.png nova/engines/torrentdownloads.py nova/engines/torrentreactor.png - nova/engines/isohunt.py - nova/engines/btdigg.py - nova/engines/btjunkie.py - nova/engines/kickasstorrents.py - nova/engines/extratorrent.png - nova/engines/piratebay.png + nova/engines/torrentreactor.py + nova/engines/vertor.png nova/engines/vertor.py - nova/engines/btdigg.png - nova3/sgmllib3.py + nova3/helpers.py nova3/nova2.py + nova3/nova2dl.py nova3/novaprinter.py + nova3/sgmllib3.py nova3/socks.py - nova3/nova2dl.py - nova3/helpers.py - nova3/engines/vertor.png + nova3/engines/btdigg.png + nova3/engines/btdigg.py + nova3/engines/btjunkie.png + nova3/engines/btjunkie.py + nova3/engines/extratorrent.png + nova3/engines/extratorrent.py + nova3/engines/isohunt.png + nova3/engines/isohunt.py nova3/engines/kickasstorrents.png + nova3/engines/kickasstorrents.py nova3/engines/mininova.png nova3/engines/mininova.py - nova3/engines/torrentdownloads.png - nova3/engines/isohunt.png - nova3/engines/torrentreactor.py - nova3/engines/btjunkie.png - nova3/engines/extratorrent.py + nova3/engines/piratebay.png nova3/engines/piratebay.py + nova3/engines/torrentdownloads.png nova3/engines/torrentdownloads.py nova3/engines/torrentreactor.png - nova3/engines/isohunt.py - nova3/engines/btdigg.py - nova3/engines/btjunkie.py - nova3/engines/kickasstorrents.py - nova3/engines/extratorrent.png - nova3/engines/piratebay.png + nova3/engines/torrentreactor.py + nova3/engines/vertor.png nova3/engines/vertor.py - nova3/engines/btdigg.png \ No newline at end of file diff --git a/src/searchengine/searchengine.cpp b/src/searchengine/searchengine.cpp index ee1581703..0eeee236b 100644 --- a/src/searchengine/searchengine.cpp +++ b/src/searchengine/searchengine.cpp @@ -443,6 +443,12 @@ void SearchEngine::updateNova() { removePythonScriptIfExists(filePath); QFile::copy(":/"+nova_folder+"/socks.py", filePath); + if (nova_folder == "nova") { + filePath = search_dir.absoluteFilePath("fix_encoding.py"); + removePythonScriptIfExists(filePath); + QFile::copy(":/"+nova_folder+"/fix_encoding.py", filePath); + } + if (nova_folder == "nova3") { filePath = search_dir.absoluteFilePath("sgmllib3.py"); removePythonScriptIfExists(filePath);