|
|
|
#!/usr/bin/python
|
|
|
|
|
|
|
|
import re
|
|
|
|
import os
|
|
|
|
import os.path
|
|
|
|
import sys
|
|
|
|
import errno
|
|
|
|
import time
|
|
|
|
import urllib2
|
|
|
|
import subprocess
|
|
|
|
import argparse
|
|
|
|
import configobj
|
|
|
|
|
|
|
|
# parse command line options
|
|
|
|
parser = argparse.ArgumentParser(
|
|
|
|
description='Hosts fetcher for py-i2phosts.',
|
|
|
|
epilog='Report bugs to http://zzz.i2p/topics/733')
|
|
|
|
parser.add_argument('-d', '--debug', action='store_true',
|
|
|
|
help='write debug messages to stdout instead of log file'),
|
|
|
|
parser.add_argument('-c', '--config', default='/etc/py-i2phosts/fetcher.conf', dest='config_file',
|
|
|
|
help='config file to use')
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
|
|
# read config
|
|
|
|
config = configobj.ConfigObj(args.config_file, file_error=True)
|
|
|
|
if 'include' in config:
|
|
|
|
config_included = configobj.ConfigObj(config['include'])
|
|
|
|
config.merge(config_included)
|
|
|
|
|
|
|
|
if 'DJANGO_PROJECT_PATH' in config:
|
|
|
|
sys.path.insert(1, config['DJANGO_PROJECT_PATH'])
|
|
|
|
else:
|
|
|
|
sys.stderr.write('"DJANGO_PROJECT_PATH" is missing in config\n')
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
# we want open urls through proxy
|
|
|
|
if 'proxyurl' in config:
|
|
|
|
proxy_handler = urllib2.ProxyHandler({'http': config['proxyurl']})
|
|
|
|
else:
|
|
|
|
log.critical('"proxyurl" is missing in config')
|
|
|
|
sys.exit(1)
|
|
|
|
opener = urllib2.build_opener(proxy_handler)
|
|
|
|
|
|
|
|
if not 'sources' in config:
|
|
|
|
log.critical('"sources" is missing in config')
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
# FIXME: use as_list() here
|
|
|
|
for source in config['sources']:
|
|
|
|
# cut hostname.i2p from url
|
|
|
|
source_hostname = re.sub(r'.*//(.+?)/.+', r'\1', source)
|
|
|
|
# use separate file for each host
|
|
|
|
filename = 'hosts.txt.' + source_hostname
|
|
|
|
# build last-modified info from file mtime
|
|
|
|
try:
|
|
|
|
mtime = os.path.getmtime(filename)
|
|
|
|
except OSError, e:
|
|
|
|
if e.errno == errno.ENOENT:
|
|
|
|
pass
|
|
|
|
else:
|
|
|
|
sys.stderr.write('fatal error: %s', e)
|
|
|
|
os.exit(1)
|
|
|
|
else:
|
|
|
|
last_modified = time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.localtime(mtime))
|
|
|
|
# prevent redownloading of hosts-file by passing If-Modified-Since http header
|
|
|
|
opener.addheaders = [('If-Modified-Since', last_modified)]
|
|
|
|
try:
|
|
|
|
resp = opener.open(source, timeout=60)
|
|
|
|
except urllib2.URLError, e:
|
|
|
|
if hasattr(e, 'reason'):
|
|
|
|
print 'failed to reach server %s, reason: %s' % (source_hostname, e.reason)
|
|
|
|
elif hasattr(e, 'code'):
|
|
|
|
if e.code == 304:
|
|
|
|
print '%s: not modified' % source_hostname
|
|
|
|
else:
|
|
|
|
print 'server %s can\'t finish the request, error code: %s' \
|
|
|
|
% (source_hostname, e.code)
|
|
|
|
continue
|
|
|
|
# read data from remote and write it to local file
|
|
|
|
content = resp.read()
|
|
|
|
f = open(filename, 'w')
|
|
|
|
f.write(content)
|
|
|
|
f.close()
|
|
|
|
# get last-modified info from header and change file's mtime
|
|
|
|
lm = resp.headers.get('Last-Modified')
|
|
|
|
print '%s: %s' % (source_hostname, lm)
|
|
|
|
if lm:
|
|
|
|
target_mtime = int(time.mktime(time.strptime(lm, '%a, %d %b %Y %H:%M:%S GMT')))
|
|
|
|
os.utime(filename, (target_mtime, target_mtime))
|
|
|
|
|
|
|
|
# form commnd-line for invoke injector
|
|
|
|
path = os.path.dirname(sys.argv[0])
|
|
|
|
print 'adding hosts from: %s' % source_hostname
|
|
|
|
sp_args = [path + '/py-i2phosts-injector', '-s', '-a', '-f', filename, '-d',
|
|
|
|
'Auto-added from ' + source_hostname]
|
|
|
|
p = subprocess.Popen(sp_args, shell=False, stdin=None,
|
|
|
|
stdout=None, stderr=subprocess.STDOUT)
|
|
|
|
p.wait()
|