py-i2phosts/py-i2phosts-fetcher

#!/usr/bin/python

import re
import os
import os.path
import sys
import errno
import time
import urllib2
import subprocess
import argparse
import configobj

# parse command line options
parser = argparse.ArgumentParser(
		description='Hosts fetcher for py-i2phosts.',
		epilog='Report bugs to http://zzz.i2p/topics/733')
parser.add_argument('-d', '--debug', action='store_true',
		help='write debug messages to stdout instead of log file'),
parser.add_argument('-c', '--config', default='/etc/py-i2phosts/fetcher.conf', dest='config_file',
		help='config file to use')
args = parser.parse_args()

# read config
config = configobj.ConfigObj(args.config_file, file_error=True)
if 'include' in config:
	config_included = configobj.ConfigObj(config['include'])
	config.merge(config_included)

if 'DJANGO_PROJECT_PATH' in config:
	sys.path.insert(1, config['DJANGO_PROJECT_PATH'])
else:
	sys.stderr.write('"DJANGO_PROJECT_PATH" is missing in config\n')
	sys.exit(1)
from web.lib.utils import get_logger

# configure logger
if args.debug == True:
	log_level = 'debug'
	log_file = None
else:
	log_level = config['log_level']
	log_file = config['log_file']
log = get_logger(filename=log_file, log_level=log_level)

# we want open urls through proxy
if 'proxyurl' in config:
	proxy_handler = urllib2.ProxyHandler({'http': config['proxyurl']})
else:
	log.critical('"proxyurl" is missing in config')
	sys.exit(1)
opener = urllib2.build_opener(proxy_handler)

if not 'sources' in config:
	log.critical('"sources" is missing in config')
	sys.exit(1)

# FIXME: use as_list() here
for source in config['sources']:
	# cut hostname.i2p from url
	source_hostname = re.sub(r'.*//(.+?)/.+', r'\1', source)
	# use separate file for each host
	filename = 'hosts.txt.' + source_hostname
	# build last-modified info from file mtime
	try:
		mtime = os.path.getmtime(filename)
	except OSError, e:
		if e.errno == errno.ENOENT:
			pass
		else:
			log.critical('fatal error: %s', e)
			sys.exit(1)
	else:
		last_modified = time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.localtime(mtime))
		# prevent redownloading of hosts-file by passing If-Modified-Since http header
		opener.addheaders = [('If-Modified-Since', last_modified)]
	try:
		log.debug('fetching hosts from: %s', source_hostname)
		resp = opener.open(source, timeout=60)
	except urllib2.URLError, e:
		if hasattr(e, 'reason'):
			log.warning('failed to reach server %s, reason: %s', source_hostname, e.reason)
		elif hasattr(e, 'code'):
			if e.code == 304:
				log.info('%s: not modified', source_hostname)
			else:
				log.warning('server %s can\'t finish the request, error code: %s',
					source_hostname, e.code)
		continue
	# read data from remote and write it to local file
	try:
		content = resp.read()
	except:
		log.warning('failed to read data from %s', source_hostname)
		continue
	f = open(filename, 'w')
	f.write(content)
	f.close()
	# get last-modified info from header and change file's mtime
	lm = resp.headers.get('Last-Modified')
	log.debug('%s Last-Modified: %s', source_hostname, lm)
	if lm:
		target_mtime = int(time.mktime(time.strptime(lm, '%a, %d %b %Y %H:%M:%S GMT')))
		os.utime(filename, (target_mtime, target_mtime))

	# form commnd-line for invoke injector
	path = os.path.dirname(sys.argv[0])
	log.info('adding hosts from: %s', source_hostname)
	sp_args = [path + '/py-i2phosts-injector', '-s', '-a', '-f', filename, '-d',
			'Auto-added from ' + source_hostname]
	p = subprocess.Popen(sp_args, shell=False, stdin=None,
			stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
	out = p.communicate()[0]
	log.info('injector output: \n%s', out)