r4sas/py-i2phosts
1
0
Fork 0
mirror of https://github.com/r4sas/py-i2phosts synced 2025-08-30 07:31:54 +00:00
Code Issues Releases Wiki Activity

py-i2phosts-fetcher: improve log messages

Browse Source
This commit is contained in:
Hidden Z 2010-11-07 20:08:03 +00:00
parent 7b400f6a29
commit 8bef1fd666
1 changed files with 13 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
py-i2phosts-fetcher
View File

@ -64,32 +64,36 @@ opener = urllib2.build_opener(proxy_handler)
all_sources = ExternalSource.objects.filter(active=True) all_sources = ExternalSource.objects.filter(active=True)
for source in all_sources: for source in all_sources:
log.debug('%s: starting work', source.name)
if source.last_modified: if source.last_modified:
last_modified = source.last_modified.strftime('%a, %d %b %Y %H:%M:%S GMT') last_modified = source.last_modified.strftime('%a, %d %b %Y %H:%M:%S GMT')
# prevent redownloading of hosts-file by passing If-Modified-Since http header # prevent redownloading of hosts-file by passing If-Modified-Since http header
opener.addheaders = [('If-Modified-Since', last_modified)] opener.addheaders = [('If-Modified-Since', last_modified)]
log.debug('%s: appending If-Modified-Since: %s', source.name, last_modified)
if source.etag: if source.etag:
opener.addheaders = [('If-None-Match', source.etag)] opener.addheaders = [('If-None-Match', source.etag)]
log.debug('%s: appending If-None-Match: %s', source.name, source.etag)
try: try:
log.debug('fetching hosts from: %s', source.name) log.debug('%s: sending GET...', source.name)
resp = opener.open(source.url, timeout=60) resp = opener.open(source.url, timeout=60)
except urllib2.URLError, e: except urllib2.URLError, e:
if hasattr(e, 'reason'): if hasattr(e, 'reason'):
log.warning('failed to reach server %s, reason: %s', source.name, e.reason) log.warning('%s: failed to reach server, reason: %s', source.name, e.reason)
elif hasattr(e, 'code'): elif hasattr(e, 'code'):
if e.code == 304: if e.code == 304:
log.info('%s: not modified', source.name) log.info('%s: not modified', source.name)
source.last_success = datetime.datetime.utcnow() source.last_success = datetime.datetime.utcnow()
source.save() source.save()
else: else:
log.warning('server %s can\'t finish the request, error code: %s', log.warning('%s: %s can\'t finish the request, error code: %s',
source.name, e.code) source.name, e.code)
continue continue
# read data from remote and write it to local file # read data from remote and write it to local file
try: try:
log.debug('%s: reading response data', source.name)
content = resp.read() content = resp.read()
except: except:
log.warning('failed to read data from %s', source.name) log.warning('%s: failed to read data', source.name)
continue continue
# save fetched content into temporary file # save fetched content into temporary file
fd, tmpfile = tempfile.mkstemp(text=True) fd, tmpfile = tempfile.mkstemp(text=True)
@ -99,22 +103,23 @@ for source in all_sources:
# get last-modified info from header # get last-modified info from header
lm = resp.headers.get('Last-Modified') lm = resp.headers.get('Last-Modified')
if lm: if lm:
log.debug('%s Last-Modified: %s', source.name, lm) log.debug('%s: Last-Modified: %s', source.name, lm)
source.last_modified = datetime.datetime.strptime(lm, '%a, %d %b %Y %H:%M:%S GMT') source.last_modified = datetime.datetime.strptime(lm, '%a, %d %b %Y %H:%M:%S GMT')
# get ETag # get ETag
etag = resp.headers.get('ETag') etag = resp.headers.get('ETag')
if etag: if etag:
log.debug('%s ETag: %s', source.name, etag) log.debug('%s: ETag: %s', source.name, etag)
source.etag = etag source.etag = etag
# form commnd-line for invoke injector # form commnd-line for invoke injector
log.info('adding hosts from: %s', source.name) log.info('%s: adding hosts...', source.name)
sp_args = ['py-i2phosts-injector', '-s', '-f', tmpfile, '-d', sp_args = ['py-i2phosts-injector', '-s', '-f', tmpfile, '-d',
'Auto-added from ' + source.name] 'Auto-added from ' + source.name]
p = subprocess.Popen(sp_args, shell=False, stdin=None, p = subprocess.Popen(sp_args, shell=False, stdin=None,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out = p.communicate()[0] out = p.communicate()[0]
os.remove(tmpfile) os.remove(tmpfile)
log.info('injector output: \n%s', out) log.info('%s: injector output: \n%s', source.name, out)
# update last_success # update last_success
source.last_success = datetime.datetime.utcnow() source.last_success = datetime.datetime.utcnow()
log.debug('%s: updating last_success timestamp: %s', source.name, source.last_success)
source.save() source.save()