|
|
@ -98,10 +98,13 @@ def main(max_items): |
|
|
|
feed = feedparser.parse(feed_url) |
|
|
|
feed = feedparser.parse(feed_url) |
|
|
|
n_items = 0 |
|
|
|
n_items = 0 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# store posts for later since we want to post them in chronological order |
|
|
|
|
|
|
|
msgs = [] |
|
|
|
|
|
|
|
|
|
|
|
for i, e in enumerate(feed.entries): |
|
|
|
for i, e in enumerate(feed.entries): |
|
|
|
eid = '{0}|{1}'.format(feed_url,e.id) |
|
|
|
eid = '{0}|{1}'.format(feed_url,e.id) |
|
|
|
|
|
|
|
|
|
|
|
if db.has_key(eid) and not args.repost_existing: # been there, done that (or not - for a reason) |
|
|
|
if eid in db.keys() and not args.repost_existing: # been there, done that (or not - for a reason) |
|
|
|
logging.debug('Skipping duplicate {0}'.format(eid)) |
|
|
|
logging.debug('Skipping duplicate {0}'.format(eid)) |
|
|
|
|
|
|
|
|
|
|
|
else: # format as a <=140 character string |
|
|
|
else: # format as a <=140 character string |
|
|
@ -133,20 +136,14 @@ def main(max_items): |
|
|
|
logging.warn(u'Link too long at {0}'.format(eid)) |
|
|
|
logging.warn(u'Link too long at {0}'.format(eid)) |
|
|
|
continue |
|
|
|
continue |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
logging.info(u'will post {0}'.format(msg)) |
|
|
|
logging.info(u'posting {0}'.format(msg)) |
|
|
|
msgs.append((eid, msg, utfmsg)) |
|
|
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
|
|
|
next_k = get_next_k(twister, main_config['username']) |
|
|
|
|
|
|
|
twister.newpostmsg(main_config['username'], next_k, msg) |
|
|
|
|
|
|
|
db[eid] = utfmsg # anydbm can't handle unicode, so it's a good thing we've also kept the utf-8 :) |
|
|
|
|
|
|
|
except Exception, e: |
|
|
|
|
|
|
|
logging.error(`e`) # usually not very informative :( |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
n_items+=1 |
|
|
|
n_items+=1 |
|
|
|
|
|
|
|
|
|
|
|
if n_items >= max_items: |
|
|
|
if n_items >= max_items: |
|
|
|
logging.warn(u'Quota reached. Skipping {0} items:'.format(len(feed.entries[i+1:]))) |
|
|
|
logging.warn(u'Quota reached. Skipping {0} items:'.format(len(feed.entries[i+1:]))) |
|
|
|
|
|
|
|
|
|
|
|
for ee in feed.entries[i+1:]: |
|
|
|
for ee in feed.entries[i+1:]: |
|
|
|
eeid = '{0}|{1}'.format(feed_url, ee.id) |
|
|
|
eeid = '{0}|{1}'.format(feed_url, ee.id) |
|
|
|
logging.warn(u' {0}|{1}'.format(feed_url, ee.id)) |
|
|
|
logging.warn(u' {0}|{1}'.format(feed_url, ee.id)) |
|
|
@ -159,6 +156,19 @@ def main(max_items): |
|
|
|
db[eeid] = utf8msg |
|
|
|
db[eeid] = utf8msg |
|
|
|
break |
|
|
|
break |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# done parsing this feed, now post what we found, but in chronological order |
|
|
|
|
|
|
|
msgs.reverse() |
|
|
|
|
|
|
|
for (eid, msg, utfmsg) in msgs: |
|
|
|
|
|
|
|
try: |
|
|
|
|
|
|
|
logging.info(u'now posting {0}'.format(msg)) |
|
|
|
|
|
|
|
next_k = get_next_k(twister, main_config['username']) |
|
|
|
|
|
|
|
twister.newpostmsg(main_config['username'], next_k, msg) |
|
|
|
|
|
|
|
db[eid] = utfmsg # anydbm can't handle unicode, so it's a good thing we've also kept the utf-8 :) |
|
|
|
|
|
|
|
except Exception, e: |
|
|
|
|
|
|
|
logging.error(repr(e)) # usually not very informative :( |
|
|
|
|
|
|
|
if e.error: |
|
|
|
|
|
|
|
logging.error(e.error) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__=='__main__': |
|
|
|
if __name__=='__main__': |
|
|
|
if args.maxitems != None: |
|
|
|
if args.maxitems != None: |
|
|
|