From 65d1fc63a0d92cb68dd2218f9001d0888067d0c7 Mon Sep 17 00:00:00 2001 From: Daan Wynen Date: Tue, 20 Jan 2015 00:49:41 +0100 Subject: [PATCH 1/2] fix the error I introduced that made old items being posted when nothing new was there. --- feed2twister.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/feed2twister.py b/feed2twister.py index 59f8a5b..20079e7 100755 --- a/feed2twister.py +++ b/feed2twister.py @@ -133,9 +133,6 @@ def main(max_items): logging.warn(u'Link too long at {0}'.format(eid)) continue - if n_items >= max_items: # Avoid accidental flooding - logging.warn(u'Skipping "over quota" item: {0}'.format(msg)) - continue logging.info(u'posting {0}'.format(msg)) @@ -151,7 +148,15 @@ def main(max_items): if n_items >= max_items: logging.warn(u'Quota reached. Skipping {0} items:'.format(len(feed.entries[i+1:]))) for ee in feed.entries[i+1:]: + eeid = '{0}|{1}'.format(feed_url, ee.id) logging.warn(u' {0}|{1}'.format(feed_url, ee.id)) + # already saved this item to db anyways, so we're done here + if eeid in db.keys(): + continue + # this is a *new* message we're skipping. build some fake post message in case + # we want to have a look at the database for debugging or such + utf8msg = truncated_utf8(u'Skipped: {0}'.format(e.title), 140) + db[eeid] = utf8msg break From 55c2b27164bbefa91ced518cbddec9dddbac6223 Mon Sep 17 00:00:00 2001 From: Daan Wynen Date: Tue, 20 Jan 2015 00:58:34 +0100 Subject: [PATCH 2/2] fix #8 items now get posted in chronological order (per feed). --- feed2twister.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/feed2twister.py b/feed2twister.py index 20079e7..2e39bd9 100755 --- a/feed2twister.py +++ b/feed2twister.py @@ -98,10 +98,13 @@ def main(max_items): feed = feedparser.parse(feed_url) n_items = 0 + # store posts for later since we want to post them in chronological order + msgs = [] + for i, e in enumerate(feed.entries): eid = '{0}|{1}'.format(feed_url,e.id) - if db.has_key(eid) and not args.repost_existing: # been there, done that (or not - for a reason) + if eid in db.keys() and not args.repost_existing: # been there, done that (or not - for a reason) logging.debug('Skipping duplicate {0}'.format(eid)) else: # format as a <=140 character string @@ -133,20 +136,14 @@ def main(max_items): logging.warn(u'Link too long at {0}'.format(eid)) continue - - logging.info(u'posting {0}'.format(msg)) - - try: - next_k = get_next_k(twister, main_config['username']) - twister.newpostmsg(main_config['username'], next_k, msg) - db[eid] = utfmsg # anydbm can't handle unicode, so it's a good thing we've also kept the utf-8 :) - except Exception, e: - logging.error(`e`) # usually not very informative :( + logging.info(u'will post {0}'.format(msg)) + msgs.append((eid, msg, utfmsg)) n_items+=1 if n_items >= max_items: logging.warn(u'Quota reached. Skipping {0} items:'.format(len(feed.entries[i+1:]))) + for ee in feed.entries[i+1:]: eeid = '{0}|{1}'.format(feed_url, ee.id) logging.warn(u' {0}|{1}'.format(feed_url, ee.id)) @@ -159,6 +156,19 @@ def main(max_items): db[eeid] = utf8msg break + # done parsing this feed, now post what we found, but in chronological order + msgs.reverse() + for (eid, msg, utfmsg) in msgs: + try: + logging.info(u'now posting {0}'.format(msg)) + next_k = get_next_k(twister, main_config['username']) + twister.newpostmsg(main_config['username'], next_k, msg) + db[eid] = utfmsg # anydbm can't handle unicode, so it's a good thing we've also kept the utf-8 :) + except Exception, e: + logging.error(repr(e)) # usually not very informative :( + if e.error: + logging.error(e.error) + if __name__=='__main__': if args.maxitems != None: