From ced61c12dd03a47db4d91d82153631395f2774d7 Mon Sep 17 00:00:00 2001 From: Marcel van der Boom Date: Tue, 13 May 2014 15:48:01 +0200 Subject: [PATCH 1/2] Add support for using is.gd shortener USE_SHORTENER steers toggle of shortening urls. As twister has a hard limit at the moment of 140 characters, the more room we create for content the better. --- README.md | 1 + conf-example.py | 1 + feed2twister.py | 12 ++++++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 50a4e72..afd64e4 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Feed2twister is a simple script to post items from RSS/ATOM feeds to [Twister](h * [Twister](http://twister.net.co/) (of course) * [python-bitcoinrpc](https://pypi.python.org/pypi/python-bitcoinrpc/) * [feedparser](https://pypi.python.org/pypi/feedparser/) +* [gdshortener](https://github.com/torre76/gd_shortener/) (optional) ### Installing diff --git a/conf-example.py b/conf-example.py index 90ef61e..4f8f9bd 100644 --- a/conf-example.py +++ b/conf-example.py @@ -6,6 +6,7 @@ RPC_URL = 'http://MYRPCUSER:MYRPCPASSWORD@127.0.0.1:28332' # change to rpcuser a DB_FILENAME = 'items.db' # db is mainly there to keep track of "what not to post again" :) (debugging too, I guess) MAX_URL_LENGTH = 100 # this leaves 36 characters and a ... to get to 140. If we don't have that, we skip the item :( MAX_NEW_ITEMS_PER_FEED = 3 # we don't want to flood more than that in a single run. +USE_SHORTENER=True FEEDS = [ # Use your own feeds, of course :) 'https://swatwt.com/favs/rss/en', 'https://github.com/thedod.atom' diff --git a/feed2twister.py b/feed2twister.py index 4ee8430..651b48a 100644 --- a/feed2twister.py +++ b/feed2twister.py @@ -2,6 +2,9 @@ from conf import * import feedparser,anydbm,sys from bitcoinrpc.authproxy import AuthServiceProxy +if USE_SHORTENER: + import gdshortener + ### truncated_utf8() is based on http://stackoverflow.com/a/13738452 def _is_utf8_lead_byte(b): '''A UTF-8 intermediate byte starts with the bits 10xxxxxx.''' @@ -26,6 +29,8 @@ def get_next_k(twister,username): def main(max_items): db = anydbm.open(DB_FILENAME,'c') + if USE_SHORTENER: + s = gdshortener.ISGDShortener() twister = AuthServiceProxy(RPC_URL) for feed_url in FEEDS: logging.info(feed_url) @@ -36,8 +41,11 @@ def main(max_items): if db.has_key(eid): # been there, done that (or not - for a reason) logging.debug('Skipping duplicate {0}'.format(eid)) else: # format as a <=140 character string - if len(e.link)<=MAX_URL_LENGTH: - msg = u'{0} {1}'.format(e.link,e.title) + # Construct the link, possibly with shortener + entry_url = s.shorten(e.link)[0] if USE_SHORTENER else e.link + + if len(entry_url)<=MAX_URL_LENGTH: + msg = u'{0} {1}'.format(entry_url,e.title) if len(msg)>140: # Truncate (and hope it's still meaningful) msg = msg[:137]+u'...' else: # Link too long. Not enough space left for text :( From 89c3110710f128f70c461bd7552496c13810a115 Mon Sep 17 00:00:00 2001 From: The Dod Date: Tue, 13 May 2014 23:43:27 +0300 Subject: [PATCH 2/2] Don't assume gdshortener is installed Added optional is.gd stats --- conf-example.py | 3 ++- feed2twister.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/conf-example.py b/conf-example.py index 4f8f9bd..4f0f73e 100644 --- a/conf-example.py +++ b/conf-example.py @@ -6,7 +6,8 @@ RPC_URL = 'http://MYRPCUSER:MYRPCPASSWORD@127.0.0.1:28332' # change to rpcuser a DB_FILENAME = 'items.db' # db is mainly there to keep track of "what not to post again" :) (debugging too, I guess) MAX_URL_LENGTH = 100 # this leaves 36 characters and a ... to get to 140. If we don't have that, we skip the item :( MAX_NEW_ITEMS_PER_FEED = 3 # we don't want to flood more than that in a single run. -USE_SHORTENER=True +USE_SHORTENER = False # to enable this, you need gdshortener: https://github.com/torre76/gd_shortener/ +SHORTENER_STATS = True # tell is.gd to publicly show statistics for the shortened url FEEDS = [ # Use your own feeds, of course :) 'https://swatwt.com/favs/rss/en', 'https://github.com/thedod.atom' diff --git a/feed2twister.py b/feed2twister.py index 651b48a..13eaaab 100644 --- a/feed2twister.py +++ b/feed2twister.py @@ -3,7 +3,10 @@ import feedparser,anydbm,sys from bitcoinrpc.authproxy import AuthServiceProxy if USE_SHORTENER: - import gdshortener + try: + import gdshortener + except ImportError: + USE_SHORTENER = False ### truncated_utf8() is based on http://stackoverflow.com/a/13738452 def _is_utf8_lead_byte(b): @@ -29,8 +32,6 @@ def get_next_k(twister,username): def main(max_items): db = anydbm.open(DB_FILENAME,'c') - if USE_SHORTENER: - s = gdshortener.ISGDShortener() twister = AuthServiceProxy(RPC_URL) for feed_url in FEEDS: logging.info(feed_url) @@ -42,8 +43,7 @@ def main(max_items): logging.debug('Skipping duplicate {0}'.format(eid)) else: # format as a <=140 character string # Construct the link, possibly with shortener - entry_url = s.shorten(e.link)[0] if USE_SHORTENER else e.link - + entry_url = gdshortener.ISGDShortener().shorten(url=e.link, log_stat=SHORTENER_STATS)[0] if USE_SHORTENER else e.link if len(entry_url)<=MAX_URL_LENGTH: msg = u'{0} {1}'.format(entry_url,e.title) if len(msg)>140: # Truncate (and hope it's still meaningful)