|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import pickle
|
|
|
|
from operator import attrgetter
|
|
|
|
from threading import Thread
|
|
|
|
from time import sleep
|
|
|
|
from os.path import expanduser
|
|
|
|
|
|
|
|
import feedparser
|
|
|
|
|
|
|
|
from twistscraper import TwisterScraper
|
|
|
|
|
|
|
|
__author__ = 'Giacomo Lacava'
|
|
|
|
|
|
|
|
|
|
|
|
class FeedCache:
|
|
|
|
_shared_state = {}
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
self.__dict__ = self._shared_state
|
|
|
|
if len(self.__dict__) == 0:
|
|
|
|
# first instance setup
|
|
|
|
self.cacheFile = expanduser('~/.twister/_twm_cache')
|
|
|
|
self.cache = {}
|
|
|
|
self._load_cache()
|
|
|
|
|
|
|
|
|
|
|
|
def _load_cache(self):
|
|
|
|
try:
|
|
|
|
with open(self.cacheFile, 'rb') as f:
|
|
|
|
self.cache = pickle.load(f)
|
|
|
|
except FileNotFoundError:
|
|
|
|
self.cache = {}
|
|
|
|
|
|
|
|
def _save_cache(self):
|
|
|
|
with open(self.cacheFile, 'wb') as f:
|
|
|
|
pickle.dump(self.cache, f)
|
|
|
|
|
|
|
|
def get_feed_cache(self, feedUrl):
|
|
|
|
if feedUrl not in self.cache:
|
|
|
|
self.cache[feedUrl] = []
|
|
|
|
return self.cache[feedUrl]
|
|
|
|
|
|
|
|
def add_entry(self, feedUrl, entryID):
|
|
|
|
feed = self.get_feed_cache(feedUrl)
|
|
|
|
feed.append(entryID)
|
|
|
|
self._save_cache()
|
|
|
|
|
|
|
|
|
|
|
|
class TwisterMonitor(Thread):
|
|
|
|
MESSAGE = '{repo}: {msg} - {url}'
|
|
|
|
GITHUB_REPO_URL = 'https://github.com/{user}/{repo}'
|
|
|
|
GITHUB_COMMIT_FEED_TEMPLATE = GITHUB_REPO_URL + '/commits/master.atom'
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, scraperObj, username, github_user, github_repo):
|
|
|
|
Thread.__init__(self)
|
|
|
|
self.ts = scraperObj
|
|
|
|
self.cacheObj = FeedCache()
|
|
|
|
self.username = username
|
|
|
|
self.feed = TwisterMonitor.GITHUB_COMMIT_FEED_TEMPLATE.format(user=github_user, repo=github_repo)
|
|
|
|
self.repo = TwisterMonitor.GITHUB_REPO_URL.format(user=github_user, repo=github_repo)
|
|
|
|
self.github_user = github_user
|
|
|
|
self.github_repo = github_repo
|
|
|
|
self.cache = self.cacheObj.get_feed_cache(self.feed)
|
|
|
|
|
|
|
|
|
|
|
|
def get_commits(self):
|
|
|
|
print("Fetching {0}".format(self.feed))
|
|
|
|
f = feedparser.parse(self.feed)
|
|
|
|
if f['bozo'] == 1:
|
|
|
|
raise Exception('Bad feed! Status: {status} - Error {err}'.format(status=f.status, err=f.bozo_exception))
|
|
|
|
|
|
|
|
f.entries.sort(key=attrgetter('updated_parsed'))
|
|
|
|
for entry in f.entries:
|
|
|
|
print("Checking {0}".format(entry.id))
|
|
|
|
if entry.id not in self.cache:
|
|
|
|
message = TwisterMonitor.MESSAGE.format(msg=entry.title,
|
|
|
|
url=self.repo,
|
|
|
|
repo=self.github_repo)
|
|
|
|
cut = 1
|
|
|
|
while len(message) >= 140:
|
|
|
|
message = TwisterMonitor.MESSAGE.format(msg=(entry.title[:-cut] + '...'),
|
|
|
|
url=self.repo,
|
|
|
|
repo=self.github_repo)
|
|
|
|
cut += 1
|
|
|
|
|
|
|
|
print("Checking last post key...")
|
|
|
|
key = 1
|
|
|
|
lastpost = self.ts.twister.getposts(1, [{"username": self.username}])
|
|
|
|
if len(lastpost) == 1:
|
|
|
|
key = lastpost[0]['userpost']['k'] + 1
|
|
|
|
print("Posting '{0}' with key {1}...".format(message, key))
|
|
|
|
self.ts.twister.newpostmsg(self.username, key, message)
|
|
|
|
print("Posted!")
|
|
|
|
self.cacheObj.add_entry(self.feed, entry.id)
|
|
|
|
sleep(10 * 60)
|
|
|
|
|
|
|
|
|
|
|
|
def run(self):
|
|
|
|
while True:
|
|
|
|
try:
|
|
|
|
self.get_commits()
|
|
|
|
except Exception as e:
|
|
|
|
print("Exception following!")
|
|
|
|
print(e)
|
|
|
|
sleep(60 * 60) # in seconds
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
botID = 'twmonitor'
|
|
|
|
ts = TwisterScraper(expanduser('~/.twister/_localusersdb'))
|
|
|
|
monitor = TwisterMonitor(ts, botID, "miguelfreitas", "twister-core")
|
|
|
|
monitor.start()
|
|
|
|
sleep(4 * 60)
|
|
|
|
monitor_ui = TwisterMonitor(ts, botID, "miguelfreitas", "twister-html")
|
|
|
|
monitor_ui.start()
|
|
|
|
sleep(6 * 60)
|
|
|
|
monitor_seed = TwisterMonitor(ts, botID, "miguelfreitas", "twister-seeder")
|
|
|
|
monitor_seed.start()
|