You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
100 lines
3.7 KiB
100 lines
3.7 KiB
# -*- coding: utf-8 -*- |
|
import pickle |
|
from operator import attrgetter |
|
from threading import Thread |
|
from time import sleep |
|
from os.path import expanduser |
|
|
|
import feedparser |
|
|
|
from twistscraper import TwisterScraper, GeoLocationService, User |
|
|
|
__author__ = 'Giacomo Lacava' |
|
|
|
GITHUB_REPO_URL = 'https://github.com/{user}/{repo}' |
|
GITHUB_COMMIT_FEED_TEMPLATE = GITHUB_REPO_URL + '/commits/master.atom' |
|
|
|
CORE_COMMIT_FEED = GITHUB_COMMIT_FEED_TEMPLATE.format(user='miguelfreitas', repo='twister-core') |
|
HTML_COMMIT_FEED = GITHUB_COMMIT_FEED_TEMPLATE.format(user='miguelfreitas', repo='twister-html') |
|
SEED_COMMIT_FEED = GITHUB_COMMIT_FEED_TEMPLATE.format(user='miguelfreitas', repo='twister-seeder') |
|
CORE_REPO_URL = GITHUB_REPO_URL.format(user='miguelfreitas', repo='twister-core') |
|
HTML_REPO_URL = GITHUB_REPO_URL.format(user='miguelfreitas', repo='twister-html') |
|
SEED_REPO_URL = GITHUB_REPO_URL.format(user='miguelfreitas', repo='twister-seeder') |
|
|
|
|
|
class TwisterMonitor(Thread): |
|
MESSAGE = 'Twister update: {msg} - Pull it now: {url}' |
|
|
|
def __init__(self, twister_monitor, username, repo_feed=CORE_COMMIT_FEED, repo_url=CORE_REPO_URL): |
|
Thread.__init__(self) |
|
self.ts = twister_monitor |
|
self.cacheFile = expanduser('~/.twister/_twm_cache') |
|
self.cache = {} |
|
self.username = username |
|
self.feed = repo_feed |
|
self.repo = repo_url |
|
self.loadCache() |
|
|
|
def loadCache(self): |
|
try: |
|
with open(self.cacheFile, 'rb') as f: |
|
self.cache = pickle.load(f) |
|
except FileNotFoundError: |
|
self.cache = {} |
|
|
|
def get_commits(self): |
|
print("Fetching {0}".format(self.feed)) |
|
f = feedparser.parse(self.feed) |
|
if f['bozo'] == 1: |
|
raise Exception('Bad feed! Status: {status} - Error {err}'.format(status=f.status, err=f.bozo_exception)) |
|
|
|
if self.feed not in self.cache: |
|
self.cache[self.feed] = [] |
|
|
|
f.entries.sort(key=attrgetter('updated_parsed')) |
|
for entry in f.entries: |
|
print("Checking {0}".format(entry.id)) |
|
if entry.id not in self.cache[self.feed]: |
|
message = TwisterMonitor.MESSAGE.format(msg=entry.title, url=self.repo) |
|
cut = 1 |
|
while len(message) >= 140: |
|
message = TwisterMonitor.MESSAGE.format(msg=(entry.title[:-cut] + '...'), url=self.repo) |
|
cut += 1 |
|
|
|
print("Checking last post key...") |
|
key = 1 |
|
lastpost = self.ts.twister.getposts(1, [{"username": self.username}]) |
|
if len(lastpost) == 1: |
|
key = lastpost[0]['userpost']['k'] + 1 |
|
print("Posting '{0}' with key {1}...".format(message, key)) |
|
self.ts.twister.newpostmsg(self.username, key, message) |
|
print("Posted!") |
|
self.cache[self.feed].append(entry.id) |
|
self.saveCache() |
|
sleep(10 * 60) |
|
|
|
def saveCache(self): |
|
with open(self.cacheFile, 'wb') as f: |
|
pickle.dump(self.cache, f) |
|
|
|
def run(self): |
|
while True: |
|
try: |
|
self.get_commits() |
|
except Exception as e: |
|
print("Exception following!") |
|
print(e) |
|
sleep(60 * 60) # in seconds |
|
|
|
|
|
if __name__ == '__main__': |
|
botID = 'twmonitor' |
|
ts = TwisterScraper(expanduser('~/.twister/_localusersdb')) |
|
monitor = TwisterMonitor(ts, botID, CORE_COMMIT_FEED, CORE_REPO_URL) |
|
monitor.start() |
|
sleep(4 * 60) |
|
monitor_ui = TwisterMonitor(ts, botID, HTML_COMMIT_FEED, HTML_REPO_URL) |
|
monitor_ui.start() |
|
sleep(6 * 60) |
|
monitor_seed = TwisterMonitor(ts, botID, SEED_COMMIT_FEED, SEED_REPO_URL) |
|
monitor_seed.start()
|
|
|