twisterarmy/pytwister
1
0
Fork 0
mirror of https://github.com/twisterarmy/pytwister.git synced 2025-02-11 14:34:22 +00:00
Code Issues Projects Releases Wiki Activity

trying to avoid db corruption when script breaks

Browse Source
This commit is contained in:
toyg 2014-05-18 18:40:39 +01:00
parent 2b2de8954c
commit e377857fe9
1 changed files with 16 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
twistscraper.py
View File

@ -54,7 +54,7 @@ class GeoLocationService:
self.db[location] = coords
self.saveDb()
return coords
# still here? it's all rubbish
# still here? it's all rubbish
return None
def saveDb(self):
@ -164,8 +164,20 @@ class TwisterScraper:
print("Connection error retrieving user {0}: {1}".format(u, str(e)))
def saveDb(self):
with open(self.dbFile, 'wb') as dbFile:
pickle.dump(self.db, dbFile)
try:
with open(self.dbFile, 'wb') as dbFile:
pickle.dump(self.db, dbFile)
except (KeyboardInterrupt, Exception):
print("Closing db before quitting...")
if dbFile:
# close the hung descriptor and re-try the dumping
try:
dbFile.close()
except Exception:
pass
with open(self.dbFile, 'wb') as dbFile:
pickle.dump(self.db, dbFile)
def get_posts_since(self, username, dateObj, maxNum=1000):
since_epoch = time.mktime(dateObj.timetuple())
@ -217,3 +229,4 @@ if __name__ == '__main__':
ts = TwisterScraper(expanduser('~/.twister/_localusersdb'), 'localhost')
ts.scrape_users()
print("Total users in db: {0}".format(len(ts.db.users)))