From 700580aaab62c9ef63df627522340507351a1f91 Mon Sep 17 00:00:00 2001 From: emkael Date: Wed, 24 Feb 2016 23:10:17 +0100 Subject: * moing python scripts to subdirectory --- bin/fetch_cals.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ bin/import_cals.py | 68 ++++++++++++++++++++++++++++++++++++++++++ bin/init_db.py | 16 ++++++++++ bin/pyrun.sh | 6 ++++ 4 files changed, 176 insertions(+) create mode 100644 bin/fetch_cals.py create mode 100644 bin/import_cals.py create mode 100644 bin/init_db.py create mode 100755 bin/pyrun.sh (limited to 'bin') diff --git a/bin/fetch_cals.py b/bin/fetch_cals.py new file mode 100644 index 0000000..13c4eca --- /dev/null +++ b/bin/fetch_cals.py @@ -0,0 +1,86 @@ +import datetime + +import dateutil.parser as dateparser +import ics +import requests +from rcal.db import Session +from rcal.model import Calendar, Entry + + +def update_event_data(db_event, ical_event): + db_event.name = ical_event.name + db_event.location = ical_event.location + db_event.begin_date = ical_event.begin.datetime + db_event.end_date = ical_event.end.datetime + db_event.all_day = ( + (db_event.end_date - db_event.begin_date).seconds % 86400 == 0 + ) and ( + db_event.begin_date.time() == datetime.time.min) + if db_event.all_day: + db_event.end_date = db_event.end_date - datetime.timedelta(days=1) + return db_event + + +def update_event(db_event, ical_event): + update_event_data(db_event, ical_event) + + +def add_event(event, calendar, session): + entry = Entry() + entry.uid = event.uid + entry.calendar = calendar + entry = update_event_data(entry, event) + session.add(entry) + + +def remove_event(event, session): + session.delete(event) + + +def get_last_modification_time(event): + for unused in event.__dict__['_unused']: + if unused.name == 'LAST-MODIFIED': + return dateparser.parse(unused.value) + return None + + +def fetch_calendar(calendar, session): + cal_data = requests.get(calendar.url) + cal_object = ics.Calendar(cal_data.content.decode(cal_data.encoding)) + cal_events = {e.uid: e for e in cal_object.events} + db_events = {e.uid: e for e in calendar.entries} + new_events = [e for u, e in cal_events.iteritems() + if u not in db_events.keys()] + old_events = [e for u, e in db_events.iteritems() + if u not in cal_events.keys()] + mod_events = [{'ics': cal_events[u], 'db': e} + for u, e in db_events.iteritems() if u in cal_events.keys()] + for event in mod_events: + modified_date = get_last_modification_time(event['ics']) + if not modified_date or \ + not calendar.last_updated or \ + modified_date.time() > calendar.last_updated.time(): + print 'Updating event %s' % event['db'].uid + update_event(event['db'], event['ics']) + for event in new_events: + print 'Adding event %s' % event.uid + add_event(event, calendar, session) + for event in old_events: + print 'Removing event %s' % event.uid + remove_event(event, session) + calendar.last_updated = datetime.datetime.now() + + +def main(): + session = Session.create() + + calendars = session.query(Calendar).all() + for calendar in calendars: + # print 'Fetching %s' % calendar.url + fetch_calendar(calendar, session) + + session.commit() + + +if __name__ == '__main__': + main() diff --git a/bin/import_cals.py b/bin/import_cals.py new file mode 100644 index 0000000..34e607d --- /dev/null +++ b/bin/import_cals.py @@ -0,0 +1,68 @@ +import json +import os +import re +import time +import urllib2 +import urlparse + +from rcal.db import Session +from rcal.model import Calendar, Category + +CONFIG = json.load(open( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '..', + 'conf', + 'reddit-import.json'), + 'r')) + + +def main(): + cal_list = None + + if not os.path.exists(CONFIG['cache']) or \ + int(time.time()) - int(os.path.getmtime(CONFIG['cache'])) > \ + CONFIG['cache_time']: + opener = urllib2.build_opener() + opener.addheaders = [('User-Agent', CONFIG['user_agent'])] + cal_list = json.loads(opener.open(CONFIG['reddit_url']).read()) + cal_list = cal_list[0]['data']['children'][0]['data']['selftext'] + with open(CONFIG['cache'], 'w') as cache_file: + cache_file.write(cal_list) + cache_file.close() + else: + cal_list = open(CONFIG['cache'], 'r').read() + + session = Session.create() + + ical_markdown = re.compile(r'^\[iCal\]\((.*)\)$') + + cells = [row.split('|') for row in cal_list.split('\n')] + for row in cells: + if len(row) == 7: + row = [r for r in row if r] + if len(row) == 5: + markdown_match = re.match(ical_markdown, row[2]) + if markdown_match: + ical_url = urlparse.urlparse(markdown_match.group(1)) + if ical_url.netloc == 'calendar.google.com': + ical_path = re.sub( + '^/?calendar/ical/', '', ical_url.path).split('/') + if len(ical_path) == 3: + calendar_uid = ical_path[0] + calendar = Calendar.fetch( + calendar_uid, + session, + row[0], + ical_url.geturl()) + calendar.website = row[4] + calendar.category = Category.fetch(row[1], session) + else: + print ical_url.geturl() + else: + print ical_url.geturl() + + session.commit() + +if __name__ == '__main__': + main() diff --git a/bin/init_db.py b/bin/init_db.py new file mode 100644 index 0000000..7d48e80 --- /dev/null +++ b/bin/init_db.py @@ -0,0 +1,16 @@ +import sys + +from rcal.db import Session +from rcal.model import BASE + + +def main(): + session = Session.create() + + if len(sys.argv) > 1 and sys.argv[1] == 'force': + BASE.metadata.drop_all(session.get_bind()) + + BASE.metadata.create_all(session.get_bind()) + +if __name__ == '__main__': + main() diff --git a/bin/pyrun.sh b/bin/pyrun.sh new file mode 100755 index 0000000..188a0eb --- /dev/null +++ b/bin/pyrun.sh @@ -0,0 +1,6 @@ +#!/bin/bash +DIR=$(dirname $0) +SCRIPT=$(echo $1 | sed 's/\.py\$//') +shift +export PYTHONPATH=$DIR/.. +python $DIR/$SCRIPT.py $@ -- cgit v1.2.3