From 700580aaab62c9ef63df627522340507351a1f91 Mon Sep 17 00:00:00 2001 From: emkael Date: Wed, 24 Feb 2016 23:10:17 +0100 Subject: * moing python scripts to subdirectory --- bin/fetch_cals.py | 86 +++++++++++++++++++++++++++++++++++++++++++++++++ bin/import_cals.py | 68 ++++++++++++++++++++++++++++++++++++++ bin/init_db.py | 16 +++++++++ bin/pyrun.sh | 6 ++++ conf/reddit-import.json | 2 +- fetch_cals.py | 86 ------------------------------------------------- import_cals.py | 67 -------------------------------------- init_db.py | 16 --------- 8 files changed, 177 insertions(+), 170 deletions(-) create mode 100644 bin/fetch_cals.py create mode 100644 bin/import_cals.py create mode 100644 bin/init_db.py create mode 100755 bin/pyrun.sh delete mode 100644 fetch_cals.py delete mode 100644 import_cals.py delete mode 100644 init_db.py diff --git a/bin/fetch_cals.py b/bin/fetch_cals.py new file mode 100644 index 0000000..13c4eca --- /dev/null +++ b/bin/fetch_cals.py @@ -0,0 +1,86 @@ +import datetime + +import dateutil.parser as dateparser +import ics +import requests +from rcal.db import Session +from rcal.model import Calendar, Entry + + +def update_event_data(db_event, ical_event): + db_event.name = ical_event.name + db_event.location = ical_event.location + db_event.begin_date = ical_event.begin.datetime + db_event.end_date = ical_event.end.datetime + db_event.all_day = ( + (db_event.end_date - db_event.begin_date).seconds % 86400 == 0 + ) and ( + db_event.begin_date.time() == datetime.time.min) + if db_event.all_day: + db_event.end_date = db_event.end_date - datetime.timedelta(days=1) + return db_event + + +def update_event(db_event, ical_event): + update_event_data(db_event, ical_event) + + +def add_event(event, calendar, session): + entry = Entry() + entry.uid = event.uid + entry.calendar = calendar + entry = update_event_data(entry, event) + session.add(entry) + + +def remove_event(event, session): + session.delete(event) + + +def get_last_modification_time(event): + for unused in event.__dict__['_unused']: + if unused.name == 'LAST-MODIFIED': + return dateparser.parse(unused.value) + return None + + +def fetch_calendar(calendar, session): + cal_data = requests.get(calendar.url) + cal_object = ics.Calendar(cal_data.content.decode(cal_data.encoding)) + cal_events = {e.uid: e for e in cal_object.events} + db_events = {e.uid: e for e in calendar.entries} + new_events = [e for u, e in cal_events.iteritems() + if u not in db_events.keys()] + old_events = [e for u, e in db_events.iteritems() + if u not in cal_events.keys()] + mod_events = [{'ics': cal_events[u], 'db': e} + for u, e in db_events.iteritems() if u in cal_events.keys()] + for event in mod_events: + modified_date = get_last_modification_time(event['ics']) + if not modified_date or \ + not calendar.last_updated or \ + modified_date.time() > calendar.last_updated.time(): + print 'Updating event %s' % event['db'].uid + update_event(event['db'], event['ics']) + for event in new_events: + print 'Adding event %s' % event.uid + add_event(event, calendar, session) + for event in old_events: + print 'Removing event %s' % event.uid + remove_event(event, session) + calendar.last_updated = datetime.datetime.now() + + +def main(): + session = Session.create() + + calendars = session.query(Calendar).all() + for calendar in calendars: + # print 'Fetching %s' % calendar.url + fetch_calendar(calendar, session) + + session.commit() + + +if __name__ == '__main__': + main() diff --git a/bin/import_cals.py b/bin/import_cals.py new file mode 100644 index 0000000..34e607d --- /dev/null +++ b/bin/import_cals.py @@ -0,0 +1,68 @@ +import json +import os +import re +import time +import urllib2 +import urlparse + +from rcal.db import Session +from rcal.model import Calendar, Category + +CONFIG = json.load(open( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '..', + 'conf', + 'reddit-import.json'), + 'r')) + + +def main(): + cal_list = None + + if not os.path.exists(CONFIG['cache']) or \ + int(time.time()) - int(os.path.getmtime(CONFIG['cache'])) > \ + CONFIG['cache_time']: + opener = urllib2.build_opener() + opener.addheaders = [('User-Agent', CONFIG['user_agent'])] + cal_list = json.loads(opener.open(CONFIG['reddit_url']).read()) + cal_list = cal_list[0]['data']['children'][0]['data']['selftext'] + with open(CONFIG['cache'], 'w') as cache_file: + cache_file.write(cal_list) + cache_file.close() + else: + cal_list = open(CONFIG['cache'], 'r').read() + + session = Session.create() + + ical_markdown = re.compile(r'^\[iCal\]\((.*)\)$') + + cells = [row.split('|') for row in cal_list.split('\n')] + for row in cells: + if len(row) == 7: + row = [r for r in row if r] + if len(row) == 5: + markdown_match = re.match(ical_markdown, row[2]) + if markdown_match: + ical_url = urlparse.urlparse(markdown_match.group(1)) + if ical_url.netloc == 'calendar.google.com': + ical_path = re.sub( + '^/?calendar/ical/', '', ical_url.path).split('/') + if len(ical_path) == 3: + calendar_uid = ical_path[0] + calendar = Calendar.fetch( + calendar_uid, + session, + row[0], + ical_url.geturl()) + calendar.website = row[4] + calendar.category = Category.fetch(row[1], session) + else: + print ical_url.geturl() + else: + print ical_url.geturl() + + session.commit() + +if __name__ == '__main__': + main() diff --git a/bin/init_db.py b/bin/init_db.py new file mode 100644 index 0000000..7d48e80 --- /dev/null +++ b/bin/init_db.py @@ -0,0 +1,16 @@ +import sys + +from rcal.db import Session +from rcal.model import BASE + + +def main(): + session = Session.create() + + if len(sys.argv) > 1 and sys.argv[1] == 'force': + BASE.metadata.drop_all(session.get_bind()) + + BASE.metadata.create_all(session.get_bind()) + +if __name__ == '__main__': + main() diff --git a/bin/pyrun.sh b/bin/pyrun.sh new file mode 100755 index 0000000..188a0eb --- /dev/null +++ b/bin/pyrun.sh @@ -0,0 +1,6 @@ +#!/bin/bash +DIR=$(dirname $0) +SCRIPT=$(echo $1 | sed 's/\.py\$//') +shift +export PYTHONPATH=$DIR/.. +python $DIR/$SCRIPT.py $@ diff --git a/conf/reddit-import.json b/conf/reddit-import.json index 4101f80..f34d522 100644 --- a/conf/reddit-import.json +++ b/conf/reddit-import.json @@ -1,5 +1,5 @@ { - "cache": "cache/list.md", + "cache": "../cache/list.md", "cache_time": 86400, "reddit_url": "https://www.reddit.com/r/MotorsportsCalendar/comments/45uv93/2016_calendar_list.json", "user_agent": "/u/emkael" diff --git a/fetch_cals.py b/fetch_cals.py deleted file mode 100644 index 13c4eca..0000000 --- a/fetch_cals.py +++ /dev/null @@ -1,86 +0,0 @@ -import datetime - -import dateutil.parser as dateparser -import ics -import requests -from rcal.db import Session -from rcal.model import Calendar, Entry - - -def update_event_data(db_event, ical_event): - db_event.name = ical_event.name - db_event.location = ical_event.location - db_event.begin_date = ical_event.begin.datetime - db_event.end_date = ical_event.end.datetime - db_event.all_day = ( - (db_event.end_date - db_event.begin_date).seconds % 86400 == 0 - ) and ( - db_event.begin_date.time() == datetime.time.min) - if db_event.all_day: - db_event.end_date = db_event.end_date - datetime.timedelta(days=1) - return db_event - - -def update_event(db_event, ical_event): - update_event_data(db_event, ical_event) - - -def add_event(event, calendar, session): - entry = Entry() - entry.uid = event.uid - entry.calendar = calendar - entry = update_event_data(entry, event) - session.add(entry) - - -def remove_event(event, session): - session.delete(event) - - -def get_last_modification_time(event): - for unused in event.__dict__['_unused']: - if unused.name == 'LAST-MODIFIED': - return dateparser.parse(unused.value) - return None - - -def fetch_calendar(calendar, session): - cal_data = requests.get(calendar.url) - cal_object = ics.Calendar(cal_data.content.decode(cal_data.encoding)) - cal_events = {e.uid: e for e in cal_object.events} - db_events = {e.uid: e for e in calendar.entries} - new_events = [e for u, e in cal_events.iteritems() - if u not in db_events.keys()] - old_events = [e for u, e in db_events.iteritems() - if u not in cal_events.keys()] - mod_events = [{'ics': cal_events[u], 'db': e} - for u, e in db_events.iteritems() if u in cal_events.keys()] - for event in mod_events: - modified_date = get_last_modification_time(event['ics']) - if not modified_date or \ - not calendar.last_updated or \ - modified_date.time() > calendar.last_updated.time(): - print 'Updating event %s' % event['db'].uid - update_event(event['db'], event['ics']) - for event in new_events: - print 'Adding event %s' % event.uid - add_event(event, calendar, session) - for event in old_events: - print 'Removing event %s' % event.uid - remove_event(event, session) - calendar.last_updated = datetime.datetime.now() - - -def main(): - session = Session.create() - - calendars = session.query(Calendar).all() - for calendar in calendars: - # print 'Fetching %s' % calendar.url - fetch_calendar(calendar, session) - - session.commit() - - -if __name__ == '__main__': - main() diff --git a/import_cals.py b/import_cals.py deleted file mode 100644 index 09c3272..0000000 --- a/import_cals.py +++ /dev/null @@ -1,67 +0,0 @@ -import json -import os -import re -import time -import urllib2 -import urlparse - -from rcal.db import Session -from rcal.model import Calendar, Category - -CONFIG = json.load(open( - os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'conf', - 'reddit-import.json'), - 'r')) - - -def main(): - cal_list = None - - if not os.path.exists(CONFIG['cache']) or \ - int(time.time()) - int(os.path.getmtime(CONFIG['cache'])) > \ - CONFIG['cache_time']: - opener = urllib2.build_opener() - opener.addheaders = [('User-Agent', CONFIG['user_agent'])] - cal_list = json.loads(opener.open(CONFIG['reddit_url']).read()) - cal_list = cal_list[0]['data']['children'][0]['data']['selftext'] - with open(CONFIG['cache'], 'w') as cache_file: - cache_file.write(cal_list) - cache_file.close() - else: - cal_list = open(CONFIG['cache'], 'r').read() - - session = Session.create() - - ical_markdown = re.compile(r'^\[iCal\]\((.*)\)$') - - cells = [row.split('|') for row in cal_list.split('\n')] - for row in cells: - if len(row) == 7: - row = [r for r in row if r] - if len(row) == 5: - markdown_match = re.match(ical_markdown, row[2]) - if markdown_match: - ical_url = urlparse.urlparse(markdown_match.group(1)) - if ical_url.netloc == 'calendar.google.com': - ical_path = re.sub( - '^/?calendar/ical/', '', ical_url.path).split('/') - if len(ical_path) == 3: - calendar_uid = ical_path[0] - calendar = Calendar.fetch( - calendar_uid, - session, - row[0], - ical_url.geturl()) - calendar.website = row[4] - calendar.category = Category.fetch(row[1], session) - else: - print ical_url.geturl() - else: - print ical_url.geturl() - - session.commit() - -if __name__ == '__main__': - main() diff --git a/init_db.py b/init_db.py deleted file mode 100644 index 7d48e80..0000000 --- a/init_db.py +++ /dev/null @@ -1,16 +0,0 @@ -import sys - -from rcal.db import Session -from rcal.model import BASE - - -def main(): - session = Session.create() - - if len(sys.argv) > 1 and sys.argv[1] == 'force': - BASE.metadata.drop_all(session.get_bind()) - - BASE.metadata.create_all(session.get_bind()) - -if __name__ == '__main__': - main() -- cgit v1.2.3