diff options
Diffstat (limited to 'app/python')
| -rw-r--r-- | app/python/.gitignore | 1 | ||||
| l--------- | app/python/config | 1 | ||||
| -rw-r--r-- | app/python/fetch_cals.py | 91 | ||||
| -rw-r--r-- | app/python/generate_cal_urls.py | 17 | ||||
| -rw-r--r-- | app/python/import_cals.py | 132 | ||||
| -rw-r--r-- | app/python/init_db.py | 16 | ||||
| -rw-r--r-- | app/python/rcal/__init__.py | 0 | ||||
| -rw-r--r-- | app/python/rcal/db.py | 34 | ||||
| -rw-r--r-- | app/python/rcal/model.py | 153 | ||||
| -rw-r--r-- | app/python/weekdays.py | 32 |
10 files changed, 0 insertions, 477 deletions
diff --git a/app/python/.gitignore b/app/python/.gitignore deleted file mode 100644 index df04015..0000000 --- a/app/python/.gitignore +++ /dev/null @@ -1 +0,0 @@ -rcal/*.pyc diff --git a/app/python/config b/app/python/config deleted file mode 120000 index 899f698..0000000 --- a/app/python/config +++ /dev/null @@ -1 +0,0 @@ -../../config
\ No newline at end of file diff --git a/app/python/fetch_cals.py b/app/python/fetch_cals.py deleted file mode 100644 index 8b53254..0000000 --- a/app/python/fetch_cals.py +++ /dev/null @@ -1,91 +0,0 @@ -import datetime - -import dateutil.parser as dateparser -import ics -import pytz -import requests -from rcal.db import Session -from rcal.model import Calendar, Entry - - -def update_event_data(db_event, ical_event): - db_event.name = ical_event.name - db_event.location = ical_event.location - db_event.begin_date = ical_event.begin.datetime - db_event.end_date = ical_event.end.datetime - db_event.all_day = ( - (db_event.end_date - db_event.begin_date).seconds % 86400 == 0 - ) and ( - db_event.begin_date.time() == datetime.time.min) - db_event.last_modified = get_last_modification_time(ical_event) - return db_event - - -def update_event(db_event, ical_event): - update_event_data(db_event, ical_event) - - -def add_event(event, calendar, session): - entry = Entry() - entry.uid = event.uid - entry.calendar = calendar - entry = update_event_data(entry, event) - session.add(entry) - - -def remove_event(event, session): - session.delete(event) - - -def get_last_modification_time(event): - for unused in event.__dict__['_unused']: - if unused.name == 'LAST-MODIFIED': - return dateparser.parse(unused.value) - return None - - -def fetch_calendar(calendar, session): - cal_data = requests.get(calendar.url) - cal_object = ics.Calendar(cal_data.content.decode(cal_data.encoding)) - cal_events = {e.uid: e for e in cal_object.events} - db_events = {e.uid: e for e in calendar.entries} - new_events = [e for u, e in cal_events.iteritems() - if u not in db_events.keys()] - old_events = [e for u, e in db_events.iteritems() - if u not in cal_events.keys()] - mod_events = [{'ics': cal_events[u], 'db': e} - for u, e in db_events.iteritems() if u in cal_events.keys()] - changes_present = False - for event in mod_events: - modified_date = get_last_modification_time(event['ics']) - if not modified_date or \ - not event['db'].last_modified or \ - modified_date > event['db'].last_modified.replace(tzinfo=pytz.UTC): - print 'Updating event %s' % event['db'].uid - update_event(event['db'], event['ics']) - changes_present = True - for event in new_events: - print 'Adding event %s' % event.uid - add_event(event, calendar, session) - changes_present = True - for event in old_events: - print 'Removing event %s' % event.uid - remove_event(event, session) - changes_present = True - if changes_present: - calendar.last_updated = datetime.datetime.now() - - -def main(): - session = Session.create() - - calendars = session.query(Calendar).all() - for calendar in calendars: - # print 'Fetching %s' % calendar.url - fetch_calendar(calendar, session) - - session.commit() - - -if __name__ == '__main__': - main() diff --git a/app/python/generate_cal_urls.py b/app/python/generate_cal_urls.py deleted file mode 100644 index a79f9d0..0000000 --- a/app/python/generate_cal_urls.py +++ /dev/null @@ -1,17 +0,0 @@ -from slugify import slugify - -from rcal.db import Session -from rcal.model import Calendar - - -def main(): - session = Session.create() - for calendar in session.query(Calendar).all(): - calendar.custom_url = slugify( - calendar.custom_name - if calendar.custom_name - else calendar.name) - session.commit() - -if __name__ == '__main__': - main() diff --git a/app/python/import_cals.py b/app/python/import_cals.py deleted file mode 100644 index f0b5262..0000000 --- a/app/python/import_cals.py +++ /dev/null @@ -1,132 +0,0 @@ -import json -import os -import re -import time -import urllib2 -import urlparse - -from sqlalchemy import inspect - -from rcal.db import Session -from rcal.model import Calendar, Category - -BASEPATH = os.path.join( - os.environ['PYTHONPATH'], - '..', - '..') - -CONFIG = json.load(open( - os.path.join( - os.environ['PYTHONPATH'], - 'config', - 'reddit-import.json'), - 'r')) - - -def get_cal_list(): - cache_path = os.path.join( - BASEPATH, - CONFIG['cache']) - if not os.path.exists(cache_path) or \ - int(time.time()) - int(os.path.getmtime(cache_path)) > \ - CONFIG['cache_time']: - opener = urllib2.build_opener() - opener.addheaders = [('User-Agent', CONFIG['user_agent'])] - cal_list = json.loads(opener.open(CONFIG['reddit_url']).read()) - cal_list = cal_list['data']['content_md'] - with open(cache_path, 'w') as cache_file: - cache_file.write(cal_list) - cache_file.close() - else: - cal_list = open(cache_path, 'r').read() - return cal_list - - -def update_calendar(cal, session): - db_cal = Calendar.fetch(cal['uid'], session) - - if inspect(db_cal).pending: - print 'Adding calendar %s (%s)' % (cal['name'], cal['uid']) - db_cal.name = cal['name'] - db_cal.url = cal['url'] - db_cal.website = cal['website'] - db_cal.category = Category.fetch(cal['category'], session) - - if db_cal.name != cal['name']: - print 'Updating calendar name: %s -> %s (%s)' % ( - db_cal.name, cal['name'], db_cal.uid) - db_cal.name = cal['name'] - if db_cal.url != cal['url']: - print 'Updating calendar url: %s -> %s (%s)' % ( - db_cal.url, cal['url'], db_cal.uid) - db_cal.url = cal['url'] - if db_cal.website != cal['website']: - print 'Updating calendar website: %s -> %s (%s)' % ( - db_cal.website, cal['website'], db_cal.uid) - db_cal.website = cal['website'] - - if 'category_mapping' in CONFIG and \ - cal['category'] in CONFIG['category_mapping']: - cal['category'] = CONFIG['category_mapping'][cal['category']] - - # informational only - if db_cal.category.name != cal['category']: - print 'Calendar category changed: %s -> %s (%s, %s)' % ( - db_cal.category.name, cal['category'], db_cal.name, db_cal.uid) - - -def get_imported_calendars(cells, ical_markdown): - imported_calendars = [] - for row in cells: - row = [r for r in row if r.strip()] - if len(row) == 5: - markdown_match = re.match(ical_markdown, row[2]) - if markdown_match: - ical_url = urlparse.urlparse(markdown_match.group(1)) - if ical_url.netloc == 'calendar.google.com': - ical_path = re.sub( - '^/?calendar/ical/', '', ical_url.path).split('/') - if len(ical_path) == 3: - imported_calendars.append({ - 'uid': ical_path[0], - 'url': ical_url.geturl(), - 'name': row[0], - 'website': row[4].split()[0], - 'category': row[1] - }) - else: - print 'Unknown iCal URL format: %s' % ( - ical_url.geturl()) - else: - print 'Unknown iCal URL format: %s' % ( - ical_url.geturl()) - return imported_calendars - - -def main(): - session = Session.create() - - cal_list = get_cal_list() - - ical_markdown = re.compile(r'^\[iCal\]\((.*)\)$') - cells = [row.split('|') for row in cal_list.split('\n')] - - imported_calendars = get_imported_calendars(cells, ical_markdown) - imported_calendar_uids = [c['uid'] for c in imported_calendars] - - db_only_calendars = session.query(Calendar).filter( - ~Calendar.uid.in_(imported_calendar_uids)).all() - - if len(db_only_calendars): - print 'Local calendars not in remote source:' - for cal in db_only_calendars: - print '%s (%s)' % (cal.name, cal.uid) - print - - for cal in imported_calendars: - update_calendar(cal, session) - - session.commit() - -if __name__ == '__main__': - main() diff --git a/app/python/init_db.py b/app/python/init_db.py deleted file mode 100644 index 7d48e80..0000000 --- a/app/python/init_db.py +++ /dev/null @@ -1,16 +0,0 @@ -import sys - -from rcal.db import Session -from rcal.model import BASE - - -def main(): - session = Session.create() - - if len(sys.argv) > 1 and sys.argv[1] == 'force': - BASE.metadata.drop_all(session.get_bind()) - - BASE.metadata.create_all(session.get_bind()) - -if __name__ == '__main__': - main() diff --git a/app/python/rcal/__init__.py b/app/python/rcal/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/app/python/rcal/__init__.py +++ /dev/null diff --git a/app/python/rcal/db.py b/app/python/rcal/db.py deleted file mode 100644 index 7e892a1..0000000 --- a/app/python/rcal/db.py +++ /dev/null @@ -1,34 +0,0 @@ -import json -from os import path - -import sqlalchemy.engine.url as url -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - - -class Session(object): - - engine = None - - def __init__(self): - config = json.load( - open(path.join(path.dirname( - path.realpath(__file__)), '..', 'config', 'db.json'))) - db_str = url.URL( - drivername=config['type'], - host=config['host'], - username=config['user'], - password=config['pass'], - database=config['name'], - query={'charset': config['cset']} - ) - self.engine = create_engine(db_str, encoding=config['cset']) - - def get_maker(self): - return sessionmaker(bind=self.engine) - - @staticmethod - def create(): - session = Session() - maker = session.get_maker() - return maker() diff --git a/app/python/rcal/model.py b/app/python/rcal/model.py deleted file mode 100644 index 512c75e..0000000 --- a/app/python/rcal/model.py +++ /dev/null @@ -1,153 +0,0 @@ -# pylint: disable=too-few-public-methods, invalid-name -from sqlalchemy import Column, ForeignKey, Table -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship -from sqlalchemy.types import (TIMESTAMP, Boolean, DateTime, Integer, String, - TypeDecorator) - -from dateutil.tz import tzutc - - -# pylint: disable=abstract-method, unused-argument -class UTCDateTime(TypeDecorator): - impl = DateTime - - def process_bind_param(self, value, engine): - if value is not None: - return value.astimezone(tzutc()) - - def process_result_value(self, value, engine): - if value is not None: - return value.replace(tzinfo=tzutc()) - - -BASE = declarative_base() - - -class Calendar(BASE): - __tablename__ = 'calendars' - - uid = Column(String(255), primary_key=True) - url = Column(String(255)) - name = Column(String(255), index=True) - website = Column(String(255)) - visible = Column(Boolean, index=True) - custom_name = Column(String(255)) - custom_image = Column(String(255)) - custom_url = Column(String(255), unique=True, nullable=False, index=True) - last_updated = Column(TIMESTAMP) - - _category = Column( - Integer, - ForeignKey( - 'categories.id', - onupdate='CASCADE', - ondelete='SET NULL')) - category = relationship( - 'Category', - back_populates='calendars', - order_by='Calendar.name') - - entries = relationship( - 'Entry', - back_populates='calendar', - cascade="all", - passive_deletes=True, - order_by='Entry.begin_date') - - @staticmethod - def fetch(uid, session, name=None, url=None): - calendar = session.query(Calendar).filter(Calendar.uid == uid).first() - if not calendar: - calendar = Calendar() - calendar.uid = uid - session.add(calendar) - if name: - calendar.name = name - if url: - calendar.url = url - return calendar - - -class Entry(BASE): - __tablename__ = 'entries' - - id = Column(Integer, primary_key=True) - uid = Column(String(255), index=True, unique=True, nullable=False) - begin_date = Column(UTCDateTime, index=True) - end_date = Column(UTCDateTime) - all_day = Column(Boolean) - name = Column(String(255)) - location = Column(String(255)) - last_modified = Column(UTCDateTime) - - _calendar = Column( - String(255), - ForeignKey( - 'calendars.uid', - onupdate='CASCADE', - ondelete='CASCADE')) - calendar = relationship( - 'Calendar', - back_populates='entries', - order_by='Entry.begin_date') - - @staticmethod - def fetch(uid, session): - entry = session.query(Entry).filter(Entry.uid == uid).first() - if not entry: - entry = Entry() - session.add(entry) - return entry - - -class Category(BASE): - __tablename__ = 'categories' - - id = Column(Integer, primary_key=True) - name = Column(String(255), index=True) - priority = Column(Integer, index=True) - - calendars = relationship( - 'Calendar', - back_populates='category', - cascade="all", - passive_deletes=True, - order_by='Calendar.name') - - @staticmethod - def fetch(name, session): - category = session.query(Category).filter( - Category.name == name).first() - if not category: - category = Category() - category.name = name - session.add(category) - return category - - -user_selections = Table( - 'user_selections', BASE.metadata, - Column('_user', Integer, - ForeignKey('users.id', onupdate='CASCADE', ondelete='CASCADE'), - primary_key=True), - Column('_calendar', String(255), - ForeignKey('calendars.uid', onupdate='CASCADE', ondelete='CASCADE'), - primary_key=True) -) - - -class User(BASE): - __tablename__ = 'users' - - id = Column(Integer, primary_key=True) - login = Column(String(255), unique=True, index=True) - password = Column(String(255)) - is_admin = Column(Boolean) - timezone = Column(String(255)) - last_login = Column(UTCDateTime) - - calendars = relationship('Calendar', - secondary=user_selections) - -__all__ = ('Calendar', 'Entry', 'Category', 'User') diff --git a/app/python/weekdays.py b/app/python/weekdays.py deleted file mode 100644 index 547f6b2..0000000 --- a/app/python/weekdays.py +++ /dev/null @@ -1,32 +0,0 @@ -import json -import os -import urllib - -from lxml import etree - - -def main(): - week_day_config = {} - - supplemental_data = etree.fromstring( - urllib.urlopen( - 'http://unicode.org/repos/cldr/trunk/common/supplemental/' + - 'supplementalData.xml' - ).read()) - for first_day in supplemental_data.xpath('weekData/firstDay[not(@alt)]'): - day = first_day.get('day') - territories = first_day.get('territories').split() - for territory in territories: - week_day_config[territory] = day - - json.dump(week_day_config, - file(os.path.join( - os.environ['PYTHONPATH'], - 'config', - 'weekdays.json'), 'w'), - sort_keys=True, - indent=4, - separators=(',', ': ')) - -if __name__ == '__main__': - main() |