diff options
| -rw-r--r-- | Makefile | 16 | ||||
| -rwxr-xr-x | bin/datafiles-build.sh | 4 | ||||
| -rwxr-xr-x | bin/group-tools-json.sh | 2 | ||||
| -rwxr-xr-x | bin/menus-build.sh | 2 | ||||
| -rwxr-xr-x | bin/rankings-tables-generate.sh | 2 | ||||
| -rwxr-xr-x | bin/statics-generate.sh | 4 | ||||
| -rw-r--r-- | docs/scripts.md | 2 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | scripts/datafiles-generate-minimal.py | 2 | ||||
| -rw-r--r-- | scripts/datafiles-generate.py | 2 | ||||
| -rw-r--r-- | scripts/group-tools-json-generate.py | 2 | ||||
| -rw-r--r-- | scripts/menus-compile.py | 4 | ||||
| -rw-r--r-- | scripts/menus-write.py | 6 | ||||
| -rw-r--r-- | scripts/players-compile.py | 10 | ||||
| -rw-r--r-- | scripts/players-prepare-template.py | 6 | ||||
| -rw-r--r-- | scripts/pyranking/db.py | 2 | ||||
| -rw-r--r-- | scripts/rankings-csv-convert.py | 4 | ||||
| -rw-r--r-- | scripts/rankings-editions.py | 10 | ||||
| -rw-r--r-- | scripts/rankings-tables-compile.py | 8 | ||||
| -rw-r--r-- | scripts/statics-compile.py | 6 |
20 files changed, 48 insertions, 47 deletions
@@ -20,33 +20,33 @@ tables: editions: $(rankfiles) $(rankfiles): - python scripts/rankings-editions.py $(patsubst %.ed,%,$@) + python3 scripts/rankings-editions.py $(patsubst %.ed,%,$@) players: player-pages player-data player-pages: menus mkdir -p http/players - python scripts/players-prepare-template.py http/players http/players/.menu.html + python3 scripts/players-prepare-template.py http/players http/players/.menu.html player-data: - python scripts/players-compile.py config/dates.json http/players + python3 scripts/players-compile.py config/dates.json http/players statics: - python scripts/menus-compile.py config/static.json http http > http/.menu.html + python3 scripts/menus-compile.py config/static.json http http > http/.menu.html bin/statics-generate.sh config/static.json static http group-json: bin/group-tools-json.sh config/dates.json http group-tools: - python scripts/statics-compile.py static/group-intro.html static/group-form-loading.html static/group-form.html > http/ranking-grupowy.html - python scripts/menus-compile.py config/static.json http http > http/.menu.html - python scripts/menus-write.py http/ranking-grupowy.html http/.menu.html + python3 scripts/statics-compile.py static/group-intro.html static/group-form-loading.html static/group-form.html > http/ranking-grupowy.html + python3 scripts/menus-compile.py config/static.json http http > http/.menu.html + python3 scripts/menus-write.py http/ranking-grupowy.html http/.menu.html minimize: $(tmpfiles) $(tmpfiles): - python -m htmlmin.command $(patsubst %.tmp,%,$@) $@ + python3 -m htmlmin.command $(patsubst %.tmp,%,$@) $@ mv $@ $(patsubst %.tmp,%,$@) deploy: diff --git a/bin/datafiles-build.sh b/bin/datafiles-build.sh index ed9d8f9..f2c560d 100755 --- a/bin/datafiles-build.sh +++ b/bin/datafiles-build.sh @@ -6,7 +6,7 @@ cat $CONFIG | jq -r '.[] | .date' | while read DATE do - python scripts/datafiles-generate.py $DATE $PREV_DATE > "$DIR/$DATE.json" - python scripts/datafiles-generate-minimal.py $DATE > "$DIR/$DATE.minimal.json" + python3 scripts/datafiles-generate.py $DATE $PREV_DATE > "$DIR/$DATE.json" + python3 scripts/datafiles-generate-minimal.py $DATE > "$DIR/$DATE.minimal.json" PREV_DATE=$DATE done diff --git a/bin/group-tools-json.sh b/bin/group-tools-json.sh index e6f04c8..e32014e 100755 --- a/bin/group-tools-json.sh +++ b/bin/group-tools-json.sh @@ -2,4 +2,4 @@ CONFIG=$1 OUTPUT=$2 LAST_DATE=`cat $CONFIG | jq -r '.[] | .date' | sort | tail -n1` -python scripts/group-tools-json-generate.py $LAST_DATE > $OUTPUT/_data/group-data.json +python3 scripts/group-tools-json-generate.py $LAST_DATE > $OUTPUT/_data/group-data.json diff --git a/bin/menus-build.sh b/bin/menus-build.sh index 0ac3a92..1a5c2a9 100755 --- a/bin/menus-build.sh +++ b/bin/menus-build.sh @@ -5,5 +5,5 @@ mkdir -p $DIRECTORY/players find $DIRECTORY -type d -not -name _\* | while read HTMLDIR do - python scripts/menus-compile.py $CONFIG_FILE $DIRECTORY $HTMLDIR > $HTMLDIR/.menu.html + python3 scripts/menus-compile.py $CONFIG_FILE $DIRECTORY $HTMLDIR > $HTMLDIR/.menu.html done diff --git a/bin/rankings-tables-generate.sh b/bin/rankings-tables-generate.sh index 06743c8..71d02d9 100755 --- a/bin/rankings-tables-generate.sh +++ b/bin/rankings-tables-generate.sh @@ -7,7 +7,7 @@ INDEX=$4 AGEMENU=$5 NAME=$6 -python scripts/rankings-tables-compile.py "$NAME" $INDEX $DIR/.menu.html $DATE "$AGEMENU" > $DIR/$URL +python3 scripts/rankings-tables-compile.py "$NAME" $INDEX $DIR/.menu.html $DATE "$AGEMENU" > $DIR/$URL shift 6 LINKS=$@ diff --git a/bin/statics-generate.sh b/bin/statics-generate.sh index 9c2fa92..c75cc8c 100755 --- a/bin/statics-generate.sh +++ b/bin/statics-generate.sh @@ -10,7 +10,7 @@ cat $1 | read OUTPUT_FILE if [ -n "$CONTENT_FILE" ] then - python scripts/statics-compile.py $CONTENT_DIR/$CONTENT_FILE "$HEADER" > $OUTPUT_DIR/$OUTPUT_FILE - python scripts/menus-write.py $OUTPUT_DIR/$OUTPUT_FILE $OUTPUT_DIR/.menu.html + python3 scripts/statics-compile.py $CONTENT_DIR/$CONTENT_FILE "$HEADER" > $OUTPUT_DIR/$OUTPUT_FILE + python3 scripts/menus-write.py $OUTPUT_DIR/$OUTPUT_FILE $OUTPUT_DIR/.menu.html fi done diff --git a/docs/scripts.md b/docs/scripts.md index e01adfe..70b20f5 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -68,7 +68,7 @@ bin/hidden-import.sh data/hidden.csv followed by: ``` -python scripts/rankings-csv-convert.py data/rankings/imports/DATE.csv data/rankings/DATE.csv +python3 scripts/rankings-csv-convert.py data/rankings/imports/DATE.csv data/rankings/DATE.csv bin/rankings-csv-import.sh data/rankings/DATE.csv ``` diff --git a/requirements.txt b/requirements.txt index 1b75e27..17962bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ beautifulsoup4 htmlmin lxml +simplejson # mysql-connector==2.1.4 diff --git a/scripts/datafiles-generate-minimal.py b/scripts/datafiles-generate-minimal.py index 896a1ec..b6f34a5 100644 --- a/scripts/datafiles-generate-minimal.py +++ b/scripts/datafiles-generate-minimal.py @@ -12,4 +12,4 @@ for row in ranking: 'r': float(row['score']) } -print json.dumps(output) +print(json.dumps(output)) diff --git a/scripts/datafiles-generate.py b/scripts/datafiles-generate.py index 2115c20..6fda1f6 100644 --- a/scripts/datafiles-generate.py +++ b/scripts/datafiles-generate.py @@ -36,4 +36,4 @@ for row in ranking: if isinstance(row[field], Decimal): row[field] = float(row[field]) -print json.dumps(ranking) +print(json.dumps(ranking)) diff --git a/scripts/group-tools-json-generate.py b/scripts/group-tools-json-generate.py index 54bb26d..6f2b717 100644 --- a/scripts/group-tools-json-generate.py +++ b/scripts/group-tools-json-generate.py @@ -28,4 +28,4 @@ for row in cursor.fetchall(): 'rank': float(row['rank']) } -print json.dumps(result) +print(json.dumps(result)) diff --git a/scripts/menus-compile.py b/scripts/menus-compile.py index 7c9fd64..9e96118 100644 --- a/scripts/menus-compile.py +++ b/scripts/menus-compile.py @@ -6,7 +6,7 @@ config_file = sys.argv[1] base_directory = sys.argv[2] menu_directory = sys.argv[3] -for menu_item in json.load(file(config_file)): +for menu_item in json.load(open(config_file)): href = os.path.relpath( os.path.join(base_directory, menu_item['url']), menu_directory @@ -15,4 +15,4 @@ for menu_item in json.load(file(config_file)): link.a['href'] = href link.a['title'] = menu_item['header'] link.a.string = menu_item['label'] - print link.a + print(link.a) diff --git a/scripts/menus-write.py b/scripts/menus-write.py index e30c702..0a4f1a7 100644 --- a/scripts/menus-write.py +++ b/scripts/menus-write.py @@ -3,12 +3,12 @@ import copy, json, os, sys from bs4 import BeautifulSoup as bs4 content_file = sys.argv[1] -menu_content = file(sys.argv[2]) +menu_content = open(sys.argv[2]) -content = bs4(file(content_file), 'lxml') +content = bs4(open(content_file), 'lxml') for menu_container in content.select('.static-menu'): menu_container.clear() menu_container.append(bs4(menu_content, 'html.parser')) -file(content_file, 'w').write(content.prettify().encode('utf-8')) +open(content_file, 'w').write(content.prettify()) diff --git a/scripts/players-compile.py b/scripts/players-compile.py index 4007cfa..289643c 100644 --- a/scripts/players-compile.py +++ b/scripts/players-compile.py @@ -7,13 +7,13 @@ output_directory = sys.argv[2] pagesize = 100.0 dates = {} -for date_config in json.load(file(sys.argv[1])): +for date_config in json.load(open(sys.argv[1])): dates[date_config['date']] = date_config['url'] players = {} for date in sorted(dates.keys()): - for player, ranking in fetch_ranking(date, True).iteritems(): + for player, ranking in fetch_ranking(date, True).items(): if not ranking['hidden']: if player not in players: players[player] = {'rankings':{}} @@ -29,14 +29,14 @@ for date in sorted(dates.keys()): players[player]['rankings'][date][field + '-change-class'] = 'primary' pcount = 0 -for pid, player in players.iteritems(): +for pid, player in players.items(): player['url'] = 'https://msc.com.pl/cezar/?p=21&pid=%d' % (pid) for date in dates: if date not in player['rankings']: player['rankings'][date] = {'place': None} prev = None - for date, ranking in sorted(player['rankings'].iteritems(), lambda x,y: cmp(x[0], y[0])): + for date, ranking in [(d, player['rankings'][d]) for d in sorted(player['rankings'])]: if prev is not None and prev['place'] is not None and ranking['place'] is not None: ranking['change'] = prev['place'] - ranking['place'] for field in ['gender', 'age', 'region']: @@ -71,7 +71,7 @@ for pid, player in players.iteritems(): field, ranking[field] ) - json.dump(player, file(os.path.join(output_directory, '%d.json' % pid), 'w')) + json.dump(player, open(os.path.join(output_directory, '%d.json' % pid), 'w')) pcount += 1 sys.stdout.write("[%d/%d]\r" % (pcount, len(players))) diff --git a/scripts/players-prepare-template.py b/scripts/players-prepare-template.py index 3d81482..48632da 100644 --- a/scripts/players-prepare-template.py +++ b/scripts/players-prepare-template.py @@ -4,10 +4,10 @@ from bs4 import BeautifulSoup as bs4 output_directory = sys.argv[1] -template = bs4(file('templates/player.html'), 'lxml') +template = bs4(open('templates/player.html'), 'lxml') menu_file = sys.argv[2] -menu_content = bs4(file(menu_file), 'html.parser') +menu_content = bs4(open(menu_file), 'html.parser') menu = template.select('div.static-menu')[0] menu.append(copy.copy(menu_content)) -file(os.path.join(output_directory, 'index.html'), 'w').write(template.prettify().encode('utf-8')) +open(os.path.join(output_directory, 'index.html'), 'w').write(template.prettify()) diff --git a/scripts/pyranking/db.py b/scripts/pyranking/db.py index 16595d3..f30a555 100644 --- a/scripts/pyranking/db.py +++ b/scripts/pyranking/db.py @@ -1,7 +1,7 @@ import json import mysql.connector -settings = json.load(file('config/db.json')) +settings = json.load(open('config/db.json')) connection = mysql.connector.connect( user=settings['user'], diff --git a/scripts/rankings-csv-convert.py b/scripts/rankings-csv-convert.py index 4dcee92..6648b15 100644 --- a/scripts/rankings-csv-convert.py +++ b/scripts/rankings-csv-convert.py @@ -1,8 +1,8 @@ import csv, sys -data = list(csv.reader(file(sys.argv[1]), delimiter=";")) +data = list(csv.reader(open(sys.argv[1], encoding='cp1250'), delimiter=";")) -output = csv.writer(file(sys.argv[2], 'w')) +output = csv.writer(open(sys.argv[2], 'w')) date = data[1][3] for row in data[4:-1]: diff --git a/scripts/rankings-editions.py b/scripts/rankings-editions.py index de1f2d6..bdbe46f 100644 --- a/scripts/rankings-editions.py +++ b/scripts/rankings-editions.py @@ -3,8 +3,8 @@ from collections import OrderedDict from bs4 import BeautifulSoup as bs4 -dates_config = json.load(file('config/dates.json')) -output_file = bs4(file(sys.argv[1]), 'lxml') +dates_config = json.load(open('config/dates.json')) +output_file = bs4(open(sys.argv[1]), 'lxml') editions = OrderedDict() for date_config in dates_config: @@ -17,12 +17,12 @@ for date_config in dates_config: date_config['date'] )) -template = bs4(file('templates/ranking.html'), 'lxml') +template = bs4(open('templates/ranking.html'), 'lxml') date_group = template.select('#editions')[0].extract() year_group = date_group.select('div[role="group"]')[0].extract() ranking_link = year_group.select('.btn-default')[0].extract() -for year, dates in editions.iteritems(): +for year, dates in editions.items(): group = copy.copy(year_group) group.select('.year')[0].string = str(year) for date in dates[::-1]: @@ -34,4 +34,4 @@ for year, dates in editions.iteritems(): date_group.insert(0, group) output_file.select('#editions')[0].replace_with(date_group) -file(sys.argv[1], 'w').write(output_file.prettify().encode('utf-8')) +open(sys.argv[1], 'w').write(output_file.prettify()) diff --git a/scripts/rankings-tables-compile.py b/scripts/rankings-tables-compile.py index 0270c54..45f72b5 100644 --- a/scripts/rankings-tables-compile.py +++ b/scripts/rankings-tables-compile.py @@ -6,7 +6,7 @@ subtitle = 'notowanie %s (%s), stan na %s' % ( sys.argv[1], sys.argv[2], '.'.join(ranking_date.split('-')[::-1]) ) -table = bs4(file('templates/ranking.html'), 'lxml') +table = bs4(open('templates/ranking.html'), 'lxml') table.select('.page-header h2 small')[0].string = subtitle table.select('table.data-table')[0]['data-ranking'] = '_data/%s.json?%d' % ( @@ -22,14 +22,14 @@ rawlink = table.select('a#rawlink')[0] rawlink['href'] = '%s/%s.csv' % (rawlink['href'], ranking_date) age_menu = sys.argv[5] -age_menu_file = file('templates/menu-age-%s.html' % (age_menu)) +age_menu_file = open('templates/menu-age-%s.html' % (age_menu)) age_menu_placeholder = table.select('div[data-menu="age"]')[0] age_menu_placeholder.clear() age_menu_placeholder.append(bs4(age_menu_file, 'html.parser')) -menu_file = file(sys.argv[3]) +menu_file = open(sys.argv[3]) menu = table.select('div.static-menu')[0] menu.clear() menu.append(bs4(menu_file, 'html.parser')) -print table.prettify().encode('utf-8') +print(table.prettify()) diff --git a/scripts/statics-compile.py b/scripts/statics-compile.py index 16d82ae..dc2f53a 100644 --- a/scripts/statics-compile.py +++ b/scripts/statics-compile.py @@ -15,7 +15,7 @@ while True: if len(arguments) == 0: break -template = bs4(file('templates/static.html'), 'lxml') +template = bs4(open('templates/static.html'), 'lxml') content_wrapper = template.find('div', {'id': 'wrapper'}).extract() del content_wrapper['id'] @@ -27,8 +27,8 @@ footer = template.find('div', {'id': 'footer'}) for content_file in content_files: content = copy.copy(content_wrapper) content.div.append( - bs4(file(content_file).read(), 'html.parser') + bs4(open(content_file).read(), 'html.parser') ) footer.insert_before(content) -print template.prettify().encode('utf-8') +print(template.prettify()) |