From 8f795b1d445441daab5ffb75060a6dbf8ed0934a Mon Sep 17 00:00:00 2001 From: emkael Date: Fri, 8 Sep 2017 01:28:21 +0200 Subject: Renaming scripts so that their names make sense --- Makefile | 30 +++++------ _cron/cezar-db | 2 +- bin/build-datafiles.sh | 11 ---- bin/build-players.sh | 4 -- bin/build-rankings.sh | 10 ---- bin/cezar-db-fetch.sh | 13 +++++ bin/cezar-db-import.sh | 9 ++++ bin/convert-cezar-ranking-csv.py | 9 ---- bin/datafiles-build.sh | 11 ++++ bin/fetch-cezar-db.sh | 13 ----- bin/generate-json.sh | 5 -- bin/generate-statics.sh | 16 ------ bin/group-tools-json.sh | 5 ++ bin/import-cezar-db.sh | 9 ---- bin/import-rankings.sh | 16 ------ bin/menus-build.sh | 9 ++++ bin/players-build.sh | 4 ++ bin/rankings-csv-import.sh | 16 ++++++ bin/rankings-tables-build.sh | 10 ++++ bin/rankings-target-files.sh | 3 ++ bin/statics-generate.sh | 16 ++++++ bin/target-ranking-files.sh | 3 -- bin/write-menus.sh | 9 ---- scripts/datafile.py | 30 ----------- scripts/datafiles-generate.py | 30 +++++++++++ scripts/editions.py | 36 ------------- scripts/generate-json.py | 31 ------------ scripts/generate-static-menu.py | 18 ------- scripts/group-tools-json-generate.py | 31 ++++++++++++ scripts/menus-compile.py | 18 +++++++ scripts/menus-write.py | 14 ++++++ scripts/players-compile.py | 98 ++++++++++++++++++++++++++++++++++++ scripts/players.py | 98 ------------------------------------ scripts/ranking.py | 26 ---------- scripts/rankings-csv-convert.py | 9 ++++ scripts/rankings-editions.py | 36 +++++++++++++ scripts/rankings-tables-compile.py | 26 ++++++++++ scripts/static-menu.py | 14 ------ scripts/static.py | 34 ------------- scripts/statics-compile.py | 34 +++++++++++++ 40 files changed, 408 insertions(+), 408 deletions(-) delete mode 100755 bin/build-datafiles.sh delete mode 100755 bin/build-players.sh delete mode 100755 bin/build-rankings.sh create mode 100755 bin/cezar-db-fetch.sh create mode 100755 bin/cezar-db-import.sh delete mode 100644 bin/convert-cezar-ranking-csv.py create mode 100755 bin/datafiles-build.sh delete mode 100755 bin/fetch-cezar-db.sh delete mode 100755 bin/generate-json.sh delete mode 100755 bin/generate-statics.sh create mode 100755 bin/group-tools-json.sh delete mode 100755 bin/import-cezar-db.sh delete mode 100755 bin/import-rankings.sh create mode 100755 bin/menus-build.sh create mode 100755 bin/players-build.sh create mode 100755 bin/rankings-csv-import.sh create mode 100755 bin/rankings-tables-build.sh create mode 100755 bin/rankings-target-files.sh create mode 100755 bin/statics-generate.sh delete mode 100755 bin/target-ranking-files.sh delete mode 100755 bin/write-menus.sh delete mode 100644 scripts/datafile.py create mode 100644 scripts/datafiles-generate.py delete mode 100644 scripts/editions.py delete mode 100644 scripts/generate-json.py delete mode 100644 scripts/generate-static-menu.py create mode 100644 scripts/group-tools-json-generate.py create mode 100644 scripts/menus-compile.py create mode 100644 scripts/menus-write.py create mode 100644 scripts/players-compile.py delete mode 100644 scripts/players.py delete mode 100644 scripts/ranking.py create mode 100644 scripts/rankings-csv-convert.py create mode 100644 scripts/rankings-editions.py create mode 100644 scripts/rankings-tables-compile.py delete mode 100644 scripts/static-menu.py delete mode 100644 scripts/static.py create mode 100644 scripts/statics-compile.py diff --git a/Makefile b/Makefile index 7ccc986..e108664 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,39 @@ all: statics rankings players -targetfiles := $(shell bin/target-ranking-files.sh config/dates.json) +targetfiles := $(shell bin/rankings-target-files.sh config/dates.json) tmpfiles := $(patsubst %.html,http/%.html.tmp,$(targetfiles)) rankfiles := $(patsubst %.html,http/%.html.ed,$(targetfiles)) -rankings: datafiles menus tables editions json +rankings: datafiles menus tables editions group-json datafiles: - bin/build-datafiles.sh config/dates.json http/_data + bin/datafiles-build.sh config/dates.json http/_data menus: - bin/write-menus.sh config/static.json http + bin/menus-build.sh config/static.json http tables: - bin/build-rankings.sh config/dates.json http + bin/rankings-tables-build.sh config/dates.json http editions: $(rankfiles) $(rankfiles): - python scripts/editions.py $(patsubst %.ed,%,$@) - -json: - bin/generate-json.sh config/dates.json http + python scripts/rankings-editions.py $(patsubst %.ed,%,$@) players: - bin/build-players.sh http + bin/players-build.sh http statics: - python scripts/generate-static-menu.py config/static.json http http > http/.menu.html - bin/generate-statics.sh config/static.json static http + python scripts/menus-compile.py config/static.json http http > http/.menu.html + bin/statics-generate.sh config/static.json static http + +group-json: + bin/group-tools-json.sh config/dates.json http group-tools: - python scripts/static.py static/group-intro.html static/group-form-loading.html static/group-form.html > http/ranking-grupowy.html - python scripts/generate-static-menu.py config/static.json http http > http/.menu.html - python scripts/static-menu.py http/ranking-grupowy.html http/.menu.html + python scripts/statics-compile.py static/group-intro.html static/group-form-loading.html static/group-form.html > http/ranking-grupowy.html + python scripts/menus-compile.py config/static.json http http > http/.menu.html + python scripts/menus-write.py http/ranking-grupowy.html http/.menu.html minimize: $(tmpfiles) diff --git a/_cron/cezar-db b/_cron/cezar-db index ffa3271..8a16851 100644 --- a/_cron/cezar-db +++ b/_cron/cezar-db @@ -1 +1 @@ -0 1 1 1,4,7,10 * $SITEPATH/bin/fetch-cezar-db.sh +0 1 1 1,4,7,10 * $SITEPATH/bin/cezar-db-fetch.sh diff --git a/bin/build-datafiles.sh b/bin/build-datafiles.sh deleted file mode 100755 index 82c2e9c..0000000 --- a/bin/build-datafiles.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -CONFIG=$1 -DIR=$2 -PREV_DATE="" -cat $CONFIG | - jq -r '.[] | .date' | - while read DATE - do - python scripts/datafile.py $DATE $PREV_DATE > "$DIR/$DATE.json" - PREV_DATE=$DATE - done diff --git a/bin/build-players.sh b/bin/build-players.sh deleted file mode 100755 index e1d099d..0000000 --- a/bin/build-players.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -DIR=$1 -mkdir -p $DIR/players -python scripts/players.py $DIR/players $DIR/players/.menu.html diff --git a/bin/build-rankings.sh b/bin/build-rankings.sh deleted file mode 100755 index c51fa1d..0000000 --- a/bin/build-rankings.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -CONFIG=$1 -DIR=$2 -cat $CONFIG | - jq '.[] | .date, .url, .index, .name' | - xargs -n4 | - while read DATE URL INDEX NAME - do - python scripts/ranking.py "$NAME" $INDEX $DIR/.menu.html $DATE > $DIR/$URL - done diff --git a/bin/cezar-db-fetch.sh b/bin/cezar-db-fetch.sh new file mode 100755 index 0000000..3af7357 --- /dev/null +++ b/bin/cezar-db-fetch.sh @@ -0,0 +1,13 @@ +#!/bin/bash +PAGEDIR=`dirname $0` +DATE=`date --date='yesterday' +%Y%m%d` +OUTPUTFILE=`realpath $PAGEDIR/../data/cezar/$DATE.csv` +TMPFILE=$PAGEDIR/temp +wget http://msc.com.pl/cezar/download/baza.csv -q -O - | iconv -f windows-1250 -t utf-8 > $TMPFILE +if [ -s $TMPFILE ]; then + tail -n +2 $TMPFILE | sort > $OUTPUTFILE + echo "Downloaded Cezar CSV into $OUTPUTFILE" +else + echo 'Cezar CSV fetch failed' +fi +rm $TMPFILE diff --git a/bin/cezar-db-import.sh b/bin/cezar-db-import.sh new file mode 100755 index 0000000..4db5261 --- /dev/null +++ b/bin/cezar-db-import.sh @@ -0,0 +1,9 @@ +#!/bin/bash +DBCONFIG=($(jq -r '.[]' config/import-db.json)) +mysql -h ${DBCONFIG[2]} -P ${DBCONFIG[3]} -u ${DBCONFIG[0]} --password=${DBCONFIG[1]} -e "SET foreign_key_checks = 0; +LOAD DATA LOCAL INFILE '$1' +REPLACE +INTO TABLE players +FIELDS TERMINATED BY ';' ENCLOSED BY '\"'; +SET foreign_key_checks = 1 +" ${DBCONFIG[4]} diff --git a/bin/convert-cezar-ranking-csv.py b/bin/convert-cezar-ranking-csv.py deleted file mode 100644 index 4dcee92..0000000 --- a/bin/convert-cezar-ranking-csv.py +++ /dev/null @@ -1,9 +0,0 @@ -import csv, sys - -data = list(csv.reader(file(sys.argv[1]), delimiter=";")) - -output = csv.writer(file(sys.argv[2], 'w')) - -date = data[1][3] -for row in data[4:-1]: - output.writerow([row[0], date, row[2], row[1]]) diff --git a/bin/datafiles-build.sh b/bin/datafiles-build.sh new file mode 100755 index 0000000..5c336ff --- /dev/null +++ b/bin/datafiles-build.sh @@ -0,0 +1,11 @@ +#!/bin/bash +CONFIG=$1 +DIR=$2 +PREV_DATE="" +cat $CONFIG | + jq -r '.[] | .date' | + while read DATE + do + python scripts/datafiles-generate.py $DATE $PREV_DATE > "$DIR/$DATE.json" + PREV_DATE=$DATE + done diff --git a/bin/fetch-cezar-db.sh b/bin/fetch-cezar-db.sh deleted file mode 100755 index 3af7357..0000000 --- a/bin/fetch-cezar-db.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -PAGEDIR=`dirname $0` -DATE=`date --date='yesterday' +%Y%m%d` -OUTPUTFILE=`realpath $PAGEDIR/../data/cezar/$DATE.csv` -TMPFILE=$PAGEDIR/temp -wget http://msc.com.pl/cezar/download/baza.csv -q -O - | iconv -f windows-1250 -t utf-8 > $TMPFILE -if [ -s $TMPFILE ]; then - tail -n +2 $TMPFILE | sort > $OUTPUTFILE - echo "Downloaded Cezar CSV into $OUTPUTFILE" -else - echo 'Cezar CSV fetch failed' -fi -rm $TMPFILE diff --git a/bin/generate-json.sh b/bin/generate-json.sh deleted file mode 100755 index d856a89..0000000 --- a/bin/generate-json.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -CONFIG=$1 -OUTPUT=$2 -LAST_DATE=`cat $CONFIG | jq -r '.[] | .date' | sort | tail -n1` -python scripts/generate-json.py $LAST_DATE > $OUTPUT/_data/group-data.json diff --git a/bin/generate-statics.sh b/bin/generate-statics.sh deleted file mode 100755 index 4369aa7..0000000 --- a/bin/generate-statics.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -CONFIG=$1 -CONTENT_DIR=$2 -OUTPUT_DIR=$3 -cat $1 | - jq -r '.[] | .content, .header, .url' | - while read CONTENT_FILE - do - read HEADER - read OUTPUT_FILE - if [ -n "$CONTENT_FILE" ] - then - python scripts/static.py $CONTENT_DIR/$CONTENT_FILE "$HEADER" > $OUTPUT_DIR/$OUTPUT_FILE - python scripts/static-menu.py $OUTPUT_DIR/$OUTPUT_FILE $OUTPUT_DIR/.menu.html - fi - done diff --git a/bin/group-tools-json.sh b/bin/group-tools-json.sh new file mode 100755 index 0000000..e6f04c8 --- /dev/null +++ b/bin/group-tools-json.sh @@ -0,0 +1,5 @@ +#!/bin/bash +CONFIG=$1 +OUTPUT=$2 +LAST_DATE=`cat $CONFIG | jq -r '.[] | .date' | sort | tail -n1` +python scripts/group-tools-json-generate.py $LAST_DATE > $OUTPUT/_data/group-data.json diff --git a/bin/import-cezar-db.sh b/bin/import-cezar-db.sh deleted file mode 100755 index 4db5261..0000000 --- a/bin/import-cezar-db.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -DBCONFIG=($(jq -r '.[]' config/import-db.json)) -mysql -h ${DBCONFIG[2]} -P ${DBCONFIG[3]} -u ${DBCONFIG[0]} --password=${DBCONFIG[1]} -e "SET foreign_key_checks = 0; -LOAD DATA LOCAL INFILE '$1' -REPLACE -INTO TABLE players -FIELDS TERMINATED BY ';' ENCLOSED BY '\"'; -SET foreign_key_checks = 1 -" ${DBCONFIG[4]} diff --git a/bin/import-rankings.sh b/bin/import-rankings.sh deleted file mode 100755 index 90ce6bc..0000000 --- a/bin/import-rankings.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -DBCONFIG=($(jq -r '.[]' config/import-db.json)) -mysql -h ${DBCONFIG[2]} -P ${DBCONFIG[3]} -u ${DBCONFIG[0]} --password=${DBCONFIG[1]} -e "SET foreign_key_checks = 0; -DELETE FROM temp_rankings; -LOAD DATA LOCAL INFILE '$1' -REPLACE -INTO TABLE temp_rankings -FIELDS TERMINATED BY ','; -INSERT INTO rankings ( - SELECT pid, \`date\`, place, score, region, flags, rank, club - FROM temp_rankings - JOIN players - ON players.id = temp_rankings.pid -); -SET foreign_key_checks = 1 -" ${DBCONFIG[4]} diff --git a/bin/menus-build.sh b/bin/menus-build.sh new file mode 100755 index 0000000..0ac3a92 --- /dev/null +++ b/bin/menus-build.sh @@ -0,0 +1,9 @@ +#!/bin/bash +CONFIG_FILE=$1 +DIRECTORY=$2 +mkdir -p $DIRECTORY/players +find $DIRECTORY -type d -not -name _\* | + while read HTMLDIR + do + python scripts/menus-compile.py $CONFIG_FILE $DIRECTORY $HTMLDIR > $HTMLDIR/.menu.html + done diff --git a/bin/players-build.sh b/bin/players-build.sh new file mode 100755 index 0000000..06fdf4f --- /dev/null +++ b/bin/players-build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +DIR=$1 +mkdir -p $DIR/players +python scripts/players-compile.py $DIR/players $DIR/players/.menu.html diff --git a/bin/rankings-csv-import.sh b/bin/rankings-csv-import.sh new file mode 100755 index 0000000..90ce6bc --- /dev/null +++ b/bin/rankings-csv-import.sh @@ -0,0 +1,16 @@ +#!/bin/bash +DBCONFIG=($(jq -r '.[]' config/import-db.json)) +mysql -h ${DBCONFIG[2]} -P ${DBCONFIG[3]} -u ${DBCONFIG[0]} --password=${DBCONFIG[1]} -e "SET foreign_key_checks = 0; +DELETE FROM temp_rankings; +LOAD DATA LOCAL INFILE '$1' +REPLACE +INTO TABLE temp_rankings +FIELDS TERMINATED BY ','; +INSERT INTO rankings ( + SELECT pid, \`date\`, place, score, region, flags, rank, club + FROM temp_rankings + JOIN players + ON players.id = temp_rankings.pid +); +SET foreign_key_checks = 1 +" ${DBCONFIG[4]} diff --git a/bin/rankings-tables-build.sh b/bin/rankings-tables-build.sh new file mode 100755 index 0000000..0d85972 --- /dev/null +++ b/bin/rankings-tables-build.sh @@ -0,0 +1,10 @@ +#!/bin/bash +CONFIG=$1 +DIR=$2 +cat $CONFIG | + jq '.[] | .date, .url, .index, .name' | + xargs -n4 | + while read DATE URL INDEX NAME + do + python scripts/rankings-tables-compile.py "$NAME" $INDEX $DIR/.menu.html $DATE > $DIR/$URL + done diff --git a/bin/rankings-target-files.sh b/bin/rankings-target-files.sh new file mode 100755 index 0000000..001834f --- /dev/null +++ b/bin/rankings-target-files.sh @@ -0,0 +1,3 @@ +#!/bin/bash +CONFIG=$1 +jq -jr '.[] | .url + " "' $CONFIG diff --git a/bin/statics-generate.sh b/bin/statics-generate.sh new file mode 100755 index 0000000..9c2fa92 --- /dev/null +++ b/bin/statics-generate.sh @@ -0,0 +1,16 @@ +#!/bin/bash +CONFIG=$1 +CONTENT_DIR=$2 +OUTPUT_DIR=$3 +cat $1 | + jq -r '.[] | .content, .header, .url' | + while read CONTENT_FILE + do + read HEADER + read OUTPUT_FILE + if [ -n "$CONTENT_FILE" ] + then + python scripts/statics-compile.py $CONTENT_DIR/$CONTENT_FILE "$HEADER" > $OUTPUT_DIR/$OUTPUT_FILE + python scripts/menus-write.py $OUTPUT_DIR/$OUTPUT_FILE $OUTPUT_DIR/.menu.html + fi + done diff --git a/bin/target-ranking-files.sh b/bin/target-ranking-files.sh deleted file mode 100755 index 001834f..0000000 --- a/bin/target-ranking-files.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -CONFIG=$1 -jq -jr '.[] | .url + " "' $CONFIG diff --git a/bin/write-menus.sh b/bin/write-menus.sh deleted file mode 100755 index 500db52..0000000 --- a/bin/write-menus.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -CONFIG_FILE=$1 -DIRECTORY=$2 -mkdir -p $DIRECTORY/players -find $DIRECTORY -type d -not -name _\* | - while read HTMLDIR - do - python scripts/generate-static-menu.py $CONFIG_FILE $DIRECTORY $HTMLDIR > $HTMLDIR/.menu.html - done diff --git a/scripts/datafile.py b/scripts/datafile.py deleted file mode 100644 index 5cee451..0000000 --- a/scripts/datafile.py +++ /dev/null @@ -1,30 +0,0 @@ -import json, sys -from decimal import Decimal - -from pyranking.fetch import fetch_ranking - -ranking_date = sys.argv[1] -ranking = fetch_ranking(ranking_date) -old_ranking = fetch_ranking(sys.argv[2], True) if len(sys.argv) > 2 else {} - -for row in ranking: - if row['pid'] in old_ranking: - row['place-change'] = old_ranking[row['pid']]['place'] - row['place'] - row['place-change-class'] = 'success' if row['place-change'] > 0 else 'danger' - row['place-change'] = '%+d' % (row['place-change']) - if row['place-change'] == '+0': - row['place-change'] = '=' - row['place-change-class'] = 'default' - for category in ['gender', 'age', 'region']: - if row[category] == old_ranking[row['pid']][category]: - row[category + '-change'] = old_ranking[row['pid']][category + '-place'] - row[category + '-place'] - row[category + '-change-class'] = 'success' if row[category + '-change'] > 0 else 'danger' - row[category + '-change'] = '%+d' % (row[category + '-change']) - if row[category + '-change'] == '+0': - row[category + '-change'] = '=' - row[category + '-change-class'] = 'default' - for field in row: - if isinstance(row[field], Decimal): - row[field] = float(row[field]) - -print json.dumps(ranking) diff --git a/scripts/datafiles-generate.py b/scripts/datafiles-generate.py new file mode 100644 index 0000000..5cee451 --- /dev/null +++ b/scripts/datafiles-generate.py @@ -0,0 +1,30 @@ +import json, sys +from decimal import Decimal + +from pyranking.fetch import fetch_ranking + +ranking_date = sys.argv[1] +ranking = fetch_ranking(ranking_date) +old_ranking = fetch_ranking(sys.argv[2], True) if len(sys.argv) > 2 else {} + +for row in ranking: + if row['pid'] in old_ranking: + row['place-change'] = old_ranking[row['pid']]['place'] - row['place'] + row['place-change-class'] = 'success' if row['place-change'] > 0 else 'danger' + row['place-change'] = '%+d' % (row['place-change']) + if row['place-change'] == '+0': + row['place-change'] = '=' + row['place-change-class'] = 'default' + for category in ['gender', 'age', 'region']: + if row[category] == old_ranking[row['pid']][category]: + row[category + '-change'] = old_ranking[row['pid']][category + '-place'] - row[category + '-place'] + row[category + '-change-class'] = 'success' if row[category + '-change'] > 0 else 'danger' + row[category + '-change'] = '%+d' % (row[category + '-change']) + if row[category + '-change'] == '+0': + row[category + '-change'] = '=' + row[category + '-change-class'] = 'default' + for field in row: + if isinstance(row[field], Decimal): + row[field] = float(row[field]) + +print json.dumps(ranking) diff --git a/scripts/editions.py b/scripts/editions.py deleted file mode 100644 index 42d036f..0000000 --- a/scripts/editions.py +++ /dev/null @@ -1,36 +0,0 @@ -import copy, json, sys - -from bs4 import BeautifulSoup as bs4 - -dates_config = json.load(file('config/dates.json')) -output_file = bs4(file(sys.argv[1]), 'lxml') - -editions = {} -for date_config in dates_config: - year = date_config['name'].split(' ')[1] - if year not in editions: - editions[year] = [] - editions[year].append(( - '%s (%s)' % (date_config['name'].split(' ')[0], date_config['index']), - date_config['url'], - date_config['date'] - )) - -template = bs4(file('templates/ranking.html'), 'lxml') - -date_group = template.select('#editions')[0].extract() -year_group = date_group.select('div[role="group"]')[0].extract() -ranking_link = year_group.select('.btn-default')[0].extract() -for year, dates in editions.iteritems(): - group = copy.copy(year_group) - group.select('.year')[0].string = str(year) - for date in dates[::-1]: - link = copy.copy(ranking_link) - link.string = date[0] - link['href'] = date[1] - link['datetime'] = date[2] - group.append(link) - date_group.append(group) - -output_file.select('#editions')[0].replace_with(date_group) -file(sys.argv[1], 'w').write(output_file.prettify().encode('utf-8')) diff --git a/scripts/generate-json.py b/scripts/generate-json.py deleted file mode 100644 index 54bb26d..0000000 --- a/scripts/generate-json.py +++ /dev/null @@ -1,31 +0,0 @@ -import json, sys -from pyranking.db import cursor - -date = sys.argv[1] - -sql = '''SELECT - rankings.place, - players.id, - players.rank, - rankings.score -FROM players -LEFT JOIN rankings - ON players.id = rankings.pid - AND rankings.date = %(date)s -''' -cursor.execute(sql, {'date': date}) - -result = {} -for row in cursor.fetchall(): - if row['place'] is not None: - result[row['id']] = { - 'place': int(row['place']), - 'rank': float(row['rank']), - 'score': float(row['score']) - } - else: - result[row['id']] = { - 'rank': float(row['rank']) - } - -print json.dumps(result) diff --git a/scripts/generate-static-menu.py b/scripts/generate-static-menu.py deleted file mode 100644 index 7c9fd64..0000000 --- a/scripts/generate-static-menu.py +++ /dev/null @@ -1,18 +0,0 @@ -import json, os, sys - -from bs4 import BeautifulSoup as bs4 - -config_file = sys.argv[1] -base_directory = sys.argv[2] -menu_directory = sys.argv[3] - -for menu_item in json.load(file(config_file)): - href = os.path.relpath( - os.path.join(base_directory, menu_item['url']), - menu_directory - ) - link = bs4('', 'html.parser') - link.a['href'] = href - link.a['title'] = menu_item['header'] - link.a.string = menu_item['label'] - print link.a diff --git a/scripts/group-tools-json-generate.py b/scripts/group-tools-json-generate.py new file mode 100644 index 0000000..54bb26d --- /dev/null +++ b/scripts/group-tools-json-generate.py @@ -0,0 +1,31 @@ +import json, sys +from pyranking.db import cursor + +date = sys.argv[1] + +sql = '''SELECT + rankings.place, + players.id, + players.rank, + rankings.score +FROM players +LEFT JOIN rankings + ON players.id = rankings.pid + AND rankings.date = %(date)s +''' +cursor.execute(sql, {'date': date}) + +result = {} +for row in cursor.fetchall(): + if row['place'] is not None: + result[row['id']] = { + 'place': int(row['place']), + 'rank': float(row['rank']), + 'score': float(row['score']) + } + else: + result[row['id']] = { + 'rank': float(row['rank']) + } + +print json.dumps(result) diff --git a/scripts/menus-compile.py b/scripts/menus-compile.py new file mode 100644 index 0000000..7c9fd64 --- /dev/null +++ b/scripts/menus-compile.py @@ -0,0 +1,18 @@ +import json, os, sys + +from bs4 import BeautifulSoup as bs4 + +config_file = sys.argv[1] +base_directory = sys.argv[2] +menu_directory = sys.argv[3] + +for menu_item in json.load(file(config_file)): + href = os.path.relpath( + os.path.join(base_directory, menu_item['url']), + menu_directory + ) + link = bs4('', 'html.parser') + link.a['href'] = href + link.a['title'] = menu_item['header'] + link.a.string = menu_item['label'] + print link.a diff --git a/scripts/menus-write.py b/scripts/menus-write.py new file mode 100644 index 0000000..e30c702 --- /dev/null +++ b/scripts/menus-write.py @@ -0,0 +1,14 @@ +import copy, json, os, sys + +from bs4 import BeautifulSoup as bs4 + +content_file = sys.argv[1] +menu_content = file(sys.argv[2]) + +content = bs4(file(content_file), 'lxml') + +for menu_container in content.select('.static-menu'): + menu_container.clear() + menu_container.append(bs4(menu_content, 'html.parser')) + +file(content_file, 'w').write(content.prettify().encode('utf-8')) diff --git a/scripts/players-compile.py b/scripts/players-compile.py new file mode 100644 index 0000000..613c01f --- /dev/null +++ b/scripts/players-compile.py @@ -0,0 +1,98 @@ +import copy, json, os, sys +from bs4 import BeautifulSoup as bs4 +from math import ceil +from pyranking.fetch import fetch_ranking + +output_directory = sys.argv[1] +pagesize = 100.0 + +menu_file = sys.argv[2] +menu_content = bs4(file(menu_file), 'html.parser') + +dates = {} +for date_config in json.load(file('config/dates.json')): + dates[date_config['date']] = date_config['url'] + +players = {} + +for date in sorted(dates.keys()): + for player, ranking in fetch_ranking(date, True).iteritems(): + if player not in players: + players[player] = {'rankings':{}} + players[player]['name'] = ranking['player'] + players[player]['club'] = ranking['club'] + players[player]['rankings'][date] = {'change': 'N','change-class':'primary'} + for field in ['place', 'score']: + players[player]['rankings'][date][field] = ranking[field] + for field in ['gender', 'age', 'region']: + players[player]['rankings'][date][field] = ranking[field] + players[player]['rankings'][date][field + '-place'] = ranking[field + '-place'] + players[player]['rankings'][date][field + '-change'] = 'N' + players[player]['rankings'][date][field + '-change-class'] = 'primary' + +for pid, player in players.iteritems(): + template = bs4(file('templates/player.html'), 'lxml') + template.select('h2.name')[0].insert(0, player['name']) + template.select('h3.club')[0].string = player['club'] + template.select('a.pid-link')[0]['href'] = 'https://msc.com.pl/cezar/?p=21&pid=%d' % (pid) + + missing_row = template.select('tr.missing')[0].extract() + normal_row = template.select('tr.normal')[0].extract() + + for date in dates: + if date not in player['rankings']: + player['rankings'][date] = None + prev = None + for date, ranking in sorted(player['rankings'].iteritems(), lambda x,y: cmp(x[0], y[0])): + if prev is not None and ranking is not None: + ranking['change'] = prev['place'] - ranking['place'] + for field in ['gender', 'age', 'region']: + if prev[field] == ranking[field]: + ranking[field + '-change'] = prev[field + '-place'] - ranking[field + '-place'] + for field in ['', 'gender-', 'age-', 'region-']: + if ranking[field+'change'] == 0: + ranking[field+'change'] = '=' + ranking[field+'change-class'] = 'default' + elif ranking[field+'change'] == 'N': + ranking[field+'change-class'] = 'primary' + else: + ranking[field+'change-class'] = 'success' if ranking[field+'change'] > 0 else 'danger' + ranking[field+'change'] = '%+d' % (ranking[field+'change']) + prev = ranking + + row = copy.copy(missing_row) if ranking is None else copy.copy(normal_row) + rank_link = row.find('td').a + rank_link.string = '.'.join(date.split('-')[::-1]) + base_rank_link = '../%s' % (dates[date]) + if ranking is not None and ranking['place'] > int(pagesize): + rank_link['href'] = '../%s#page:%d' % ( + dates[date], ceil(ranking['place'] / pagesize) + ) + else: + rank_link['href'] = base_rank_link + if ranking is not None: + score_cell = row.select('.score span')[0] + score_cell.string = '%.2f' % (ranking['score']) + score_cell['title'] = str(ranking['score']) + for field in ['region', 'age', 'gender']: + link = row.select('td.'+field+' a')[0] + if ranking[field+'-place'] > int(pagesize): + link['href'] = base_rank_link + '#%s:%s;page:%d' % ( + field, ranking[field], ceil(ranking[field+'-place'] / pagesize) + ) + else: + link['href'] = base_rank_link + '#%s:%s' % ( + field, ranking[field] + ) + link.string = ranking[field] if len(ranking[field]) else '-' + for field in ['', 'region-', 'age-', 'gender-']: + row.select('td.'+field+'place')[0].string = '%d.' % (ranking[field+'place']) + change_label = row.select('td.'+field+'place-change span.label')[0] + change_label.string = ranking[field+'change'] + change_label['class'] = change_label['class'] + ['label-'+ranking[field+'change-class']] + template.select('table.table tbody')[0].insert(0, row) + + menu = template.select('div.static-menu')[0] + menu.append(copy.copy(menu_content)) + + file(os.path.join(output_directory, '%d.html' % pid), 'w').write(template.prettify().encode('utf-8')) diff --git a/scripts/players.py b/scripts/players.py deleted file mode 100644 index 613c01f..0000000 --- a/scripts/players.py +++ /dev/null @@ -1,98 +0,0 @@ -import copy, json, os, sys -from bs4 import BeautifulSoup as bs4 -from math import ceil -from pyranking.fetch import fetch_ranking - -output_directory = sys.argv[1] -pagesize = 100.0 - -menu_file = sys.argv[2] -menu_content = bs4(file(menu_file), 'html.parser') - -dates = {} -for date_config in json.load(file('config/dates.json')): - dates[date_config['date']] = date_config['url'] - -players = {} - -for date in sorted(dates.keys()): - for player, ranking in fetch_ranking(date, True).iteritems(): - if player not in players: - players[player] = {'rankings':{}} - players[player]['name'] = ranking['player'] - players[player]['club'] = ranking['club'] - players[player]['rankings'][date] = {'change': 'N','change-class':'primary'} - for field in ['place', 'score']: - players[player]['rankings'][date][field] = ranking[field] - for field in ['gender', 'age', 'region']: - players[player]['rankings'][date][field] = ranking[field] - players[player]['rankings'][date][field + '-place'] = ranking[field + '-place'] - players[player]['rankings'][date][field + '-change'] = 'N' - players[player]['rankings'][date][field + '-change-class'] = 'primary' - -for pid, player in players.iteritems(): - template = bs4(file('templates/player.html'), 'lxml') - template.select('h2.name')[0].insert(0, player['name']) - template.select('h3.club')[0].string = player['club'] - template.select('a.pid-link')[0]['href'] = 'https://msc.com.pl/cezar/?p=21&pid=%d' % (pid) - - missing_row = template.select('tr.missing')[0].extract() - normal_row = template.select('tr.normal')[0].extract() - - for date in dates: - if date not in player['rankings']: - player['rankings'][date] = None - prev = None - for date, ranking in sorted(player['rankings'].iteritems(), lambda x,y: cmp(x[0], y[0])): - if prev is not None and ranking is not None: - ranking['change'] = prev['place'] - ranking['place'] - for field in ['gender', 'age', 'region']: - if prev[field] == ranking[field]: - ranking[field + '-change'] = prev[field + '-place'] - ranking[field + '-place'] - for field in ['', 'gender-', 'age-', 'region-']: - if ranking[field+'change'] == 0: - ranking[field+'change'] = '=' - ranking[field+'change-class'] = 'default' - elif ranking[field+'change'] == 'N': - ranking[field+'change-class'] = 'primary' - else: - ranking[field+'change-class'] = 'success' if ranking[field+'change'] > 0 else 'danger' - ranking[field+'change'] = '%+d' % (ranking[field+'change']) - prev = ranking - - row = copy.copy(missing_row) if ranking is None else copy.copy(normal_row) - rank_link = row.find('td').a - rank_link.string = '.'.join(date.split('-')[::-1]) - base_rank_link = '../%s' % (dates[date]) - if ranking is not None and ranking['place'] > int(pagesize): - rank_link['href'] = '../%s#page:%d' % ( - dates[date], ceil(ranking['place'] / pagesize) - ) - else: - rank_link['href'] = base_rank_link - if ranking is not None: - score_cell = row.select('.score span')[0] - score_cell.string = '%.2f' % (ranking['score']) - score_cell['title'] = str(ranking['score']) - for field in ['region', 'age', 'gender']: - link = row.select('td.'+field+' a')[0] - if ranking[field+'-place'] > int(pagesize): - link['href'] = base_rank_link + '#%s:%s;page:%d' % ( - field, ranking[field], ceil(ranking[field+'-place'] / pagesize) - ) - else: - link['href'] = base_rank_link + '#%s:%s' % ( - field, ranking[field] - ) - link.string = ranking[field] if len(ranking[field]) else '-' - for field in ['', 'region-', 'age-', 'gender-']: - row.select('td.'+field+'place')[0].string = '%d.' % (ranking[field+'place']) - change_label = row.select('td.'+field+'place-change span.label')[0] - change_label.string = ranking[field+'change'] - change_label['class'] = change_label['class'] + ['label-'+ranking[field+'change-class']] - template.select('table.table tbody')[0].insert(0, row) - - menu = template.select('div.static-menu')[0] - menu.append(copy.copy(menu_content)) - - file(os.path.join(output_directory, '%d.html' % pid), 'w').write(template.prettify().encode('utf-8')) diff --git a/scripts/ranking.py b/scripts/ranking.py deleted file mode 100644 index 52b036f..0000000 --- a/scripts/ranking.py +++ /dev/null @@ -1,26 +0,0 @@ -import os, sys -from bs4 import BeautifulSoup as bs4 - -ranking_date = sys.argv[4] -subtitle = 'notowanie %s (%s), stan na %s' % ( - sys.argv[1], sys.argv[2], '.'.join(ranking_date.split('-')[::-1]) -) - -table = bs4(file('templates/ranking.html'), 'lxml') -table.select('.page-header h2 small')[0].string = subtitle - -table.select('table.data-table')[0]['data-ranking'] = '_data/%s.json' % ( - ranking_date) - -script_src = table.select('script[src="_res/ranking.js"]')[0] -script_src['src'] = '%s?%d' % ('_res/ranking.js', os.path.getmtime('http/_res/ranking.js')) - -rawlink = table.select('a#rawlink')[0] -rawlink['href'] = '%s/%s.csv' % (rawlink['href'], ranking_date) - -menu_file = file(sys.argv[3]) -menu = table.select('div.static-menu')[0] -menu.clear() -menu.append(bs4(menu_file, 'html.parser')) - -print table.prettify().encode('utf-8') diff --git a/scripts/rankings-csv-convert.py b/scripts/rankings-csv-convert.py new file mode 100644 index 0000000..4dcee92 --- /dev/null +++ b/scripts/rankings-csv-convert.py @@ -0,0 +1,9 @@ +import csv, sys + +data = list(csv.reader(file(sys.argv[1]), delimiter=";")) + +output = csv.writer(file(sys.argv[2], 'w')) + +date = data[1][3] +for row in data[4:-1]: + output.writerow([row[0], date, row[2], row[1]]) diff --git a/scripts/rankings-editions.py b/scripts/rankings-editions.py new file mode 100644 index 0000000..42d036f --- /dev/null +++ b/scripts/rankings-editions.py @@ -0,0 +1,36 @@ +import copy, json, sys + +from bs4 import BeautifulSoup as bs4 + +dates_config = json.load(file('config/dates.json')) +output_file = bs4(file(sys.argv[1]), 'lxml') + +editions = {} +for date_config in dates_config: + year = date_config['name'].split(' ')[1] + if year not in editions: + editions[year] = [] + editions[year].append(( + '%s (%s)' % (date_config['name'].split(' ')[0], date_config['index']), + date_config['url'], + date_config['date'] + )) + +template = bs4(file('templates/ranking.html'), 'lxml') + +date_group = template.select('#editions')[0].extract() +year_group = date_group.select('div[role="group"]')[0].extract() +ranking_link = year_group.select('.btn-default')[0].extract() +for year, dates in editions.iteritems(): + group = copy.copy(year_group) + group.select('.year')[0].string = str(year) + for date in dates[::-1]: + link = copy.copy(ranking_link) + link.string = date[0] + link['href'] = date[1] + link['datetime'] = date[2] + group.append(link) + date_group.append(group) + +output_file.select('#editions')[0].replace_with(date_group) +file(sys.argv[1], 'w').write(output_file.prettify().encode('utf-8')) diff --git a/scripts/rankings-tables-compile.py b/scripts/rankings-tables-compile.py new file mode 100644 index 0000000..52b036f --- /dev/null +++ b/scripts/rankings-tables-compile.py @@ -0,0 +1,26 @@ +import os, sys +from bs4 import BeautifulSoup as bs4 + +ranking_date = sys.argv[4] +subtitle = 'notowanie %s (%s), stan na %s' % ( + sys.argv[1], sys.argv[2], '.'.join(ranking_date.split('-')[::-1]) +) + +table = bs4(file('templates/ranking.html'), 'lxml') +table.select('.page-header h2 small')[0].string = subtitle + +table.select('table.data-table')[0]['data-ranking'] = '_data/%s.json' % ( + ranking_date) + +script_src = table.select('script[src="_res/ranking.js"]')[0] +script_src['src'] = '%s?%d' % ('_res/ranking.js', os.path.getmtime('http/_res/ranking.js')) + +rawlink = table.select('a#rawlink')[0] +rawlink['href'] = '%s/%s.csv' % (rawlink['href'], ranking_date) + +menu_file = file(sys.argv[3]) +menu = table.select('div.static-menu')[0] +menu.clear() +menu.append(bs4(menu_file, 'html.parser')) + +print table.prettify().encode('utf-8') diff --git a/scripts/static-menu.py b/scripts/static-menu.py deleted file mode 100644 index e30c702..0000000 --- a/scripts/static-menu.py +++ /dev/null @@ -1,14 +0,0 @@ -import copy, json, os, sys - -from bs4 import BeautifulSoup as bs4 - -content_file = sys.argv[1] -menu_content = file(sys.argv[2]) - -content = bs4(file(content_file), 'lxml') - -for menu_container in content.select('.static-menu'): - menu_container.clear() - menu_container.append(bs4(menu_content, 'html.parser')) - -file(content_file, 'w').write(content.prettify().encode('utf-8')) diff --git a/scripts/static.py b/scripts/static.py deleted file mode 100644 index 16d82ae..0000000 --- a/scripts/static.py +++ /dev/null @@ -1,34 +0,0 @@ -import copy, os, sys -from bs4 import BeautifulSoup as bs4 - -content_files = [] -page_header = '' - -arguments = sys.argv[1:] - -while True: - page_header = arguments[0] - arguments = arguments[1:] - if os.path.exists(page_header): - content_files.append(page_header) - page_header = '' - if len(arguments) == 0: - break - -template = bs4(file('templates/static.html'), 'lxml') - -content_wrapper = template.find('div', {'id': 'wrapper'}).extract() -del content_wrapper['id'] - -template.select('h2 small')[0].string = page_header - -footer = template.find('div', {'id': 'footer'}) - -for content_file in content_files: - content = copy.copy(content_wrapper) - content.div.append( - bs4(file(content_file).read(), 'html.parser') - ) - footer.insert_before(content) - -print template.prettify().encode('utf-8') diff --git a/scripts/statics-compile.py b/scripts/statics-compile.py new file mode 100644 index 0000000..16d82ae --- /dev/null +++ b/scripts/statics-compile.py @@ -0,0 +1,34 @@ +import copy, os, sys +from bs4 import BeautifulSoup as bs4 + +content_files = [] +page_header = '' + +arguments = sys.argv[1:] + +while True: + page_header = arguments[0] + arguments = arguments[1:] + if os.path.exists(page_header): + content_files.append(page_header) + page_header = '' + if len(arguments) == 0: + break + +template = bs4(file('templates/static.html'), 'lxml') + +content_wrapper = template.find('div', {'id': 'wrapper'}).extract() +del content_wrapper['id'] + +template.select('h2 small')[0].string = page_header + +footer = template.find('div', {'id': 'footer'}) + +for content_file in content_files: + content = copy.copy(content_wrapper) + content.div.append( + bs4(file(content_file).read(), 'html.parser') + ) + footer.insert_before(content) + +print template.prettify().encode('utf-8') -- cgit v1.2.3