diff options
| -rw-r--r-- | .envrc | 2 | ||||
| -rw-r--r-- | .gitignore | 5 | ||||
| -rw-r--r-- | bbo_finder.py | 229 | ||||
| -rw-r--r-- | requirements.txt | 4 | ||||
| -rwxr-xr-x | run.sh | 24 |
5 files changed, 264 insertions, 0 deletions
@@ -0,0 +1,2 @@ +layout python3 +dotenv diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..10c089c --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.direnv +.env +*cache.sqlite +output/ +bbo_finder.log diff --git a/bbo_finder.py b/bbo_finder.py new file mode 100644 index 0000000..3931b1f --- /dev/null +++ b/bbo_finder.py @@ -0,0 +1,229 @@ +# coding: utf-8 +import csv, io, os, re, sys +from datetime import datetime, date, timedelta +from urllib.parse import urlparse, parse_qs, unquote +from xml.etree import ElementTree +from tempfile import NamedTemporaryFile + +import requests +import requests_cache +from requests.auth import HTTPBasicAuth +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry +from bs4 import BeautifulSoup as bs + +from dealconvert.formats.lin import LINFormat +from dealconvert.formats.pbn import PBNFormat +from bs_api.client import BsApiClient +from usebio.bs.result import Importer as XMLImporter + +lin_converter = LINFormat() +api_client = BsApiClient(api_username=os.getenv('BSAPI_USERNAME'), api_password=os.getenv('BSAPI_PASSWORD')) + +requests_cache.install_cache('bbo_cache', backend='sqlite', expire_after=60*60*24*2, allowable_codes=(200, 404, )) + +http = requests_cache.CachedSession() +retries = Retry(total=5, backoff_factor=15, status_forcelist=[503]) +http.mount('https://', HTTPAdapter(max_retries=retries)) + +http.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4300.0 Safari/537.36'}) + +with http.cache_disabled(): + login_form = http.get('https://www.bridgebase.com/myhands/myhands_login.php', params={ + 't': '/myhands/index.php?' + }) +login = http.post('https://www.bridgebase.com/myhands/myhands_login.php', + data={ + 'username': os.getenv('BBO_USERNAME'), + 'password': os.getenv('BBO_PASSWORD'), + 't': '/myhands/index.php?', + 'count': 1, + 'submit': 'Login', + 'keep': 'on' + }) + + +def get_url_params(url): + return parse_qs(urlparse(url).query) + +def get_bbo_url_content(url, params=None): + url = '%s/%s' % ( + 'https://www.bridgebase.com/myhands', + url) + params.update({'offset': 0}) + r = http.get( + url, + params=params + ) + r.raise_for_status() + if r.status_code != 200: + raise IOError('Logged out') + return r.text + +def get_timeframe(days=7): + if len(sys.argv) < 3: + end_time = datetime.combine(date.today(), datetime.min.time()) + else: + end_time = datetime.strptime('%s 00:00:00' % (sys.argv[2]), '%Y-%m-%d %H:%M:%S') + start_time = end_time - timedelta(days=days) + return start_time, end_time + +def get_tournaments_for_bbo_user(username, days=7): + start_time, end_time = get_timeframe(days) + content = get_bbo_url_content('hands.php', { + 'username': username, + 'start_time': int(start_time.timestamp()), + 'end_time': int(end_time.timestamp()) + }) + tree = bs(content, 'lxml') + tournaments = [] + for link in tree.select( + 'tr.tourneySummary td.tourneyName a[href]'): + t_url = urlparse(link['href']) + t_id = parse_qs(t_url.query)['t'][0] + tournaments.append((t_id, link.text)) + return tournaments + +def get_board_layout_str(board_layout): + return ' '.join(['.'.join([''.join([card for card in suit]) for suit in hand]) for hand in board_layout]) + +def get_boards_for_tournament(t_id): + r = http.get('https://webutil.bridgebase.com/v2/tview.php', + params={'t': t_id}) + try: + summary_link = bs(r.text, 'lxml').select('td a[href]')[0]['href'] + except IndexError: + return [] + tour_params = get_url_params(summary_link) + trav_content = get_bbo_url_content('hands.php', params=tour_params) + lin_links = bs(trav_content, 'lxml').select('td.movie a[onclick]') + boards = [] + for lin_link in lin_links: + lin_link = lin_link['onclick'] + lin_parts = re.match(r'hv_popuplin\((.*)\)', lin_link) + if lin_parts: + lin_record = unquote(lin_parts.group(1).split('|')[-1]).split('|') + board_layout = None + board_number = None + for field in lin_record: + if not board_layout and ',' in field: + board_layout = lin_converter._parse_md_field(field) + if not board_number and 'Board ' in field: + board_number = int(field.replace('Board ', '')) + if board_layout and board_number: + board_layout = get_board_layout_str(board_layout) + boards.append((board_number, board_layout)) + return boards + +def get_bs_tournament_data(t_id): + r = requests.get('https://r.bridgespider.com/%d/xml' % (t_id)) + r.raise_for_status() + tmp = NamedTemporaryFile(delete=False) + tmp.write(r.content) + tmp.close() + bs_result = XMLImporter(tmp.name, autoparse=False) + os.unlink(tmp.name) + bs_result.parse_data(results=False) + players = set() + for pair in bs_result.pairs.values(): + for player in pair.players: + if player.pid: + players.add((player.pid, player.fullname)) + return { + 'name': bs_result.club.session_info.event_name, + 'players': list(players) + } + +def get_bs_tournament_boards(t_id): + r = requests.get('https://r.bridgespider.com/%d/pbn' % (t_id)) + r.raise_for_status() + string = io.StringIO(r.text) + pbn_format = PBNFormat() + boards = [] + for deal in pbn_format.parse_content(string): + boards.append((deal.number, get_board_layout_str(deal.hands))) + return boards + +def get_bs_tournaments(days=7): + start_time, end_time = get_timeframe(days) + tournaments = [] + for t_id in api_client.get_tournaments(start_time, end_time): + try: + tournaments.append((t_id, get_bs_tournament_data(t_id), get_bs_tournament_boards(t_id))) + except requests.exceptions.HTTPError: + pass + return tournaments + +def get_bbo_user_db(): + r = requests.get('https://www.bridgenet.pl/beta/admin/nickcezar.php', + auth=HTTPBasicAuth(os.getenv('BRIDGENET_USERNAME'), os.getenv('BRIDGENET_PASSWORD'))) + db = {} + for row in csv.reader(r.content.decode().splitlines()): + try: + nick = row[0] + pid = int(row[1]) + if pid: + if pid not in db: + db[pid] = [] + db[pid].append(nick) + except ValueError: + pass + return db + +bbo_user_db = get_bbo_user_db() + +def get_bbo_user_for_pid(pid): + return bbo_user_db.get(pid, []) + + +def __main__(): + days = int(sys.argv[1]) if len(sys.argv) > 1 else 7 + + boards = {} + bs_players = set() + tournaments = {} + + bs_tournaments = get_bs_tournaments(days=days) + for tour in bs_tournaments: + tournament = ('bs', tour[0], tour[1]['name']) + tournaments[tournament] = [] + for board in tour[2]: + if board[1] not in boards: + boards[board[1]] = set() + boards[board[1]].add((tournament, board[0])) + for player in tour[1]['players']: + for bbo_user in get_bbo_user_for_pid(player[0]): + p = (bbo_user, player[0], player[1]) + bs_players.add(p) + tournaments[tournament].append(p) + + for player in bs_players: + for t in get_tournaments_for_bbo_user(player[0], days=days): + tournament = ('bbo', t[0], t[1]) + if tournament not in tournaments: + tournaments[tournament] = [] + for board in get_boards_for_tournament(t[0]): + if board[1] not in boards: + boards[board[1]] = set() + boards[board[1]].add((tournament, board[0])) + tournaments[tournament].append(player) + + for board, tours in boards.items(): + if len(tours) > 1: + players = {} + for t in tours: + for pl in tournaments[t[0]]: + if pl not in players: + players[pl] = [] + players[pl].append(t[0]) + print('Rozdanie %s zagrano więcej niż jeden raz:' % (board)) + for tour in tours: + print(' - turniej %s %s (%s) jako rozdanie nr %d' % (tour[0][0].upper(), tour[0][1], tour[0][2], tour[1])) + print('Gracze, którzy rozegrali rozdanie więcej niż jeden raz:') + for p, t in players.items(): + if len(t) > 1: + print(' - #%d %s (%s)' % (p[1], p[2], p[0])) + print('') + +if __name__ == '__main__': + __main__() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1194857 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +requests +BeautifulSoup4 +lxml +requests-cache @@ -0,0 +1,24 @@ +#!/bin/bash +set -e +pushd $(dirname $0) > /dev/null + +if [ "$1" == "--no-cache" ] +then + rm -f *cache*.sqlite +fi + +set -o allexport +source .env +set +o allexport + +DATE=`date +%Y-%m-%d` +OUTPUT_FILE=output/bbo_finder-$DATE.txt +python bbo_finder.py 14 $DATE 2>>bbo_finder.log > ${OUTPUT_FILE} + +if [ -s ${OUTPUT_FILE} ] +then + cat ${OUTPUT_FILE} + /root/bin/join-notify.sh "BBO finder returned data" > /dev/null 2>&1 +fi + +popd > /dev/null |