Initial commit - BBO finder script

5 files changed, 264 insertions, 0 deletions

diff --git a/.envrc b/.envrc
new file mode 100644
index 0000000..f4047e7
--- /dev/null
+++ b/.envrc
@@ -0,0 +1,2 @@
+layout python3
+dotenv
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..10c089c
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+.direnv
+.env
+*cache.sqlite
+output/
+bbo_finder.log
diff --git a/bbo_finder.py b/bbo_finder.py
new file mode 100644
index 0000000..3931b1f
--- /dev/null
+++ b/bbo_finder.py
@@ -0,0 +1,229 @@
+# coding: utf-8
+import csv, io, os, re, sys
+from datetime import datetime, date, timedelta
+from urllib.parse import urlparse, parse_qs, unquote
+from xml.etree import ElementTree
+from tempfile import NamedTemporaryFile
+
+import requests
+import requests_cache
+from requests.auth import HTTPBasicAuth
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+from bs4 import BeautifulSoup as bs
+
+from dealconvert.formats.lin import LINFormat
+from dealconvert.formats.pbn import PBNFormat
+from bs_api.client import BsApiClient
+from usebio.bs.result import Importer as XMLImporter
+
+lin_converter = LINFormat()
+api_client = BsApiClient(api_username=os.getenv('BSAPI_USERNAME'), api_password=os.getenv('BSAPI_PASSWORD'))
+
+requests_cache.install_cache('bbo_cache', backend='sqlite', expire_after=60*60*24*2, allowable_codes=(200, 404, ))
+
+http = requests_cache.CachedSession()
+retries = Retry(total=5, backoff_factor=15, status_forcelist=[503])
+http.mount('https://', HTTPAdapter(max_retries=retries))
+
+http.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4300.0 Safari/537.36'})
+
+with http.cache_disabled():
+    login_form = http.get('https://www.bridgebase.com/myhands/myhands_login.php', params={
+        't': '/myhands/index.php?'
+    })
+login = http.post('https://www.bridgebase.com/myhands/myhands_login.php',
+                  data={
+                      'username': os.getenv('BBO_USERNAME'),
+                      'password': os.getenv('BBO_PASSWORD'),
+                      't': '/myhands/index.php?',
+                      'count': 1,
+                      'submit': 'Login',
+                      'keep': 'on'
+                  })
+
+
+def get_url_params(url):
+    return parse_qs(urlparse(url).query)
+
+def get_bbo_url_content(url, params=None):
+    url = '%s/%s' % (
+        'https://www.bridgebase.com/myhands',
+        url)
+    params.update({'offset': 0})
+    r = http.get(
+        url,
+        params=params
+    )
+    r.raise_for_status()
+    if r.status_code != 200:
+        raise IOError('Logged out')
+    return r.text
+
+def get_timeframe(days=7):
+    if len(sys.argv) < 3:
+        end_time = datetime.combine(date.today(), datetime.min.time())
+    else:
+        end_time = datetime.strptime('%s 00:00:00' % (sys.argv[2]), '%Y-%m-%d %H:%M:%S')
+    start_time = end_time - timedelta(days=days)
+    return start_time, end_time
+
+def get_tournaments_for_bbo_user(username, days=7):
+    start_time, end_time = get_timeframe(days)
+    content = get_bbo_url_content('hands.php', {
+        'username': username,
+        'start_time': int(start_time.timestamp()),
+        'end_time': int(end_time.timestamp())
+    })
+    tree = bs(content, 'lxml')
+    tournaments = []
+    for link in tree.select(
+            'tr.tourneySummary td.tourneyName a[href]'):
+        t_url = urlparse(link['href'])
+        t_id = parse_qs(t_url.query)['t'][0]
+        tournaments.append((t_id, link.text))
+    return tournaments
+
+def get_board_layout_str(board_layout):
+    return ' '.join(['.'.join([''.join([card for card in suit]) for suit in hand]) for hand in board_layout])
+
+def get_boards_for_tournament(t_id):
+    r = http.get('https://webutil.bridgebase.com/v2/tview.php',
+                 params={'t': t_id})
+    try:
+        summary_link = bs(r.text, 'lxml').select('td a[href]')[0]['href']
+    except IndexError:
+        return []
+    tour_params = get_url_params(summary_link)
+    trav_content = get_bbo_url_content('hands.php', params=tour_params)
+    lin_links = bs(trav_content, 'lxml').select('td.movie a[onclick]')
+    boards = []
+    for lin_link in lin_links:
+        lin_link = lin_link['onclick']
+        lin_parts = re.match(r'hv_popuplin\((.*)\)', lin_link)
+        if lin_parts:
+            lin_record = unquote(lin_parts.group(1).split('|')[-1]).split('|')
+            board_layout = None
+            board_number = None
+            for field in lin_record:
+                if not board_layout and ',' in field:
+                    board_layout = lin_converter._parse_md_field(field)
+                if not board_number and 'Board ' in field:
+                    board_number = int(field.replace('Board ', ''))
+            if board_layout and board_number:
+                board_layout = get_board_layout_str(board_layout)
+                boards.append((board_number, board_layout))
+    return boards
+
+def get_bs_tournament_data(t_id):
+    r = requests.get('https://r.bridgespider.com/%d/xml' % (t_id))
+    r.raise_for_status()
+    tmp = NamedTemporaryFile(delete=False)
+    tmp.write(r.content)
+    tmp.close()
+    bs_result = XMLImporter(tmp.name, autoparse=False)
+    os.unlink(tmp.name)
+    bs_result.parse_data(results=False)
+    players = set()
+    for pair in bs_result.pairs.values():
+        for player in pair.players:
+            if player.pid:
+                players.add((player.pid, player.fullname))
+    return {
+        'name': bs_result.club.session_info.event_name,
+        'players': list(players)
+    }
+
+def get_bs_tournament_boards(t_id):
+    r = requests.get('https://r.bridgespider.com/%d/pbn' % (t_id))
+    r.raise_for_status()
+    string = io.StringIO(r.text)
+    pbn_format = PBNFormat()
+    boards = []
+    for deal in pbn_format.parse_content(string):
+        boards.append((deal.number, get_board_layout_str(deal.hands)))
+    return boards
+
+def get_bs_tournaments(days=7):
+    start_time, end_time = get_timeframe(days)
+    tournaments = []
+    for t_id in api_client.get_tournaments(start_time, end_time):
+        try:
+            tournaments.append((t_id, get_bs_tournament_data(t_id), get_bs_tournament_boards(t_id)))
+        except requests.exceptions.HTTPError:
+            pass
+    return tournaments
+
+def get_bbo_user_db():
+    r = requests.get('https://www.bridgenet.pl/beta/admin/nickcezar.php',
+                     auth=HTTPBasicAuth(os.getenv('BRIDGENET_USERNAME'), os.getenv('BRIDGENET_PASSWORD')))
+    db = {}
+    for row in csv.reader(r.content.decode().splitlines()):
+        try:
+            nick = row[0]
+            pid = int(row[1])
+            if pid:
+                if pid not in db:
+                    db[pid] = []
+                db[pid].append(nick)
+        except ValueError:
+            pass
+    return db
+
+bbo_user_db = get_bbo_user_db()
+
+def get_bbo_user_for_pid(pid):
+    return bbo_user_db.get(pid, [])
+
+
+def __main__():
+    days = int(sys.argv[1]) if len(sys.argv) > 1 else 7
+
+    boards = {}
+    bs_players = set()
+    tournaments = {}
+
+    bs_tournaments = get_bs_tournaments(days=days)
+    for tour in bs_tournaments:
+        tournament = ('bs', tour[0], tour[1]['name'])
+        tournaments[tournament] = []
+        for board in tour[2]:
+            if board[1] not in boards:
+                boards[board[1]] = set()
+            boards[board[1]].add((tournament, board[0]))
+        for player in tour[1]['players']:
+            for bbo_user in get_bbo_user_for_pid(player[0]):
+                p = (bbo_user, player[0], player[1])
+                bs_players.add(p)
+                tournaments[tournament].append(p)
+
+    for player in bs_players:
+        for t in get_tournaments_for_bbo_user(player[0], days=days):
+            tournament = ('bbo', t[0], t[1])
+            if tournament not in tournaments:
+                tournaments[tournament] = []
+                for board in get_boards_for_tournament(t[0]):
+                    if board[1] not in boards:
+                        boards[board[1]] = set()
+                    boards[board[1]].add((tournament, board[0]))
+            tournaments[tournament].append(player)
+
+    for board, tours in boards.items():
+        if len(tours) > 1:
+            players = {}
+            for t in tours:
+                for pl in tournaments[t[0]]:
+                    if pl not in players:
+                        players[pl] = []
+                    players[pl].append(t[0])
+            print('Rozdanie %s zagrano więcej niż jeden raz:' % (board))
+            for tour in tours:
+                print(' - turniej %s %s (%s) jako rozdanie nr %d' % (tour[0][0].upper(), tour[0][1], tour[0][2], tour[1]))
+            print('Gracze, którzy rozegrali rozdanie więcej niż jeden raz:')
+            for p, t in players.items():
+                if len(t) > 1:
+                    print(' - #%d %s (%s)' % (p[1], p[2], p[0]))
+            print('')
+
+if __name__ == '__main__':
+    __main__()
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..1194857
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,4 @@
+requests
+BeautifulSoup4
+lxml
+requests-cache
diff --git a/run.sh b/run.sh
new file mode 100755
index 0000000..2698660
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+pushd $(dirname $0) > /dev/null
+
+if [ "$1" == "--no-cache" ]
+then
+    rm -f *cache*.sqlite
+fi
+
+set -o allexport
+source .env
+set +o allexport
+
+DATE=`date +%Y-%m-%d`
+OUTPUT_FILE=output/bbo_finder-$DATE.txt
+python bbo_finder.py 14 $DATE 2>>bbo_finder.log > ${OUTPUT_FILE}
+
+if [ -s ${OUTPUT_FILE} ]
+then
+    cat ${OUTPUT_FILE}
+    /root/bin/join-notify.sh "BBO finder returned data" > /dev/null 2>&1
+fi
+
+popd > /dev/null