path: root/src/virtual_table.py

"""
Virtual tables for JFR Pary result pages.

Utility to clean up pages generated by JFR Pary from data on virtual tables.
"""

import sys
import glob
import re
import copy
import logging as log

from os import path
from bs4 import BeautifulSoup as bs4
from bs4.element import NavigableString

__version__ = '1.0.1'


def bs4_fix_file(worker_method):
    """Decorator for manipulating files with BeautifulSoup4."""
    def file_wrapper(self, file_path, encoding='utf-8'):
        """
        Wrapper for DOM manipulation.

        Wraps the inner function into BS4 invokation and file overwrite.
        """
        with file(file_path, 'r+') as content_file:
            content = bs4(content_file, 'lxml', from_encoding=encoding)
            content = worker_method(self, content)
            content_file.seek(0)
            content_file.write(
                content.prettify(encoding, formatter='html'))
            content_file.truncate()
    return file_wrapper


def fill_pair_list_table(cells, row_cell_count=20):
    """Format cell list into well-formed rows, aligned by column count."""
    content = bs4('<table />', 'lxml')
    content.append(content.new_tag('table'))
    # first filler cell of each new row
    first_cell = content.new_tag('td', **{'class': 'n'})
    first_cell.string = u'\xa0'
    # arrange cells into rows, full rows first
    while len(cells) >= row_cell_count:
        new_row = content.new_tag('tr')
        new_row.append(copy.copy(first_cell))
        for cell in cells[0:row_cell_count]:
            new_row.append(cell)
        content.table.append(new_row)
        log.getLogger('rec_list').debug('aligning cells %s to %s in a row',
                                        cells[0].a.contents,
                                        cells[row_cell_count-1].a.contents)
        del cells[0:row_cell_count]
    # last row may or may not be full
    last_row = content.new_tag('tr')
    last_row.append(copy.copy(first_cell))
    for cell in cells:
        last_row.append(cell)
    log.getLogger('rec_list').debug('leaving cells %s to %s in last row',
                                    cells[0].a.contents,
                                    cells[-1].a.contents)
    # if it wasn't full, fill it with a col-spanned last cell
    if len(cells) < row_cell_count:
        last_cell = content.new_tag('td',
                                    colspan=row_cell_count-len(cells))
        last_cell.string = u'\xa0'
        last_row.append(last_cell)
        log.getLogger('rec_list').debug('filling last row with: %s',
                                        last_cell)
    content.table.append(last_row)
    return content.table.contents


class JFRVirtualTable(object):
    """Virtual tables formatter (for JFR Pary pages)."""

    def __parse_filepaths(self, prefix):
        """Detect all filepaths for results pages."""
        file_path = path.realpath(prefix)
        log.getLogger('paths').debug('realpath = %s', file_path)
        tournament_path = path.dirname(file_path)
        log.getLogger('paths').info('tournament dir = %s', tournament_path)
        tournament_prefix = path.splitext(path.basename(file_path))[0]
        log.getLogger('paths').info('tournament prefix = %s',
                                    tournament_prefix)

        # RegEx matching traveller files for each board
        traveller_files_match = re.compile(
            re.escape(tournament_prefix) + r'([0-9]{3})\.txt')
        log.getLogger('paths').debug('traveller files regex = %s',
                                     traveller_files_match.pattern)

        def get_path(relative_path):
            """
            Compile full path for tournament file from base name.

            Converts {prefix}{anything}.{ext} filename to full path.
            """
            return path.join(tournament_path, relative_path)

        # filtering out traveller files from all TXT files
        self.__traveller_files = [f for f
                                  in glob.glob(
                                      get_path(tournament_prefix + '*.txt'))
                                  if re.search(traveller_files_match, f)]
        log.getLogger('paths').info('found %d traveller files',
                                    len(self.__traveller_files))
        log.getLogger('paths').debug(
            'traveller files:\n' + '\n'.join(self.__traveller_files))

        # RegEx for matching pair record files
        records_files_match = re.compile(
            'H-' + tournament_prefix + r'-([0-9]{1,3})\.html')
        log.getLogger('paths').debug('records files regex = %s',
                                     records_files_match.pattern)
        self.__pair_records_files = [
            f for f
            in glob.glob(get_path('H-' + tournament_prefix + '*.html'))
            if re.search(records_files_match, f)
        ]
        log.getLogger('paths').info('found %d records files',
                                    len(self.__pair_records_files))
        log.getLogger('paths').debug(
            'record files:\n' + '\n'.join(self.__pair_records_files))

        # short rersult list, from side frame
        self.__single_files['results'] = get_path(
            tournament_prefix + 'WYN.txt')
        log.getLogger('paths').info('generated results path = %s',
                                    self.__single_files['results'])
        # full results page
        self.__single_files['full_results'] = get_path(
            'W-' + tournament_prefix + '.html')
        log.getLogger('paths').info('generated full results path = %s',
                                    self.__single_files['full_results'])
        # list of pair records links page
        self.__single_files['pair_records_list'] = get_path(
            'H-' + tournament_prefix + '-lista.html')
        log.getLogger('paths').info('generated records list path = %s',
                                    self.__single_files['pair_records_list'])
        # collected scores page
        self.__single_files['collected_scores'] = get_path(
            tournament_prefix + 'zbior.html')
        log.getLogger('paths').info('generated collected scores path = %s',
                                    self.__single_files['collected_scores'])

    def __detect_virtual_pairs(self):
        """Auto-detect virtual pairs by their record file header."""
        virtual_pairs = []
        # RegEx for matching pair number and names in pair record header
        pair_header_match = re.compile('([0-9]{1,}): (.*) - (.*), .*')
        for record_file_path in self.__pair_records_files:
            log.getLogger('detect').debug('examining record file %s',
                                          record_file_path)
            with file(record_file_path) as record_file:
                record = bs4(record_file, 'lxml')
            # first <td class="o1"> with content matching
            # pair header is what we're after
            header = [con for con
                      in record.select('td.o1')[0].contents
                      if isinstance(con, NavigableString) and re.search(
                          pair_header_match, con)]
            log.getLogger('detect').debug('detected header: %s', header)
            if len(header):
                header_match = re.match(pair_header_match, header[0])
                pair_number = int(header_match.group(1))
                names = [name for name in [header_match.group(2).strip(),
                                           header_match.group(3).strip()]
                         if len(name)]
                log.getLogger('detect').debug('parsed header: %d, %s',
                                              pair_number, names)
                # virtual pair does not have any names filled
                if len(names) == 0:
                    virtual_pairs.append(pair_number)
        if len(virtual_pairs) == 0:
            log.getLogger('detect').warning('No virtual pairs detected')
        else:
            log.getLogger('detect').info('virtual pairs: %s',
                                         ' '.join(sorted(
                                             [str(pair) for pair
                                              in virtual_pairs])))
        return sorted(virtual_pairs)

    @bs4_fix_file
    def __fix_results(self, content):
        """Fix simple results list by removing virtual pair rows."""
        rows = content.select('tr')
        for row in rows:
            cells = row.select('td')
            # 6 or more cells in a "proper" result row
            # (may contain carry over or penalties)
            if len(cells) >= 6:
                try:
                    log.getLogger('results').debug('table cell: %s',
                                                   cells[2].contents)
                    # third cell in the row is pair number
                    if int(cells[2].contents[0]) in self.__virtual_pairs:
                        row.extract()
                        log.getLogger('results').info(
                            'removed: %s', cells[2].contents[0])
                except ValueError:
                    log.getLogger('results').debug(
                        'no pair number in cell: %s', cells[2].contents)
        return content.table

    @bs4_fix_file
    def __fix_full_results(self, content):
        """Fix full results file by removing virtual pair rows."""
        rows = content.select('tr')
        for row in rows:
            # select rows by cells containing pair records links
            cell_links = [link for link
                          in row.select('td a')
                          if link.has_attr('href') and
                          link['href'].startswith('H-') and
                          not link['href'].endswith('lista.html')]
            log.getLogger('f_result').debug('found pair links: %s',
                                            [c['href'] for c in cell_links])
            # remove these containing links to virtual pairs
            if len(cell_links):
                if int(cell_links[0].contents[0]) in self.__virtual_pairs:
                    row.extract()
                    log.getLogger('f_result').info('removed: %s',
                                                   cell_links[0].contents[0])
        return content

    @bs4_fix_file
    def __fix_records_list(self, content):
        """Fix the page with pair records links list."""
        # read the original column count
        row_cell_count = int(content.table.select('tr td.o')[0]['colspan'])
        log.getLogger('rec_list').debug('found %d cells in column',
                                        row_cell_count)
        rows = content.select('tr')
        # gather rows which containted any links
        link_rows = []
        # gather cells which should stay
        link_cells = []
        for row in rows:
            cells = row.select('td.u')
            cells_found = False
            log.getLogger('rec_list').debug('found %d cells in a row',
                                            len(cells))
            for cell in cells:
                # select cells by pair records links inside
                cell_links = [link for link
                              in cell.select('a.pa')
                              if link.has_attr('href') and
                              link['href'].startswith('H-') and
                              not link['href'].endswith('lista.html')]
                log.getLogger('rec_list').debug('found links in cell: %s',
                                                [c['href'] for c
                                                 in cell_links])
                if len(cell_links):
                    # delete virtual pair cells
                    if int(cell_links[0].contents[0]) in self.__virtual_pairs:
                        cell.extract()
                        log.getLogger('rec_list').info(
                            'removed: %s', cell_links[0].contents[0])
                    # store actual pair cells
                    else:
                        link_cells.append(cell)
                    cells_found = True
            # gather processed rows
            if cells_found:
                link_rows.append(row)
        # detach actual pair cells from the tree
        cells = [cell.extract() for cell in link_cells]
        log.getLogger('rec_list').info('remaining cell count: %d', len(cells))
        log.getLogger('rec_list').info('remaining row count: %d',
                                       len(link_rows))
        for row in link_rows:
            row.extract()
        for row in fill_pair_list_table(cells, row_cell_count):
            content.table.append(row)
        return content

    @bs4_fix_file
    def __fix_collected(self, content):
        """Fix collected scores tables by removing virtual pair rows."""
        rows = content.select('tr')
        for row in rows:
            cells = row.select('td')
            # "proper" rows should have 7 cells
            if len(cells) == 7:
                # ignore cells without proper pair numbers
                log.getLogger('c_scores').debug(
                    'found collected scores row: %s', cells[1:3])
                try:
                    pairs = [int(c.contents[0]) for c in cells[1:3]]
                    if int(cells[1].contents[0]) in self.__virtual_pairs:
                        if int(cells[2].contents[0]) in self.__virtual_pairs:
                            log.getLogger('c_scores').info('removed %s', pairs)
                            row.extract()
                except ValueError:
                    log.getLogger('c_scores').debug(
                        'pair numbers not found, ignoring')
            # there are some clearly broken table cells, fix them
            if len(cells) == 1 and cells[0]['colspan'] == '7':
                if cells[0].contents[0] == '&nbsp':
                    log.getLogger('c_scores').debug('fixing cell: %s',
                                                    cells[0])
                    cells[0].contents[0] = u'\xa0'
        return content

    @bs4_fix_file
    def __fix_traveller(self, content):
        """
        Fix board travellers.

        Removes virtual tables and leaving one, annotated.
        """
        # This should only happen if the traveller wasn't already processed
        # as it's the only operaton that may yield any results on second run
        # and it might break stuff.
        if not len(content.select('tr.virtualTable')):
            # looking for all the rows with more than 2 cells
            rows = [row for row
                    in content.select('tr')
                    if len(row.select('td')) >= 3]
            # only the first "virtual" row needs to be prefixed with a header
            header_added = False
            virtual_row = None
            for row in rows:
                cells = row.select('td')
                debug_string = ' '.join([
                    ''.join([cc for cc in c.contents
                             if isinstance(cc, basestring)])
                    for c in cells])
                # we're already added a header, meaning we're below the first
                # virtual table, we need to move the row above it
                # or remove it entirely
                if header_added:
                    row_below = row.extract()
                    # only move it if it has meaningful information (10 cells)
                    if len(cells) >= 10:
                        log.getLogger('traveller').debug(
                            'row moved upwards: %s', debug_string)
                        virtual_row.insert_before(row_below)
                    else:
                        log.getLogger('traveller').info(
                            'removed row %s', debug_string)
                # we're looking for a "proper" row, with at least 10 cells
                if len(cells) >= 10:
                    # and with both pair numbers virtual
                    if int(cells[1].contents[0]) in self.__virtual_pairs:
                        if int(cells[2].contents[0]) in self.__virtual_pairs:
                            # if we're already processed the first one,
                            # just drop subsequent virtual tables
                            if header_added:
                                row.extract()
                                log.getLogger('traveller').info(
                                    'removed row %s', debug_string)
                            # it's the first virtual table
                            # prefix it with a header
                            else:
                                virtual_row = content.new_tag(
                                    'tr',
                                    **{'class': 'virtualTable'})
                                virtual_row.append(
                                    content.new_tag('td', **{'class': 'n'}))
                                virtual_row_header = content.new_tag(
                                    'td',
                                    colspan=10, **{'class': 'noc'})
                                virtual_row_header.string = self.__header_text
                                virtual_row.append(virtual_row_header)
                                row.insert_before(virtual_row)
                                log.getLogger('traveller').info(
                                    'added header above row %s', debug_string)
                                # clear pair numbers
                                for cell in cells[1:3]:
                                    cell.contents = ''
                                header_added = True
        else:
            raise UserWarning('already processed, skipping')
        return content.table

    __traveller_files = []
    __pair_records_files = []
    __single_files = {
        'results': None,
        'full_results': None,
        'pair_records_list': None,
        'collected_scores': None
    }
    __header_text = ''  # text for traveller header row

    def __init__(self, path_prefix, virtual_pairs=None, header_text=''):
        """Construct main parser object."""
        log.getLogger('init').debug('parsing filepaths, prefix = %s',
                                    path_prefix)
        self.__parse_filepaths(path_prefix)
        log.getLogger('init').debug('collecting virtual pairs, %s provided',
                                    virtual_pairs)
        if virtual_pairs is None or len(virtual_pairs) == 0:
            virtual_pairs = self.__detect_virtual_pairs()
        self.__virtual_pairs = virtual_pairs
        log.getLogger('init').debug('setting header text to "%s"', header_text)
        self.__header_text = header_text

    def fix_results(self):
        """Fix results for specific detected tournament files."""
        self.__fix_results(self.__single_files['results'])

    def fix_full_results(self):
        """Fix full results for specific detected tournament files."""
        self.__fix_full_results(self.__single_files['full_results'])

    def fix_collected_scores(self):
        """Fix collected scores for specific detected tournament files."""
        if path.isfile(self.__single_files['collected_scores']):
            self.__fix_collected(self.__single_files['collected_scores'])
        else:
            log.getLogger('c_scores').warning(
                'Collected scores file %s not found',
                self.__single_files['collected_scores'])

    def fix_records_list(self):
        """Fix pair records list for specific detected tournament files."""
        self.__fix_records_list(self.__single_files['pair_records_list'])

    def fix_travellers(self):
        """Fix board travellers for specific detected tournament files."""
        for traveller_file in self.__traveller_files:
            log.getLogger('traveller').info('fixing traveller: %s',
                                            traveller_file)
            try:
                self.__fix_traveller(traveller_file)
            except UserWarning as warn:
                log.getLogger('traveller').warning('%s: %s',
                                                   traveller_file, warn)


def main():
    """Program entry point, invoked when __name__ is __main__."""
    import argparse

    argument_parser = argparse.ArgumentParser(
        description='Fix display for virtual tables in JFR Pary result pages')

    def file_path(filepath):
        """Sanitize and validate file paths from input parameters."""
        filepath = unicode(filepath, sys.getfilesystemencoding())
        if path.isfile(filepath):
            return filepath
        else:
            argument_parser.error('File %s does not exist' % filepath)

    def decoded_text(text):
        """Sanitize and normalize command line input for parameters."""
        return unicode(text, sys.getfilesystemencoding())

    argument_parser.add_argument('-V', '--version', action='version',
                                 version='%(prog)s {version}'.format(
                                     version=__version__))

    argument_parser.add_argument('path', metavar='PATH',
                                 help='tournament path with JFR prefix',
                                 type=file_path)
    argument_parser.add_argument('-t', '--text', metavar='HEADER',
                                 default='Wirtualny stolik:',
                                 help='traveller header for virtual score',
                                 type=decoded_text)
    argument_parser.add_argument('pairs', metavar='PAIR_NO', nargs='*',
                                 type=int, help='virtual pair numbers')

    console_output_args = argument_parser.add_mutually_exclusive_group()
    console_output_args.add_argument('-q', '--quiet', action='store_true',
                                     help='suppress warning on STDERR')
    console_output_args.add_argument('-v', '--verbose', action='store_true',
                                     help='be verbose on STDERR')

    argument_parser.add_argument('-l', '--log-level', metavar='LEVEL',
                                 help='file logging verbosity level',
                                 default='INFO', choices=['DEBUG',
                                                          'INFO',
                                                          'WARNING',
                                                          'ERROR',
                                                          'CRITICAL'])
    argument_parser.add_argument('-f', '--log-file', metavar='LOGFILE',
                                 help='log file path',
                                 default='virtual_table.log')

    arguments = argument_parser.parse_args()

    # primary logging facility - virtual_table.log file
    log.basicConfig(
        level=getattr(log, arguments.log_level),
        format='%(asctime)s %(levelname)-8s %(name)-8s %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S',
        filename=arguments.log_file)

    # secondary logging facility - standard error output
    console_log = log.StreamHandler()
    console_log.setLevel(log.INFO if arguments.verbose else (
        log.ERROR if arguments.quiet else log.WARNING))
    console_log.setFormatter(log.Formatter(
        '%(levelname)-8s %(name)-8s: %(message)s'))
    log.getLogger().addHandler(console_log)

    log.info('-------- program started --------')
    log.debug('parsed arguments: %s', arguments)

    try:
        table_parser = JFRVirtualTable(
            path_prefix=arguments.path,
            virtual_pairs=arguments.pairs,
            header_text=arguments.text)
        table_parser.fix_results()
        table_parser.fix_full_results()
        table_parser.fix_collected_scores()
        table_parser.fix_records_list()
        table_parser.fix_travellers()
    except Exception as ex:
        log.getLogger('root').error(ex.strerror)
        raise ex

    log.info('--------- program ended ---------')

if __name__ == '__main__':
    main()