summaryrefslogtreecommitdiff
path: root/jfr_playoff
diff options
context:
space:
mode:
authoremkael <emkael@tlen.pl>2018-02-24 11:03:14 +0100
committeremkael <emkael@tlen.pl>2018-02-24 11:03:14 +0100
commitc00ca25512ca244cd15ecb79d269ebb60a6c70bf (patch)
tree66f3cec6a4036c91b81da2049c23813c505cb2e4 /jfr_playoff
parentb70add902378a4be98ab90cdaeebc105bd7cc8b0 (diff)
parentf22476a36f2096628dc336f9adf0caa9a827dc11 (diff)
Merge branch 'master' into verbose_logging
Diffstat (limited to 'jfr_playoff')
-rw-r--r--jfr_playoff/db.py4
-rw-r--r--jfr_playoff/remote.py10
2 files changed, 10 insertions, 4 deletions
diff --git a/jfr_playoff/db.py b/jfr_playoff/db.py
index 9d26a67..bca05d2 100644
--- a/jfr_playoff/db.py
+++ b/jfr_playoff/db.py
@@ -35,9 +35,7 @@ class PlayoffDB(object):
return row
except mysql.connector.Error as e:
PlayoffLogger.get('db').error(str(e))
- raise IOError(
- message=str(e), filename=db_name,
- errno=e.errno, strerror=str(e))
+ raise IOError(e.errno, str(e), db_name)
def fetch_all(self, db_name, sql, params):
import mysql.connector
diff --git a/jfr_playoff/remote.py b/jfr_playoff/remote.py
index a113c96..339390c 100644
--- a/jfr_playoff/remote.py
+++ b/jfr_playoff/remote.py
@@ -1,3 +1,5 @@
+import re
+
import requests
from bs4 import BeautifulSoup as bs
@@ -12,7 +14,13 @@ class RemoteUrl:
PlayoffLogger.get('remote').info(
'fetching content for: %s', url)
if url not in cls.url_cache:
- cls.url_cache[url] = requests.get(url).text
+ request = requests.get(url)
+ encoding_match = re.search(
+ 'content=".*;( )?charset=(.*)"',
+ request.content, re.IGNORECASE)
+ if encoding_match:
+ request.encoding = encoding_match.group(2)
+ cls.url_cache[url] = request.text
PlayoffLogger.get('remote').info(
'content for %s not in cache: retrieved %d bytes',
url, len(cls.url_cache[url]))