diff options
| author | emkael <emkael@tlen.pl> | 2018-02-24 11:03:14 +0100 |
|---|---|---|
| committer | emkael <emkael@tlen.pl> | 2018-02-24 11:03:14 +0100 |
| commit | c00ca25512ca244cd15ecb79d269ebb60a6c70bf (patch) | |
| tree | 66f3cec6a4036c91b81da2049c23813c505cb2e4 /jfr_playoff/remote.py | |
| parent | b70add902378a4be98ab90cdaeebc105bd7cc8b0 (diff) | |
| parent | f22476a36f2096628dc336f9adf0caa9a827dc11 (diff) | |
Merge branch 'master' into verbose_logging
Diffstat (limited to 'jfr_playoff/remote.py')
| -rw-r--r-- | jfr_playoff/remote.py | 10 |
1 files changed, 9 insertions, 1 deletions
diff --git a/jfr_playoff/remote.py b/jfr_playoff/remote.py index a113c96..339390c 100644 --- a/jfr_playoff/remote.py +++ b/jfr_playoff/remote.py @@ -1,3 +1,5 @@ +import re + import requests from bs4 import BeautifulSoup as bs @@ -12,7 +14,13 @@ class RemoteUrl: PlayoffLogger.get('remote').info( 'fetching content for: %s', url) if url not in cls.url_cache: - cls.url_cache[url] = requests.get(url).text + request = requests.get(url) + encoding_match = re.search( + 'content=".*;( )?charset=(.*)"', + request.content, re.IGNORECASE) + if encoding_match: + request.encoding = encoding_match.group(2) + cls.url_cache[url] = request.text PlayoffLogger.get('remote').info( 'content for %s not in cache: retrieved %d bytes', url, len(cls.url_cache[url])) |