diff options
Diffstat (limited to 'jfr_playoff/remote.py')
| -rw-r--r-- | jfr_playoff/remote.py | 12 |
1 files changed, 10 insertions, 2 deletions
diff --git a/jfr_playoff/remote.py b/jfr_playoff/remote.py index 8f6f78f..54031a8 100644 --- a/jfr_playoff/remote.py +++ b/jfr_playoff/remote.py @@ -1,4 +1,4 @@ -import re +import re, urlparse import requests @@ -10,11 +10,19 @@ class RemoteUrl: url_cache = {} @classmethod - def fetch_raw(cls, url): + def fetch_raw(cls, url, gzip_fallback=False): PlayoffLogger.get('remote').info( 'fetching content for: %s', url) if url not in cls.url_cache: request = requests.get(url) + if request.status_code == 404 and gzip_fallback: + old_url = urlparse.urlparse(url) + new_url = old_url._replace(path=old_url.path+'.gz') + new_url = urlparse.urlunparse(new_url) + PlayoffLogger.get('remote').info( + 'URL %s not found, trying GZIP variant %s', + url, new_url) + request = requests.get(url) encoding_match = re.search( 'content=".*;( )?charset=(.*?)"', request.content, re.IGNORECASE) |