From 40c69d6a0f15f152f0ba77ea13917c1efefe6201 Mon Sep 17 00:00:00 2001 From: echel0n Date: Fri, 14 Mar 2014 15:35:05 -0700 Subject: [PATCH] Updated cacheHandler to allow us to force cache and set a max-age on the cached content, vastly improved performance with indexers. --- lib/cachecontrol/adapter.py | 4 ++-- lib/cachecontrol/controller.py | 11 +++++++-- lib/cachecontrol/wrapper.py | 4 ++-- lib/tvdb_api/tvdb_api.py | 44 +++++++++++++++++----------------- lib/tvrage_api/tvrage_api.py | 2 +- 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/lib/cachecontrol/adapter.py b/lib/cachecontrol/adapter.py index 27f58fc7..e990746c 100644 --- a/lib/cachecontrol/adapter.py +++ b/lib/cachecontrol/adapter.py @@ -7,10 +7,10 @@ from cachecontrol.cache import DictCache class CacheControlAdapter(HTTPAdapter): invalidating_methods = set(['PUT', 'DELETE']) - def __init__(self, cache=None, cache_etags=True, *args, **kw): + def __init__(self, cache=None, cache_etags=True, cache_force=False, *args, **kw): super(CacheControlAdapter, self).__init__(*args, **kw) self.cache = cache or DictCache() - self.controller = CacheController(self.cache, cache_etags=cache_etags) + self.controller = CacheController(self.cache, cache_etags=cache_etags, cache_force=cache_force) def send(self, request, **kw): """Send a request. Use the request information to see if it diff --git a/lib/cachecontrol/controller.py b/lib/cachecontrol/controller.py index 67dd840e..029c5dc9 100644 --- a/lib/cachecontrol/controller.py +++ b/lib/cachecontrol/controller.py @@ -4,6 +4,7 @@ The httplib2 algorithms ported for use with requests. import re import calendar import time +import os from cachecontrol.cache import DictCache from cachecontrol.compat import parsedate_tz @@ -24,9 +25,10 @@ def parse_uri(uri): class CacheController(object): """An interface to see if request should cached or not. """ - def __init__(self, cache=None, cache_etags=True): + def __init__(self, cache=None, cache_etags=True, cache_force=False): self.cache = cache or DictCache() self.cache_etags = cache_etags + self.cache_force = cache_force def _urlnorm(self, uri): """Normalize the URL to create a safe key for the cache""" @@ -166,7 +168,7 @@ class CacheController(object): # return the original handler return False - def add_headers(self, url): + def add_headers(self, url, resp=None): resp = self.cache.get(url) if resp and 'etag' in resp.headers: return {'If-None-Match': resp.headers['etag']} @@ -212,6 +214,11 @@ class CacheController(object): if resp.headers['expires']: self.cache.set(cache_url, resp) + # If the request is for our local cache, it means we should cache it + elif self.cache_force: + resp.headers.update({'cache-control': 'max-age=21600, private'}) + self.cache.set(cache_url, resp) + def update_cached_response(self, request, response): """On a 304 we will get a new set of headers that we want to update our cached value with, assuming we have one. diff --git a/lib/cachecontrol/wrapper.py b/lib/cachecontrol/wrapper.py index d32f60a6..38b91536 100644 --- a/lib/cachecontrol/wrapper.py +++ b/lib/cachecontrol/wrapper.py @@ -2,9 +2,9 @@ from cachecontrol.adapter import CacheControlAdapter from cachecontrol.cache import DictCache -def CacheControl(sess, cache=None, cache_etags=True): +def CacheControl(sess, cache=None, cache_etags=True, cache_force=False): cache = cache or DictCache() - adapter = CacheControlAdapter(cache, cache_etags=cache_etags) + adapter = CacheControlAdapter(cache, cache_etags=cache_etags, cache_force=cache_force) sess.mount('http://', adapter) return sess diff --git a/lib/tvdb_api/tvdb_api.py b/lib/tvdb_api/tvdb_api.py index 9d9fcd53..3c73db7d 100644 --- a/lib/tvdb_api/tvdb_api.py +++ b/lib/tvdb_api/tvdb_api.py @@ -494,7 +494,7 @@ class Tvdb: self.config['params_getSeries'] = {"seriesname": "", "language": "all"} else: self.config['url_getSeries'] = u"%(base_url)s/api/GetSeries.php" % self.config - self.config['params_getSeries'] = {"seriesname": "", "language": ""} + self.config['params_getSeries'] = {"seriesname": "", "language": self.config['language']} self.config['url_epInfo'] = u"%(base_url)s/api/%(apikey)s/series/%%s/all/%%s.xml" % self.config self.config['url_epInfo_zip'] = u"%(base_url)s/api/%(apikey)s/series/%%s/all/%%s.zip" % self.config @@ -527,7 +527,7 @@ class Tvdb: # cacheControl if self.config['cache_enabled']: - sess = CacheControl(requests.Session(), cache=FileCache(self.config['cache_location'])) + sess = CacheControl(requests.Session(), cache_force=True, cache=FileCache(self.config['cache_location'])) else: sess = requests.Session() @@ -538,26 +538,26 @@ class Tvdb: lastTimeout = datetime.datetime.now() raise tvdb_error("Could not connect to server: %s" % (e)) - # handle gzipped content, - # http://dbr.lighthouseapp.com/projects/13342/tickets/72-gzipped-data-patch - if 'gzip' in resp.headers.get("Content-Encoding", ''): - if gzip: - stream = StringIO.StringIO(resp.content) - gz = gzip.GzipFile(fileobj=stream) - return gz.read() - - raise tvdb_error("Received gzip data from thetvdb.com, but could not correctly handle it") - - if 'application/zip' in resp.headers.get("Content-Type", ''): - try: - # TODO: The zip contains actors.xml and banners.xml, which are currently ignored [GH-20] - log().debug("We recived a zip file unpacking now ...") - zipdata = StringIO.StringIO() - zipdata.write(resp.content) - myzipfile = zipfile.ZipFile(zipdata) - return myzipfile.read('%s.xml' % language) - except zipfile.BadZipfile: - raise tvdb_error("Bad zip file received from thetvdb.com, could not read it") + ## handle gzipped content, + ## http://dbr.lighthouseapp.com/projects/13342/tickets/72-gzipped-data-patch + #if 'gzip' in resp.headers.get("Content-Encoding", ''): + # if gzip: + # stream = StringIO.StringIO(resp.content) + # gz = gzip.GzipFile(fileobj=stream) + # return gz.read() + # + # raise tvdb_error("Received gzip data from thetvdb.com, but could not correctly handle it") + # + #if 'application/zip' in resp.headers.get("Content-Type", ''): + # try: + # # TODO: The zip contains actors.xml and banners.xml, which are currently ignored [GH-20] + # log().debug("We recived a zip file unpacking now ...") + # zipdata = StringIO.StringIO() + # zipdata.write(resp.content) + # myzipfile = zipfile.ZipFile(zipdata) + # return myzipfile.read('%s.xml' % language) + # except zipfile.BadZipfile: + # raise tvdb_error("Bad zip file received from thetvdb.com, could not read it") return resp.content diff --git a/lib/tvrage_api/tvrage_api.py b/lib/tvrage_api/tvrage_api.py index 9eed27a0..f4d3568b 100644 --- a/lib/tvrage_api/tvrage_api.py +++ b/lib/tvrage_api/tvrage_api.py @@ -358,7 +358,7 @@ class TVRage: # cacheControl if self.config['cache_enabled']: - sess = CacheControl(requests.Session(), cache=FileCache(self.config['cache_location'])) + sess = CacheControl(requests.Session(), cache_force=True, cache=FileCache(self.config['cache_location'])) else: sess = requests.Session()