From 9ac649444d22e1113d21685be8f613961013c678 Mon Sep 17 00:00:00 2001 From: echel0n Date: Fri, 28 Mar 2014 21:23:55 -0700 Subject: [PATCH] Added cache_auto, cache_urls, and cache_max_age features to our cache handler. Added session handler to our cache handler so that we can pass in paramaters at the request level and use them for our cache handler to process features. --- lib/cachecontrol/__init__.py | 1 + lib/cachecontrol/adapter.py | 4 ++-- lib/cachecontrol/cache.py | 1 + lib/cachecontrol/controller.py | 20 +++++++++++++------- lib/cachecontrol/session.py | 24 ++++++++++++++++++++++++ lib/cachecontrol/wrapper.py | 7 ++++--- lib/tvdb_api/tvdb_api.py | 10 +++------- lib/tvrage_api/tvrage_api.py | 10 +++------- 8 files changed, 51 insertions(+), 26 deletions(-) create mode 100644 lib/cachecontrol/session.py diff --git a/lib/cachecontrol/__init__.py b/lib/cachecontrol/__init__.py index 693e11f1..d5b3d9f3 100644 --- a/lib/cachecontrol/__init__.py +++ b/lib/cachecontrol/__init__.py @@ -11,3 +11,4 @@ import cachecontrol.patch_requests from cachecontrol.wrapper import CacheControl from cachecontrol.adapter import CacheControlAdapter from cachecontrol.controller import CacheController +from cachecontrol.session import CacheControlSession \ No newline at end of file diff --git a/lib/cachecontrol/adapter.py b/lib/cachecontrol/adapter.py index 5a7d78d9..9bc25c22 100644 --- a/lib/cachecontrol/adapter.py +++ b/lib/cachecontrol/adapter.py @@ -7,10 +7,10 @@ from cachecontrol.cache import DictCache class CacheControlAdapter(HTTPAdapter): invalidating_methods = set(['PUT', 'DELETE']) - def __init__(self, cache=None, cache_etags=True, cache_all=False, *args, **kw): + def __init__(self, sess=None, cache=None, cache_etags=True, *args, **kw): super(CacheControlAdapter, self).__init__(*args, **kw) self.cache = cache or DictCache() - self.controller = CacheController(self.cache, cache_etags=cache_etags, cache_all=cache_all) + self.controller = CacheController(sess=sess, cache=cache, cache_etags=cache_etags) def send(self, request, **kw): """Send a request. Use the request information to see if it diff --git a/lib/cachecontrol/cache.py b/lib/cachecontrol/cache.py index feb7d3ed..abcd6fda 100644 --- a/lib/cachecontrol/cache.py +++ b/lib/cachecontrol/cache.py @@ -34,3 +34,4 @@ class DictCache(BaseCache): with self.lock: if key in self.data: self.data.pop(key) +0 \ No newline at end of file diff --git a/lib/cachecontrol/controller.py b/lib/cachecontrol/controller.py index 43b3b49a..5d283697 100644 --- a/lib/cachecontrol/controller.py +++ b/lib/cachecontrol/controller.py @@ -25,10 +25,10 @@ def parse_uri(uri): class CacheController(object): """An interface to see if request should cached or not. """ - def __init__(self, cache=None, cache_etags=True, cache_all=False): + def __init__(self, sess=None, cache=None, cache_etags=True): self.cache = cache or DictCache() self.cache_etags = cache_etags - self.cache_all = cache_all + self.sess = sess def _urlnorm(self, uri): """Normalize the URL to create a safe key for the cache""" @@ -185,9 +185,14 @@ class CacheController(object): if resp.status_code not in [200, 203]: return - # If we want to cache sites not setup with cache headers then add the proper headers and keep the response - if self.cache_all and getattr(resp.headers, 'cache-control', None) is None: - headers = {'Cache-Control': 'public,max-age=%d' % int(900)} + + cache_url = self.cache_url(request.url) + if self.sess.cache_urls and not any(s in cache_url for s in self.sess.cache_urls): + return + + if self.sess.cache_auto and ('cache-control' not in resp.headers or 'Cache-Control' not in resp.headers): + cache_max_age = int(self.sess.cache_max_age) or 900 + headers = {'Cache-Control': 'public,max-age=%d' % int(cache_max_age)} resp.headers.update(headers) if getattr(resp.headers, 'expires', None) is None: @@ -199,8 +204,6 @@ class CacheController(object): cc_req = self.parse_cache_control(request.headers) cc = self.parse_cache_control(resp.headers) - cache_url = self.cache_url(request.url) - # Delete it from the cache if we happen to have it stored there no_store = cc.get('no-store') or cc_req.get('no-store') if no_store and self.cache.get(cache_url): @@ -217,6 +220,9 @@ class CacheController(object): # cache when there is a max-age > 0 if cc and cc.get('max-age'): if int(cc['max-age']) > 0: + if self.sess.cache_max_age: + cc['max-age'] = int(self.sess.cache_max_age) + resp.headers['cache-control'] = ''.join(['%s=%s' % (key, value) for (key, value) in cc.items()]) self.cache.set(cache_url, resp) # If the request can expire, it means we should cache it diff --git a/lib/cachecontrol/session.py b/lib/cachecontrol/session.py new file mode 100644 index 00000000..258405af --- /dev/null +++ b/lib/cachecontrol/session.py @@ -0,0 +1,24 @@ +import datetime +from requests.sessions import Session + +class CacheControlSession(Session): + def __init__(self, *args, **kw): + super(CacheControlSession, self).__init__(*args, **kw) + + def request(self, *args, **kw): + # auto-cache response + self.cache_auto = False + if kw.has_key('cache_auto'): + self.cache_auto = kw.pop('cache_auto') + + # urls allowed to cache + self.cache_urls = [] + if kw.has_key('cache_urls'): + self.cache_urls = [str(args[1])] + kw.pop('cache_urls') + + # timeout for cacheed responses + self.cache_max_age = None + if kw.has_key('cache_max_age'): + self.cache_max_age = int(kw.pop('cache_max_age')) + + return super(CacheControlSession, self).request(*args, **kw) \ No newline at end of file diff --git a/lib/cachecontrol/wrapper.py b/lib/cachecontrol/wrapper.py index be3d932c..5e0d4ce3 100644 --- a/lib/cachecontrol/wrapper.py +++ b/lib/cachecontrol/wrapper.py @@ -1,10 +1,11 @@ +from cachecontrol.session import CacheControlSession from cachecontrol.adapter import CacheControlAdapter from cachecontrol.cache import DictCache - -def CacheControl(sess, cache=None, cache_etags=True, cache_all=False): +def CacheControl(sess=None, cache=None, cache_etags=True): + sess = sess or CacheControlSession() cache = cache or DictCache() - adapter = CacheControlAdapter(cache, cache_etags=cache_etags, cache_all=cache_all) + adapter = CacheControlAdapter(sess, cache, cache_etags=cache_etags) sess.mount('http://', adapter) return sess diff --git a/lib/tvdb_api/tvdb_api.py b/lib/tvdb_api/tvdb_api.py index fcb7e2b2..19be0196 100644 --- a/lib/tvdb_api/tvdb_api.py +++ b/lib/tvdb_api/tvdb_api.py @@ -427,16 +427,12 @@ class Tvdb: if cache is True: self.config['cache_enabled'] = True - self.sess = cachecontrol.CacheControl(requests.Session(), - cache_all=True, - cache=caches.FileCache(self._getTempDir())) + self.sess = cachecontrol.CacheControl(cache=caches.FileCache(self._getTempDir())) elif cache is False: self.config['cache_enabled'] = False elif isinstance(cache, basestring): self.config['cache_enabled'] = True - self.sess = cachecontrol.CacheControl(requests.Session(), - cache_all=True, - cache=caches.FileCache(cache)) + self.sess = cachecontrol.CacheControl(cache=caches.FileCache(cache)) else: raise ValueError("Invalid value for Cache %r (type was %s)" % (cache, type(cache))) @@ -539,7 +535,7 @@ class Tvdb: # get response from TVDB if self.config['cache_enabled']: - resp = self.sess.get(url, params=params) + resp = self.sess.get(url, cache_auto=True, params=params) else: resp = requests.get(url, params=params) diff --git a/lib/tvrage_api/tvrage_api.py b/lib/tvrage_api/tvrage_api.py index 0144df84..5a0ea5ac 100644 --- a/lib/tvrage_api/tvrage_api.py +++ b/lib/tvrage_api/tvrage_api.py @@ -270,16 +270,12 @@ class TVRage: if cache is True: self.config['cache_enabled'] = True - self.sess = cachecontrol.CacheControl(requests.Session(), - cache_all=True, - cache=caches.FileCache(self._getTempDir())) + self.sess = cachecontrol.CacheControl(cache=caches.FileCache(self._getTempDir())) elif cache is False: self.config['cache_enabled'] = False elif isinstance(cache, basestring): self.config['cache_enabled'] = True - self.sess = cachecontrol.CacheControl(requests.Session(), - cache_all=True, - cache=caches.FileCache(cache)) + self.sess = cachecontrol.CacheControl(cache=caches.FileCache(cache)) else: raise ValueError("Invalid value for Cache %r (type was %s)" % (cache, type(cache))) @@ -372,7 +368,7 @@ class TVRage: # get response from TVRage if self.config['cache_enabled']: - resp = self.sess.get(url, params=params) + resp = self.sess.get(url, cache_auto=True, params=params) else: resp = requests.get(url, params=params)