monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2025-01-07 10:33:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove feedcache implementation and library

Browse source 
This functionality really isn't needed and the code offers no benefit for our needs
This commit is contained in:
Adam 2015-06-06 10:23:39 +08:00
parent 2331502861
commit cf0ee24cb4
61 changed files with 22 additions and 5509 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGES.md
View file

@ -27,6 +27,7 @@
* Fix show list view when no shows exist and "Group show lists shows into" is set to anything other than "One Show List" * Fix show list view when no shows exist and "Group show lists shows into" is set to anything other than "One Show List"
* Add coverage testing and coveralls support * Add coverage testing and coveralls support
* Update feedparser library 5.1.3 to 5.2.0 (8c62940) * Update feedparser library 5.1.3 to 5.2.0 (8c62940)
* Remove feedcache implementation and library
[develop changelog] [develop changelog]
* Update Requests library 2.7.0 (ab1f493) to 2.7.0 (8b5e457) * Update Requests library 2.7.0 (ab1f493) to 2.7.0 (8b5e457)

44
lib/feedcache/__init__.py
View file

@ -1,44 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""
"""
__module_id__ = "$Id$"
#
# Import system modules
#
#
# Import local modules
#
from cache import Cache
#
# Module
#

204
lib/feedcache/cache.py
View file

@ -1,204 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""
"""
__module_id__ = "$Id$"
#
# Import system modules
#
from feedparser import feedparser
import logging
import time
#
# Import local modules
#
#
# Module
#
logger = logging.getLogger('feedcache.cache')
class Cache:
"""A class to wrap Mark Pilgrim's Universal Feed Parser module
(http://www.feedparser.org) so that parameters can be used to
cache the feed results locally instead of fetching the feed every
time it is requested. Uses both etag and modified times for
caching.
"""
def __init__(self, storage, timeToLiveSeconds=300, userAgent='feedcache'):
"""
Arguments:
storage -- Backing store for the cache. It should follow
the dictionary API, with URLs used as keys. It should
persist data.
timeToLiveSeconds=300 -- The length of time content should
live in the cache before an update is attempted.
userAgent='feedcache' -- User agent string to be used when
fetching feed contents.
"""
self.storage = storage
self.time_to_live = timeToLiveSeconds
self.user_agent = userAgent
return
def purge(self, olderThanSeconds):
"""Remove cached data from the storage if the data is older than the
date given. If olderThanSeconds is None, the entire cache is purged.
"""
if olderThanSeconds is None:
logger.debug('purging the entire cache')
for key in self.storage.keys():
del self.storage[key]
else:
now = time.time()
# Iterate over the keys and load each item one at a time
# to avoid having the entire cache loaded into memory
# at one time.
for url in self.storage.keys():
(cached_time, cached_data) = self.storage[url]
age = now - cached_time
if age >= olderThanSeconds:
logger.debug('removing %s with age %d', url, age)
del self.storage[url]
return
def fetch(self, url, force_update=False, offline=False, request_headers=None):
"""Return the feed at url.
url - The URL of the feed.
force_update=False - When True, update the cache whether the
current contents have
exceeded their time-to-live
or not.
offline=False - When True, only return data from the local
cache and never access the remote
URL.
If there is data for that feed in the cache already, check
the expiration date before accessing the server. If the
cached data has not expired, return it without accessing the
server.
In cases where the server is accessed, check for updates
before deciding what to return. If the server reports a
status of 304, the previously cached content is returned.
The cache is only updated if the server returns a status of
200, to avoid holding redirected data in the cache.
"""
logger.debug('url="%s"' % url)
# Convert the URL to a value we can use
# as a key for the storage backend.
key = url
if isinstance(key, unicode):
key = key.encode('utf-8')
modified = None
etag = None
now = time.time()
cached_time, cached_content = self.storage.get(key, (None, None))
# Offline mode support (no networked requests)
# so return whatever we found in the storage.
# If there is nothing in the storage, we'll be returning None.
if offline:
logger.debug('offline mode')
return cached_content
# Does the storage contain a version of the data
# which is older than the time-to-live?
logger.debug('cache modified time: %s' % str(cached_time))
if cached_time is not None and not force_update:
if self.time_to_live:
age = now - cached_time
if age <= self.time_to_live:
logger.debug('cache contents still valid')
return cached_content
else:
logger.debug('cache contents older than TTL')
else:
logger.debug('no TTL value')
# The cache is out of date, but we have
# something. Try to use the etag and modified_time
# values from the cached content.
etag = cached_content.get('etag')
modified = cached_content.get('modified')
logger.debug('cached etag=%s' % etag)
logger.debug('cached modified=%s' % str(modified))
else:
logger.debug('nothing in the cache, or forcing update')
# We know we need to fetch, so go ahead and do it.
logger.debug('fetching...')
parsed_result = feedparser.parse(url,
agent=self.user_agent,
modified=modified,
etag=etag,
request_headers=request_headers)
status = parsed_result.get('status', None)
logger.debug('HTTP status=%s' % status)
if status == 304:
# No new data, based on the etag or modified values.
# We need to update the modified time in the
# storage, though, so we know that what we have
# stored is up to date.
self.storage[key] = (now, cached_content)
# Return the data from the cache, since
# the parsed data will be empty.
parsed_result = cached_content
elif status == 200:
# There is new content, so store it unless there was an error.
error = parsed_result.get('bozo_exception')
if not error:
logger.debug('Updating stored data for %s' % url)
self.storage[key] = (now, parsed_result)
else:
logger.warning('Not storing data with exception: %s',
error)
else:
logger.warning('Not updating cache with HTTP status %s', status)
return parsed_result

69
lib/feedcache/cachestoragelock.py
View file

@ -1,69 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
from __future__ import with_statement
"""Lock wrapper for cache storage which do not permit multi-threaded access.
"""
__module_id__ = "$Id$"
#
# Import system modules
#
import threading
#
# Import local modules
#
#
# Module
#
class CacheStorageLock:
"""Lock wrapper for cache storage which do not permit multi-threaded access.
"""
def __init__(self, shelf):
self.lock = threading.Lock()
self.shelf = shelf
return
def __getitem__(self, key):
with self.lock:
return self.shelf[key]
def get(self, key, default=None):
with self.lock:
try:
return self.shelf[key]
except KeyError:
return default
def __setitem__(self, key, value):
with self.lock:
self.shelf[key] = value

63
lib/feedcache/example.py
View file

@ -1,63 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""Example use of feedcache.Cache.
"""
__module_id__ = "$Id$"
#
# Import system modules
#
import sys
import shelve
#
# Import local modules
#
import cache
#
# Module
#
def main(urls=[]):
print 'Saving feed data to ./.feedcache'
storage = shelve.open('.feedcache')
try:
fc = cache.Cache(storage)
for url in urls:
parsed_data = fc.fetch(url)
print parsed_data.feed.title
for entry in parsed_data.entries:
print '\t', entry.title
finally:
storage.close()
return
if __name__ == '__main__':
main(sys.argv[1:])

144
lib/feedcache/example_threads.py
View file

@ -1,144 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""Example use of feedcache.Cache combined with threads.
"""
__module_id__ = "$Id$"
#
# Import system modules
#
import Queue
import sys
import shove
import threading
#
# Import local modules
#
import cache
#
# Module
#
MAX_THREADS=5
OUTPUT_DIR='/tmp/feedcache_example'
def main(urls=[]):
if not urls:
print 'Specify the URLs to a few RSS or Atom feeds on the command line.'
return
# Decide how many threads to start
num_threads = min(len(urls), MAX_THREADS)
# Add the URLs to a queue
url_queue = Queue.Queue()
for url in urls:
url_queue.put(url)
# Add poison pills to the url queue to cause
# the worker threads to break out of their loops
for i in range(num_threads):
url_queue.put(None)
# Track the entries in the feeds being fetched
entry_queue = Queue.Queue()
print 'Saving feed data to', OUTPUT_DIR
storage = shove.Shove('file://' + OUTPUT_DIR)
try:
# Start a few worker threads
worker_threads = []
for i in range(num_threads):
t = threading.Thread(target=fetch_urls,
args=(storage, url_queue, entry_queue,))
worker_threads.append(t)
t.setDaemon(True)
t.start()
# Start a thread to print the results
printer_thread = threading.Thread(target=print_entries, args=(entry_queue,))
printer_thread.setDaemon(True)
printer_thread.start()
# Wait for all of the URLs to be processed
url_queue.join()
# Wait for the worker threads to finish
for t in worker_threads:
t.join()
# Poison the print thread and wait for it to exit
entry_queue.put((None,None))
entry_queue.join()
printer_thread.join()
finally:
storage.close()
return
def fetch_urls(storage, input_queue, output_queue):
"""Thread target for fetching feed data.
"""
c = cache.Cache(storage)
while True:
next_url = input_queue.get()
if next_url is None: # None causes thread to exit
input_queue.task_done()
break
feed_data = c.fetch(next_url)
for entry in feed_data.entries:
output_queue.put( (feed_data.feed, entry) )
input_queue.task_done()
return
def print_entries(input_queue):
"""Thread target for printing the contents of the feeds.
"""
while True:
feed, entry = input_queue.get()
if feed is None: # None causes thread to exist
input_queue.task_done()
break
print '%s: %s' % (feed.title, entry.title)
input_queue.task_done()
return
if __name__ == '__main__':
main(sys.argv[1:])

323
lib/feedcache/test_cache.py
View file

@ -1,323 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""Unittests for feedcache.cache
"""
__module_id__ = "$Id$"
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)-8s %(name)s %(message)s',
)
logger = logging.getLogger('feedcache.test_cache')
#
# Import system modules
#
import copy
import time
import unittest
import UserDict
#
# Import local modules
#
import cache
from test_server import HTTPTestBase, TestHTTPServer
#
# Module
#
class CacheTestBase(HTTPTestBase):
CACHE_TTL = 30
def setUp(self):
HTTPTestBase.setUp(self)
self.storage = self.getStorage()
self.cache = cache.Cache(self.storage,
timeToLiveSeconds=self.CACHE_TTL,
userAgent='feedcache.test',
)
return
def getStorage(self):
"Return a cache storage for the test."
return {}
class CacheTest(CacheTestBase):
CACHE_TTL = 30
def getServer(self):
"These tests do not want to use the ETag or If-Modified-Since headers"
return TestHTTPServer(applyModifiedHeaders=False)
def testRetrieveNotInCache(self):
# Retrieve data not already in the cache.
feed_data = self.cache.fetch(self.TEST_URL)
self.failUnless(feed_data)
self.failUnlessEqual(feed_data.feed.title, 'CacheTest test data')
return
def testRetrieveIsInCache(self):
# Retrieve data which is alread in the cache,
# and verify that the second copy is identitical
# to the first.
# First fetch
feed_data = self.cache.fetch(self.TEST_URL)
# Second fetch
feed_data2 = self.cache.fetch(self.TEST_URL)
# Since it is the in-memory storage, we should have the
# exact same object.
self.failUnless(feed_data is feed_data2)
return
def testExpireDataInCache(self):
# Retrieve data which is in the cache but which
# has expired and verify that the second copy
# is different from the first.
# First fetch
feed_data = self.cache.fetch(self.TEST_URL)
# Change the timeout and sleep to move the clock
self.cache.time_to_live = 0
time.sleep(1)
# Second fetch
feed_data2 = self.cache.fetch(self.TEST_URL)
# Since we reparsed, the cache response should be different.
self.failIf(feed_data is feed_data2)
return
def testForceUpdate(self):
# Force cache to retrieve data which is alread in the cache,
# and verify that the new data is different.
# Pre-populate the storage with bad data
self.cache.storage[self.TEST_URL] = (time.time() + 100, self.id())
# Fetch the data
feed_data = self.cache.fetch(self.TEST_URL, force_update=True)
self.failIfEqual(feed_data, self.id())
return
def testOfflineMode(self):
# Retrieve data which is alread in the cache,
# whether it is expired or not.
# Pre-populate the storage with data
self.cache.storage[self.TEST_URL] = (0, self.id())
# Fetch it
feed_data = self.cache.fetch(self.TEST_URL, offline=True)
self.failUnlessEqual(feed_data, self.id())
return
def testUnicodeURL(self):
# Pass in a URL which is unicode
url = unicode(self.TEST_URL)
feed_data = self.cache.fetch(url)
storage = self.cache.storage
key = unicode(self.TEST_URL).encode('UTF-8')
# Verify that the storage has a key
self.failUnless(key in storage)
# Now pull the data from the storage directly
storage_timeout, storage_data = self.cache.storage.get(key)
self.failUnlessEqual(feed_data, storage_data)
return
class SingleWriteMemoryStorage(UserDict.UserDict):
"""Cache storage which only allows the cache value
for a URL to be updated one time.
"""
def __setitem__(self, url, data):
if url in self.keys():
modified, existing = self[url]
# Allow the modified time to change,
# but not the feed content.
if data[1] != existing:
raise AssertionError('Trying to update cache for %s to %s' \
% (url, data))
UserDict.UserDict.__setitem__(self, url, data)
return
class CacheConditionalGETTest(CacheTestBase):
CACHE_TTL = 0
def getStorage(self):
return SingleWriteMemoryStorage()
def testFetchOnceForEtag(self):
# Fetch data which has a valid ETag value, and verify
# that while we hit the server twice the response
# codes cause us to use the same data.
# First fetch populates the cache
response1 = self.cache.fetch(self.TEST_URL)
self.failUnlessEqual(response1.feed.title, 'CacheTest test data')
# Remove the modified setting from the cache so we know
# the next time we check the etag will be used
# to check for updates. Since we are using an in-memory
# cache, modifying response1 updates the cache storage
# directly.
response1['modified'] = None
# This should result in a 304 status, and no data from
# the server. That means the cache won't try to
# update the storage, so our SingleWriteMemoryStorage
# should not raise and we should have the same
# response object.
response2 = self.cache.fetch(self.TEST_URL)
self.failUnless(response1 is response2)
# Should have hit the server twice
self.failUnlessEqual(self.server.getNumRequests(), 2)
return
def testFetchOnceForModifiedTime(self):
# Fetch data which has a valid Last-Modified value, and verify
# that while we hit the server twice the response
# codes cause us to use the same data.
# First fetch populates the cache
response1 = self.cache.fetch(self.TEST_URL)
self.failUnlessEqual(response1.feed.title, 'CacheTest test data')
# Remove the etag setting from the cache so we know
# the next time we check the modified time will be used
# to check for updates. Since we are using an in-memory
# cache, modifying response1 updates the cache storage
# directly.
response1['etag'] = None
# This should result in a 304 status, and no data from
# the server. That means the cache won't try to
# update the storage, so our SingleWriteMemoryStorage
# should not raise and we should have the same
# response object.
response2 = self.cache.fetch(self.TEST_URL)
self.failUnless(response1 is response2)
# Should have hit the server twice
self.failUnlessEqual(self.server.getNumRequests(), 2)
return
class CacheRedirectHandlingTest(CacheTestBase):
def _test(self, response):
# Set up the server to redirect requests,
# then verify that the cache is not updated
# for the original or new URL and that the
# redirect status is fed back to us with
# the fetched data.
self.server.setResponse(response, '/redirected')
response1 = self.cache.fetch(self.TEST_URL)
# The response should include the status code we set
self.failUnlessEqual(response1.get('status'), response)
# The response should include the new URL, too
self.failUnlessEqual(response1.href, self.TEST_URL + 'redirected')
# The response should not have been cached under either URL
self.failIf(self.TEST_URL in self.storage)
self.failIf(self.TEST_URL + 'redirected' in self.storage)
return
def test301(self):
self._test(301)
def test302(self):
self._test(302)
def test303(self):
self._test(303)
def test307(self):
self._test(307)
class CachePurgeTest(CacheTestBase):
def testPurgeAll(self):
# Remove everything from the cache
self.cache.fetch(self.TEST_URL)
self.failUnless(self.storage.keys(),
'Have no data in the cache storage')
self.cache.purge(None)
self.failIf(self.storage.keys(),
'Still have data in the cache storage')
return
def testPurgeByAge(self):
# Remove old content from the cache
self.cache.fetch(self.TEST_URL)
self.failUnless(self.storage.keys(),
'have no data in the cache storage')
time.sleep(1)
remains = (time.time(), copy.deepcopy(self.storage[self.TEST_URL][1]))
self.storage['http://this.should.remain/'] = remains
self.cache.purge(1)
self.failUnlessEqual(self.storage.keys(),
['http://this.should.remain/'])
return
if __name__ == '__main__':
unittest.main()

90
lib/feedcache/test_cachestoragelock.py
View file

@ -1,90 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""Tests for shelflock.
"""
__module_id__ = "$Id$"
#
# Import system modules
#
import os
import shelve
import tempfile
import threading
import unittest
#
# Import local modules
#
from cache import Cache
from cachestoragelock import CacheStorageLock
from test_server import HTTPTestBase
#
# Module
#
class CacheShelveTest(HTTPTestBase):
def setUp(self):
HTTPTestBase.setUp(self)
handle, self.shelve_filename = tempfile.mkstemp('.shelve')
os.close(handle) # we just want the file name, so close the open handle
os.unlink(self.shelve_filename) # remove the empty file
return
def tearDown(self):
try:
os.unlink(self.shelve_filename)
except AttributeError:
pass
HTTPTestBase.tearDown(self)
return
def test(self):
storage = shelve.open(self.shelve_filename)
locking_storage = CacheStorageLock(storage)
try:
fc = Cache(locking_storage)
# First fetch the data through the cache
parsed_data = fc.fetch(self.TEST_URL)
self.failUnlessEqual(parsed_data.feed.title, 'CacheTest test data')
# Now retrieve the same data directly from the shelf
modified, shelved_data = storage[self.TEST_URL]
# The data should be the same
self.failUnlessEqual(parsed_data, shelved_data)
finally:
storage.close()
return
if __name__ == '__main__':
unittest.main()

241
lib/feedcache/test_server.py
View file

@ -1,241 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""Simple HTTP server for testing the feed cache.
"""
__module_id__ = "$Id$"
#
# Import system modules
#
import BaseHTTPServer
import logging
import md5
import threading
import time
import unittest
import urllib
#
# Import local modules
#
#
# Module
#
logger = logging.getLogger('feedcache.test_server')
def make_etag(data):
"""Given a string containing data to be returned to the client,
compute an ETag value for the data.
"""
_md5 = md5.new()
_md5.update(data)
return _md5.hexdigest()
class TestHTTPHandler(BaseHTTPServer.BaseHTTPRequestHandler):
"HTTP request handler which serves the same feed data every time."
FEED_DATA = """<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-us">
<title>CacheTest test data</title>
<link href="http://localhost/feedcache/" rel="alternate"></link>
<link href="http://localhost/feedcache/atom/" rel="self"></link>
<id>http://localhost/feedcache/</id>
<updated>2006-10-14T11:00:36Z</updated>
<entry>
<title>single test entry</title>
<link href="http://www.example.com/" rel="alternate"></link>
<updated>2006-10-14T11:00:36Z</updated>
<author>
<name>author goes here</name>
<email>authoremail@example.com</email>
</author>
<id>http://www.example.com/</id>
<summary type="html">description goes here</summary>
<link length="100" href="http://www.example.com/enclosure" type="text/html" rel="enclosure">
</link>
</entry>
</feed>"""
# The data does not change, so save the ETag and modified times
# as class attributes.
ETAG = make_etag(FEED_DATA)
# Calculated using email.utils.formatdate(usegmt=True)
MODIFIED_TIME = 'Sun, 08 Apr 2012 20:16:48 GMT'
def do_GET(self):
"Handle GET requests."
logger.debug('GET %s', self.path)
if self.path == '/shutdown':
# Shortcut to handle stopping the server
logger.debug('Stopping server')
self.server.stop()
self.send_response(200)
else:
# Record the request for tests that count them
self.server.requests.append(self.path)
# Process the request
logger.debug('pre-defined response code: %d', self.server.response)
handler_method_name = 'do_GET_%d' % self.server.response
handler_method = getattr(self, handler_method_name)
handler_method()
return
def do_GET_3xx(self):
"Handle redirects"
if self.path.endswith('/redirected'):
logger.debug('already redirected')
# We have already redirected, so return the data.
return self.do_GET_200()
new_path = self.server.new_path
logger.debug('redirecting to %s', new_path)
self.send_response(self.server.response)
self.send_header('Location', new_path)
return
do_GET_301 = do_GET_3xx
do_GET_302 = do_GET_3xx
do_GET_303 = do_GET_3xx
do_GET_307 = do_GET_3xx
def do_GET_200(self):
logger.debug('Etag: %s' % self.ETAG)
logger.debug('Last-Modified: %s' % self.MODIFIED_TIME)
incoming_etag = self.headers.get('If-None-Match', None)
logger.debug('Incoming ETag: "%s"' % incoming_etag)
incoming_modified = self.headers.get('If-Modified-Since', None)
logger.debug('Incoming If-Modified-Since: %s' % incoming_modified)
send_data = True
# Does the client have the same version of the data we have?
if self.server.apply_modified_headers:
if incoming_etag == self.ETAG:
logger.debug('Response 304, etag')
self.send_response(304)
send_data = False
elif incoming_modified == self.MODIFIED_TIME:
logger.debug('Response 304, modified time')
self.send_response(304)
send_data = False
# Now optionally send the data, if the client needs it
if send_data:
logger.debug('Response 200')
self.send_response(200)
self.send_header('Content-Type', 'application/atom+xml')
logger.debug('Outgoing Etag: %s' % self.ETAG)
self.send_header('ETag', self.ETAG)
logger.debug('Outgoing modified time: %s' % self.MODIFIED_TIME)
self.send_header('Last-Modified', self.MODIFIED_TIME)
self.end_headers()
logger.debug('Sending data')
self.wfile.write(self.FEED_DATA)
return
class TestHTTPServer(BaseHTTPServer.HTTPServer):
"""HTTP Server which counts the number of requests made
and can stop based on client instructions.
"""
def __init__(self, applyModifiedHeaders=True, handler=TestHTTPHandler):
self.apply_modified_headers = applyModifiedHeaders
self.keep_serving = True
self.requests = []
self.setResponse(200)
BaseHTTPServer.HTTPServer.__init__(self, ('', 9999), handler)
return
def setResponse(self, newResponse, newPath=None):
"""Sets the response code to use for future requests, and a new
path to be used as a redirect target, if necessary.
"""
self.response = newResponse
self.new_path = newPath
return
def getNumRequests(self):
"Return the number of requests which have been made on the server."
return len(self.requests)
def stop(self):
"Stop serving requests, after the next request."
self.keep_serving = False
return
def serve_forever(self):
"Main loop for server"
while self.keep_serving:
self.handle_request()
logger.debug('exiting')
return
class HTTPTestBase(unittest.TestCase):
"Base class for tests that use a TestHTTPServer"
TEST_URL = 'http://localhost:9999/'
CACHE_TTL = 0
def setUp(self):
self.server = self.getServer()
self.server_thread = threading.Thread(target=self.server.serve_forever)
# set daemon flag so the tests don't hang if cleanup fails
self.server_thread.setDaemon(True)
self.server_thread.start()
return
def getServer(self):
"Return a web server for the test."
s = TestHTTPServer()
s.setResponse(200)
return s
def tearDown(self):
# Stop the server thread
urllib.urlretrieve(self.TEST_URL + 'shutdown')
time.sleep(1)
self.server.server_close()
self.server_thread.join()
return

89
lib/feedcache/test_shovefilesystem.py
View file

@ -1,89 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2007 Doug Hellmann.
#
#
# All Rights Reserved
#
# Permission to use, copy, modify, and distribute this software and
# its documentation for any purpose and without fee is hereby
# granted, provided that the above copyright notice appear in all
# copies and that both that copyright notice and this permission
# notice appear in supporting documentation, and that the name of Doug
# Hellmann not be used in advertising or publicity pertaining to
# distribution of the software without specific, written prior
# permission.
#
# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
"""Tests with shove filesystem storage.
"""
__module_id__ = "$Id$"
#
# Import system modules
#
import os
import shove
import tempfile
import threading
import unittest
#
# Import local modules
#
from cache import Cache
from test_server import HTTPTestBase
#
# Module
#
class CacheShoveTest(HTTPTestBase):
def setUp(self):
HTTPTestBase.setUp(self)
self.shove_dirname = tempfile.mkdtemp('shove')
return
def tearDown(self):
try:
os.system('rm -rf %s' % self.storage_dirname)
except AttributeError:
pass
HTTPTestBase.tearDown(self)
return
def test(self):
# First fetch the data through the cache
storage = shove.Shove('file://' + self.shove_dirname)
try:
fc = Cache(storage)
parsed_data = fc.fetch(self.TEST_URL)
self.failUnlessEqual(parsed_data.feed.title, 'CacheTest test data')
finally:
storage.close()
# Now retrieve the same data directly from the shelf
storage = shove.Shove('file://' + self.shove_dirname)
try:
modified, shelved_data = storage[self.TEST_URL]
finally:
storage.close()
# The data should be the same
self.failUnlessEqual(parsed_data, shelved_data)
return
if __name__ == '__main__':
unittest.main()

519
lib/shove/__init__.py
View file

@ -1,519 +0,0 @@
# -*- coding: utf-8 -*-
'''Common object storage frontend.'''
import os
import zlib
import urllib
try:
import cPickle as pickle
except ImportError:
import pickle
from collections import deque
try:
# Import store and cache entry points if setuptools installed
import pkg_resources
stores = dict((_store.name, _store) for _store in
pkg_resources.iter_entry_points('shove.stores'))
caches = dict((_cache.name, _cache) for _cache in
pkg_resources.iter_entry_points('shove.caches'))
# Pass if nothing loaded
if not stores and not caches:
raise ImportError()
except ImportError:
# Static store backend registry
stores = dict(
bsddb='shove.store.bsdb:BsdStore',
cassandra='shove.store.cassandra:CassandraStore',
dbm='shove.store.dbm:DbmStore',
durus='shove.store.durusdb:DurusStore',
file='shove.store.file:FileStore',
firebird='shove.store.db:DbStore',
ftp='shove.store.ftp:FtpStore',
hdf5='shove.store.hdf5:HDF5Store',
leveldb='shove.store.leveldbstore:LevelDBStore',
memory='shove.store.memory:MemoryStore',
mssql='shove.store.db:DbStore',
mysql='shove.store.db:DbStore',
oracle='shove.store.db:DbStore',
postgres='shove.store.db:DbStore',
redis='shove.store.redisdb:RedisStore',
s3='shove.store.s3:S3Store',
simple='shove.store.simple:SimpleStore',
sqlite='shove.store.db:DbStore',
svn='shove.store.svn:SvnStore',
zodb='shove.store.zodb:ZodbStore',
)
# Static cache backend registry
caches = dict(
bsddb='shove.cache.bsdb:BsdCache',
file='shove.cache.file:FileCache',
filelru='shove.cache.filelru:FileLRUCache',
firebird='shove.cache.db:DbCache',
memcache='shove.cache.memcached:MemCached',
memlru='shove.cache.memlru:MemoryLRUCache',
memory='shove.cache.memory:MemoryCache',
mssql='shove.cache.db:DbCache',
mysql='shove.cache.db:DbCache',
oracle='shove.cache.db:DbCache',
postgres='shove.cache.db:DbCache',
redis='shove.cache.redisdb:RedisCache',
simple='shove.cache.simple:SimpleCache',
simplelru='shove.cache.simplelru:SimpleLRUCache',
sqlite='shove.cache.db:DbCache',
)
def getbackend(uri, engines, **kw):
'''
Loads the right backend based on a URI.
@param uri Instance or name string
@param engines A dictionary of scheme/class pairs
'''
if isinstance(uri, basestring):
mod = engines[uri.split('://', 1)[0]]
# Load module if setuptools not present
if isinstance(mod, basestring):
# Isolate classname from dot path
module, klass = mod.split(':')
# Load module
mod = getattr(__import__(module, '', '', ['']), klass)
# Load appropriate class from setuptools entry point
else:
mod = mod.load()
# Return instance
return mod(uri, **kw)
# No-op for existing instances
return uri
def synchronized(func):
'''
Decorator to lock and unlock a method (Phillip J. Eby).
@param func Method to decorate
'''
def wrapper(self, *__args, **__kw):
self._lock.acquire()
try:
return func(self, *__args, **__kw)
finally:
self._lock.release()
wrapper.__name__ = func.__name__
wrapper.__dict__ = func.__dict__
wrapper.__doc__ = func.__doc__
return wrapper
class Base(object):
'''Base Mapping class.'''
def __init__(self, engine, **kw):
'''
@keyword compress True, False, or an integer compression level (1-9).
'''
self._compress = kw.get('compress', False)
self._protocol = kw.get('protocol', pickle.HIGHEST_PROTOCOL)
def __getitem__(self, key):
raise NotImplementedError()
def __setitem__(self, key, value):
raise NotImplementedError()
def __delitem__(self, key):
raise NotImplementedError()
def __contains__(self, key):
try:
value = self[key]
except KeyError:
return False
return True
def get(self, key, default=None):
'''
Fetch a given key from the mapping. If the key does not exist,
return the default.
@param key Keyword of item in mapping.
@param default Default value (default: None)
'''
try:
return self[key]
except KeyError:
return default
def dumps(self, value):
'''Optionally serializes and compresses an object.'''
# Serialize everything but ASCII strings
value = pickle.dumps(value, protocol=self._protocol)
if self._compress:
level = 9 if self._compress is True else self._compress
value = zlib.compress(value, level)
return value
def loads(self, value):
'''Deserializes and optionally decompresses an object.'''
if self._compress:
try:
value = zlib.decompress(value)
except zlib.error:
pass
value = pickle.loads(value)
return value
class BaseStore(Base):
'''Base Store class (based on UserDict.DictMixin).'''
def __init__(self, engine, **kw):
super(BaseStore, self).__init__(engine, **kw)
self._store = None
def __cmp__(self, other):
if other is None:
return False
if isinstance(other, BaseStore):
return cmp(dict(self.iteritems()), dict(other.iteritems()))
def __del__(self):
# __init__ didn't succeed, so don't bother closing
if not hasattr(self, '_store'):
return
self.close()
def __iter__(self):
for k in self.keys():
yield k
def __len__(self):
return len(self.keys())
def __repr__(self):
return repr(dict(self.iteritems()))
def close(self):
'''Closes internal store and clears object references.'''
try:
self._store.close()
except AttributeError:
pass
self._store = None
def clear(self):
'''Removes all keys and values from a store.'''
for key in self.keys():
del self[key]
def items(self):
'''Returns a list with all key/value pairs in the store.'''
return list(self.iteritems())
def iteritems(self):
'''Lazily returns all key/value pairs in a store.'''
for k in self:
yield (k, self[k])
def iterkeys(self):
'''Lazy returns all keys in a store.'''
return self.__iter__()
def itervalues(self):
'''Lazily returns all values in a store.'''
for _, v in self.iteritems():
yield v
def keys(self):
'''Returns a list with all keys in a store.'''
raise NotImplementedError()
def pop(self, key, *args):
'''
Removes and returns a value from a store.
@param args Default to return if key not present.
'''
if len(args) > 1:
raise TypeError('pop expected at most 2 arguments, got ' + repr(
1 + len(args))
)
try:
value = self[key]
# Return default if key not in store
except KeyError:
if args:
return args[0]
del self[key]
return value
def popitem(self):
'''Removes and returns a key, value pair from a store.'''
try:
k, v = self.iteritems().next()
except StopIteration:
raise KeyError('Store is empty.')
del self[k]
return (k, v)
def setdefault(self, key, default=None):
'''
Returns the value corresponding to an existing key or sets the
to key to the default and returns the default.
@param default Default value (default: None)
'''
try:
return self[key]
except KeyError:
self[key] = default
return default
def update(self, other=None, **kw):
'''
Adds to or overwrites the values in this store with values from
another store.
other Another store
kw Additional keys and values to store
'''
if other is None:
pass
elif hasattr(other, 'iteritems'):
for k, v in other.iteritems():
self[k] = v
elif hasattr(other, 'keys'):
for k in other.keys():
self[k] = other[k]
else:
for k, v in other:
self[k] = v
if kw:
self.update(kw)
def values(self):
'''Returns a list with all values in a store.'''
return list(v for _, v in self.iteritems())
class Shove(BaseStore):
'''Common object frontend class.'''
def __init__(self, store='simple://', cache='simple://', **kw):
super(Shove, self).__init__(store, **kw)
# Load store
self._store = getbackend(store, stores, **kw)
# Load cache
self._cache = getbackend(cache, caches, **kw)
# Buffer for lazy writing and setting for syncing frequency
self._buffer, self._sync = dict(), kw.get('sync', 2)
def __getitem__(self, key):
'''Gets a item from shove.'''
try:
return self._cache[key]
except KeyError:
# Synchronize cache and store
self.sync()
value = self._store[key]
self._cache[key] = value
return value
def __setitem__(self, key, value):
'''Sets an item in shove.'''
self._cache[key] = self._buffer[key] = value
# When the buffer reaches self._limit, writes the buffer to the store
if len(self._buffer) >= self._sync:
self.sync()
def __delitem__(self, key):
'''Deletes an item from shove.'''
try:
del self._cache[key]
except KeyError:
pass
self.sync()
del self._store[key]
def keys(self):
'''Returns a list of keys in shove.'''
self.sync()
return self._store.keys()
def sync(self):
'''Writes buffer to store.'''
for k, v in self._buffer.iteritems():
self._store[k] = v
self._buffer.clear()
def close(self):
'''Finalizes and closes shove.'''
# If close has been called, pass
if self._store is not None:
try:
self.sync()
except AttributeError:
pass
self._store.close()
self._store = self._cache = self._buffer = None
class FileBase(Base):
'''Base class for file based storage.'''
def __init__(self, engine, **kw):
super(FileBase, self).__init__(engine, **kw)
if engine.startswith('file://'):
engine = urllib.url2pathname(engine.split('://')[1])
self._dir = engine
# Create directory
if not os.path.exists(self._dir):
self._createdir()
def __getitem__(self, key):
# (per Larry Meyn)
try:
item = open(self._key_to_file(key), 'rb')
data = item.read()
item.close()
return self.loads(data)
except:
raise KeyError(key)
def __setitem__(self, key, value):
# (per Larry Meyn)
try:
item = open(self._key_to_file(key), 'wb')
item.write(self.dumps(value))
item.close()
except (IOError, OSError):
raise KeyError(key)
def __delitem__(self, key):
try:
os.remove(self._key_to_file(key))
except (IOError, OSError):
raise KeyError(key)
def __contains__(self, key):
return os.path.exists(self._key_to_file(key))
def __len__(self):
return len(os.listdir(self._dir))
def _createdir(self):
'''Creates the store directory.'''
try:
os.makedirs(self._dir)
except OSError:
raise EnvironmentError(
'Cache directory "%s" does not exist and ' \
'could not be created' % self._dir
)
def _key_to_file(self, key):
'''Gives the filesystem path for a key.'''
return os.path.join(self._dir, urllib.quote_plus(key))
def keys(self):
'''Returns a list of keys in the store.'''
return [urllib.unquote_plus(name) for name in os.listdir(self._dir)]
class SimpleBase(Base):
'''Single-process in-memory store base class.'''
def __init__(self, engine, **kw):
super(SimpleBase, self).__init__(engine, **kw)
self._store = dict()
def __getitem__(self, key):
try:
return self._store[key]
except:
raise KeyError(key)
def __setitem__(self, key, value):
self._store[key] = value
def __delitem__(self, key):
try:
del self._store[key]
except:
raise KeyError(key)
def __len__(self):
return len(self._store)
def keys(self):
'''Returns a list of keys in the store.'''
return self._store.keys()
class LRUBase(SimpleBase):
def __init__(self, engine, **kw):
super(LRUBase, self).__init__(engine, **kw)
self._max_entries = kw.get('max_entries', 300)
self._hits = 0
self._misses = 0
self._queue = deque()
self._refcount = dict()
def __getitem__(self, key):
try:
value = super(LRUBase, self).__getitem__(key)
self._hits += 1
except KeyError:
self._misses += 1
raise
self._housekeep(key)
return value
def __setitem__(self, key, value):
super(LRUBase, self).__setitem__(key, value)
self._housekeep(key)
if len(self._store) > self._max_entries:
while len(self._store) > self._max_entries:
k = self._queue.popleft()
self._refcount[k] -= 1
if not self._refcount[k]:
super(LRUBase, self).__delitem__(k)
del self._refcount[k]
def _housekeep(self, key):
self._queue.append(key)
self._refcount[key] = self._refcount.get(key, 0) + 1
if len(self._queue) > self._max_entries * 4:
self._purge_queue()
def _purge_queue(self):
for i in [None] * len(self._queue):
k = self._queue.popleft()
if self._refcount[k] == 1:
self._queue.append(k)
else:
self._refcount[k] -= 1
class DbBase(Base):
'''Database common base class.'''
def __init__(self, engine, **kw):
super(DbBase, self).__init__(engine, **kw)
def __delitem__(self, key):
self._store.delete(self._store.c.key == key).execute()
def __len__(self):
return self._store.count().execute().fetchone()[0]
__all__ = ['Shove']

1
lib/shove/cache/__init__.py vendored
View file

@ -1 +0,0 @@
# -*- coding: utf-8 -*-

117
lib/shove/cache/db.py vendored
View file

@ -1,117 +0,0 @@
# -*- coding: utf-8 -*-
'''
Database object cache.
The shove psuedo-URL used for database object caches is the format used by
SQLAlchemy:
<driver>://<username>:<password>@<host>:<port>/<database>
<driver> is the database engine. The engines currently supported SQLAlchemy are
sqlite, mysql, postgres, oracle, mssql, and firebird.
<username> is the database account user name
<password> is the database accound password
<host> is the database location
<port> is the database port
<database> is the name of the specific database
For more information on specific databases see:
http://www.sqlalchemy.org/docs/dbengine.myt#dbengine_supported
'''
import time
import random
from datetime import datetime
try:
from sqlalchemy import (
MetaData, Table, Column, String, Binary, DateTime, select, update,
insert, delete,
)
from shove import DbBase
except ImportError:
raise ImportError('Requires SQLAlchemy >= 0.4')
__all__ = ['DbCache']
class DbCache(DbBase):
'''database cache backend'''
def __init__(self, engine, **kw):
super(DbCache, self).__init__(engine, **kw)
# Get table name
tablename = kw.get('tablename', 'cache')
# Bind metadata
self._metadata = MetaData(engine)
# Make cache table
self._store = Table(tablename, self._metadata,
Column('key', String(60), primary_key=True, nullable=False),
Column('value', Binary, nullable=False),
Column('expires', DateTime, nullable=False),
)
# Create cache table if it does not exist
if not self._store.exists():
self._store.create()
# Set maximum entries
self._max_entries = kw.get('max_entries', 300)
# Maximum number of entries to cull per call if cache is full
self._maxcull = kw.get('maxcull', 10)
# Set timeout
self.timeout = kw.get('timeout', 300)
def __getitem__(self, key):
row = select(
[self._store.c.value, self._store.c.expires],
self._store.c.key == key
).execute().fetchone()
if row is not None:
# Remove if item expired
if row.expires < datetime.now().replace(microsecond=0):
del self[key]
raise KeyError(key)
return self.loads(str(row.value))
raise KeyError(key)
def __setitem__(self, key, value):
timeout, value, cache = self.timeout, self.dumps(value), self._store
# Cull if too many items
if len(self) >= self._max_entries:
self._cull()
# Generate expiration time
expires = datetime.fromtimestamp(
time.time() + timeout
).replace(microsecond=0)
# Update database if key already present
if key in self:
update(
cache,
cache.c.key == key,
dict(value=value, expires=expires),
).execute()
# Insert new key if key not present
else:
insert(
cache, dict(key=key, value=value, expires=expires)
).execute()
def _cull(self):
'''Remove items in cache to make more room.'''
cache, maxcull = self._store, self._maxcull
# Remove items that have timed out
now = datetime.now().replace(microsecond=0)
delete(cache, cache.c.expires < now).execute()
# Remove any items over the maximum allowed number in the cache
if len(self) >= self._max_entries:
# Upper limit for key query
ul = maxcull * 2
# Get list of keys
keys = [
i[0] for i in select(
[cache.c.key], limit=ul
).execute().fetchall()
]
# Get some keys at random
delkeys = list(random.choice(keys) for i in xrange(maxcull))
delete(cache, cache.c.key.in_(delkeys)).execute()

46
lib/shove/cache/file.py vendored
View file

@ -1,46 +0,0 @@
# -*- coding: utf-8 -*-
'''
File-based cache
shove's psuedo-URL for file caches follows the form:
file://<path>
Where the path is a URL path to a directory on a local filesystem.
Alternatively, a native pathname to the directory can be passed as the 'engine'
argument.
'''
import time
from shove import FileBase
from shove.cache.simple import SimpleCache
class FileCache(FileBase, SimpleCache):
'''File-based cache backend'''
def __init__(self, engine, **kw):
super(FileCache, self).__init__(engine, **kw)
def __getitem__(self, key):
try:
exp, value = super(FileCache, self).__getitem__(key)
# Remove item if time has expired.
if exp < time.time():
del self[key]
raise KeyError(key)
return value
except:
raise KeyError(key)
def __setitem__(self, key, value):
if len(self) >= self._max_entries:
self._cull()
super(FileCache, self).__setitem__(
key, (time.time() + self.timeout, value)
)
__all__ = ['FileCache']

23
lib/shove/cache/filelru.py vendored
View file

@ -1,23 +0,0 @@
# -*- coding: utf-8 -*-
'''
File-based LRU cache
shove's psuedo-URL for file caches follows the form:
file://<path>
Where the path is a URL path to a directory on a local filesystem.
Alternatively, a native pathname to the directory can be passed as the 'engine'
argument.
'''
from shove import FileBase
from shove.cache.simplelru import SimpleLRUCache
class FileCache(FileBase, SimpleLRUCache):
'''File-based LRU cache backend'''
__all__ = ['FileCache']

43
lib/shove/cache/memcached.py vendored
View file

@ -1,43 +0,0 @@
# -*- coding: utf-8 -*-
'''
"memcached" cache.
The shove psuedo-URL for a memcache cache is:
memcache://<memcache_server>
'''
try:
import memcache
except ImportError:
raise ImportError("Memcache cache requires the 'memcache' library")
from shove import Base
class MemCached(Base):
'''Memcached cache backend'''
def __init__(self, engine, **kw):
super(MemCached, self).__init__(engine, **kw)
if engine.startswith('memcache://'):
engine = engine.split('://')[1]
self._store = memcache.Client(engine.split(';'))
# Set timeout
self.timeout = kw.get('timeout', 300)
def __getitem__(self, key):
value = self._store.get(key)
if value is None:
raise KeyError(key)
return self.loads(value)
def __setitem__(self, key, value):
self._store.set(key, self.dumps(value), self.timeout)
def __delitem__(self, key):
self._store.delete(key)
__all__ = ['MemCached']

38
lib/shove/cache/memlru.py vendored
View file

@ -1,38 +0,0 @@
# -*- coding: utf-8 -*-
'''
Thread-safe in-memory cache using LRU.
The shove psuedo-URL for a memory cache is:
memlru://
'''
import copy
import threading
from shove import synchronized
from shove.cache.simplelru import SimpleLRUCache
class MemoryLRUCache(SimpleLRUCache):
'''Thread-safe in-memory cache backend using LRU.'''
def __init__(self, engine, **kw):
super(MemoryLRUCache, self).__init__(engine, **kw)
self._lock = threading.Condition()
@synchronized
def __setitem__(self, key, value):
super(MemoryLRUCache, self).__setitem__(key, value)
@synchronized
def __getitem__(self, key):
return copy.deepcopy(super(MemoryLRUCache, self).__getitem__(key))
@synchronized
def __delitem__(self, key):
super(MemoryLRUCache, self).__delitem__(key)
__all__ = ['MemoryLRUCache']

38
lib/shove/cache/memory.py vendored
View file

@ -1,38 +0,0 @@
# -*- coding: utf-8 -*-
'''
Thread-safe in-memory cache.
The shove psuedo-URL for a memory cache is:
memory://
'''
import copy
import threading
from shove import synchronized
from shove.cache.simple import SimpleCache
class MemoryCache(SimpleCache):
'''Thread-safe in-memory cache backend.'''
def __init__(self, engine, **kw):
super(MemoryCache, self).__init__(engine, **kw)
self._lock = threading.Condition()
@synchronized
def __setitem__(self, key, value):
super(MemoryCache, self).__setitem__(key, value)
@synchronized
def __getitem__(self, key):
return copy.deepcopy(super(MemoryCache, self).__getitem__(key))
@synchronized
def __delitem__(self, key):
super(MemoryCache, self).__delitem__(key)
__all__ = ['MemoryCache']

45
lib/shove/cache/redisdb.py vendored
View file

@ -1,45 +0,0 @@
# -*- coding: utf-8 -*-
'''
Redis-based object cache
The shove psuedo-URL for a redis cache is:
redis://<host>:<port>/<db>
'''
import urlparse
try:
import redis
except ImportError:
raise ImportError('This store requires the redis library')
from shove import Base
class RedisCache(Base):
'''Redis cache backend'''
init = 'redis://'
def __init__(self, engine, **kw):
super(RedisCache, self).__init__(engine, **kw)
spliturl = urlparse.urlsplit(engine)
host, port = spliturl[1].split(':')
db = spliturl[2].replace('/', '')
self._store = redis.Redis(host, int(port), db)
# Set timeout
self.timeout = kw.get('timeout', 300)
def __getitem__(self, key):
return self.loads(self._store[key])
def __setitem__(self, key, value):
self._store.setex(key, self.dumps(value), self.timeout)
def __delitem__(self, key):
self._store.delete(key)
__all__ = ['RedisCache']

68
lib/shove/cache/simple.py vendored
View file

@ -1,68 +0,0 @@
# -*- coding: utf-8 -*-
'''
Single-process in-memory cache.
The shove psuedo-URL for a simple cache is:
simple://
'''
import time
import random
from shove import SimpleBase
class SimpleCache(SimpleBase):
'''Single-process in-memory cache.'''
def __init__(self, engine, **kw):
super(SimpleCache, self).__init__(engine, **kw)
# Get random seed
random.seed()
# Set maximum number of items to cull if over max
self._maxcull = kw.get('maxcull', 10)
# Set max entries
self._max_entries = kw.get('max_entries', 300)
# Set timeout
self.timeout = kw.get('timeout', 300)
def __getitem__(self, key):
exp, value = super(SimpleCache, self).__getitem__(key)
# Delete if item timed out.
if exp < time.time():
super(SimpleCache, self).__delitem__(key)
raise KeyError(key)
return value
def __setitem__(self, key, value):
# Cull values if over max # of entries
if len(self) >= self._max_entries:
self._cull()
# Set expiration time and value
exp = time.time() + self.timeout
super(SimpleCache, self).__setitem__(key, (exp, value))
def _cull(self):
'''Remove items in cache to make room.'''
num, maxcull = 0, self._maxcull
# Cull number of items allowed (set by self._maxcull)
for key in self.keys():
# Remove only maximum # of items allowed by maxcull
if num <= maxcull:
# Remove items if expired
try:
self[key]
except KeyError:
num += 1
else:
break
# Remove any additional items up to max # of items allowed by maxcull
while len(self) >= self._max_entries and num <= maxcull:
# Cull remainder of allowed quota at random
del self[random.choice(self.keys())]
num += 1
__all__ = ['SimpleCache']

18
lib/shove/cache/simplelru.py vendored
View file

@ -1,18 +0,0 @@
# -*- coding: utf-8 -*-
'''
Single-process in-memory LRU cache.
The shove psuedo-URL for a simple cache is:
simplelru://
'''
from shove import LRUBase
class SimpleLRUCache(LRUBase):
'''In-memory cache that purges based on least recently used item.'''
__all__ = ['SimpleLRUCache']

48
lib/shove/store/__init__.py
View file

@ -1,48 +0,0 @@
# -*- coding: utf-8 -*-
from urllib import url2pathname
from shove.store.simple import SimpleStore
class ClientStore(SimpleStore):
'''Base class for stores where updates have to be committed.'''
def __init__(self, engine, **kw):
super(ClientStore, self).__init__(engine, **kw)
if engine.startswith(self.init):
self._engine = url2pathname(engine.split('://')[1])
def __getitem__(self, key):
return self.loads(super(ClientStore, self).__getitem__(key))
def __setitem__(self, key, value):
super(ClientStore, self).__setitem__(key, self.dumps(value))
class SyncStore(ClientStore):
'''Base class for stores where updates have to be committed.'''
def __getitem__(self, key):
return self.loads(super(SyncStore, self).__getitem__(key))
def __setitem__(self, key, value):
super(SyncStore, self).__setitem__(key, value)
try:
self.sync()
except AttributeError:
pass
def __delitem__(self, key):
super(SyncStore, self).__delitem__(key)
try:
self.sync()
except AttributeError:
pass
__all__ = [
'bsdb', 'db', 'dbm', 'durusdb', 'file', 'ftp', 'memory', 's3', 'simple',
'svn', 'zodb', 'redisdb', 'hdf5db', 'leveldbstore', 'cassandra',
]

48
lib/shove/store/bsdb.py
View file

@ -1,48 +0,0 @@
# -*- coding: utf-8 -*-
'''
Berkeley Source Database Store.
shove's psuedo-URL for BSDDB stores follows the form:
bsddb://<path>
Where the path is a URL path to a Berkeley database. Alternatively, the native
pathname to a Berkeley database can be passed as the 'engine' parameter.
'''
try:
import bsddb
except ImportError:
raise ImportError('requires bsddb library')
import threading
from shove import synchronized
from shove.store import SyncStore
class BsdStore(SyncStore):
'''Class for Berkeley Source Database Store.'''
init = 'bsddb://'
def __init__(self, engine, **kw):
super(BsdStore, self).__init__(engine, **kw)
self._store = bsddb.hashopen(self._engine)
self._lock = threading.Condition()
self.sync = self._store.sync
@synchronized
def __getitem__(self, key):
return super(BsdStore, self).__getitem__(key)
@synchronized
def __setitem__(self, key, value):
super(BsdStore, self).__setitem__(key, value)
@synchronized
def __delitem__(self, key):
super(BsdStore, self).__delitem__(key)
__all__ = ['BsdStore']

72
lib/shove/store/cassandra.py
View file

@ -1,72 +0,0 @@
# -*- coding: utf-8 -*-
'''
Cassandra-based object store
The shove psuedo-URL for a cassandra-based store is:
cassandra://<host>:<port>/<keyspace>/<columnFamily>
'''
import urlparse
try:
import pycassa
except ImportError:
raise ImportError('This store requires the pycassa library')
from shove import BaseStore
class CassandraStore(BaseStore):
'''Cassandra based store'''
init = 'cassandra://'
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlparse.urlsplit(engine)
_, keyspace, column_family = spliturl[2].split('/')
try:
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace,
pycassa.system_manager.SIMPLE_STRATEGY,
{'replication_factor': str(kw.get('replication', 1))}
)
system_manager.create_column_family(keyspace, column_family)
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
def __getitem__(self, key):
try:
item = self._store.get(key).get(key)
if item is not None:
return self.loads(item)
raise KeyError(key)
except pycassa.NotFoundException:
raise KeyError(key)
def __setitem__(self, key, value):
self._store.insert(key, dict(key=self.dumps(value)))
def __delitem__(self, key):
# beware eventual consistency
try:
self._store.remove(key)
except pycassa.NotFoundException:
raise KeyError(key)
def clear(self):
# beware eventual consistency
self._store.truncate()
def keys(self):
return list(i[0] for i in self._store.get_range())
__all__ = ['CassandraStore']

73
lib/shove/store/db.py
View file

@ -1,73 +0,0 @@
# -*- coding: utf-8 -*-
'''
Database object store.
The shove psuedo-URL used for database object stores is the format used by
SQLAlchemy:
<driver>://<username>:<password>@<host>:<port>/<database>
<driver> is the database engine. The engines currently supported SQLAlchemy are
sqlite, mysql, postgres, oracle, mssql, and firebird.
<username> is the database account user name
<password> is the database accound password
<host> is the database location
<port> is the database port
<database> is the name of the specific database
For more information on specific databases see:
http://www.sqlalchemy.org/docs/dbengine.myt#dbengine_supported
'''
try:
from sqlalchemy import MetaData, Table, Column, String, Binary, select
from shove import BaseStore, DbBase
except ImportError, e:
raise ImportError('Error: ' + e + ' Requires SQLAlchemy >= 0.4')
class DbStore(BaseStore, DbBase):
'''Database cache backend.'''
def __init__(self, engine, **kw):
super(DbStore, self).__init__(engine, **kw)
# Get tablename
tablename = kw.get('tablename', 'store')
# Bind metadata
self._metadata = MetaData(engine)
# Make store table
self._store = Table(tablename, self._metadata,
Column('key', String(255), primary_key=True, nullable=False),
Column('value', Binary, nullable=False),
)
# Create store table if it does not exist
if not self._store.exists():
self._store.create()
def __getitem__(self, key):
row = select(
[self._store.c.value], self._store.c.key == key,
).execute().fetchone()
if row is not None:
return self.loads(str(row.value))
raise KeyError(key)
def __setitem__(self, k, v):
v, store = self.dumps(v), self._store
# Update database if key already present
if k in self:
store.update(store.c.key == k).execute(value=v)
# Insert new key if key not present
else:
store.insert().execute(key=k, value=v)
def keys(self):
'''Returns a list of keys in the store.'''
return list(i[0] for i in select(
[self._store.c.key]
).execute().fetchall())
__all__ = ['DbStore']

33
lib/shove/store/dbm.py
View file

@ -1,33 +0,0 @@
# -*- coding: utf-8 -*-
'''
DBM Database Store.
shove's psuedo-URL for DBM stores follows the form:
dbm://<path>
Where <path> is a URL path to a DBM database. Alternatively, the native
pathname to a DBM database can be passed as the 'engine' parameter.
'''
import anydbm
from shove.store import SyncStore
class DbmStore(SyncStore):
'''Class for variants of the DBM database.'''
init = 'dbm://'
def __init__(self, engine, **kw):
super(DbmStore, self).__init__(engine, **kw)
self._store = anydbm.open(self._engine, 'c')
try:
self.sync = self._store.sync
except AttributeError:
pass
__all__ = ['DbmStore']

43
lib/shove/store/durusdb.py
View file

@ -1,43 +0,0 @@
# -*- coding: utf-8 -*-
'''
Durus object database frontend.
shove's psuedo-URL for Durus stores follows the form:
durus://<path>
Where the path is a URL path to a durus FileStorage database. Alternatively, a
native pathname to a durus database can be passed as the 'engine' parameter.
'''
try:
from durus.connection import Connection
from durus.file_storage import FileStorage
except ImportError:
raise ImportError('Requires Durus library')
from shove.store import SyncStore
class DurusStore(SyncStore):
'''Class for Durus object database frontend.'''
init = 'durus://'
def __init__(self, engine, **kw):
super(DurusStore, self).__init__(engine, **kw)
self._db = FileStorage(self._engine)
self._connection = Connection(self._db)
self.sync = self._connection.commit
self._store = self._connection.get_root()
def close(self):
'''Closes all open storage and connections.'''
self.sync()
self._db.close()
super(DurusStore, self).close()
__all__ = ['DurusStore']

25
lib/shove/store/file.py
View file

@ -1,25 +0,0 @@
# -*- coding: utf-8 -*-
'''
Filesystem-based object store
shove's psuedo-URL for filesystem-based stores follows the form:
file://<path>
Where the path is a URL path to a directory on a local filesystem.
Alternatively, a native pathname to the directory can be passed as the 'engine'
argument.
'''
from shove import BaseStore, FileBase
class FileStore(FileBase, BaseStore):
'''File-based store.'''
def __init__(self, engine, **kw):
super(FileStore, self).__init__(engine, **kw)
__all__ = ['FileStore']

88
lib/shove/store/ftp.py
View file

@ -1,88 +0,0 @@
# -*- coding: utf-8 -*-
'''
FTP-accessed stores
shove's URL for FTP accessed stores follows the standard form for FTP URLs
defined in RFC-1738:
ftp://<user>:<password>@<host>:<port>/<url-path>
'''
import urlparse
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
from ftplib import FTP, error_perm
from shove import BaseStore
class FtpStore(BaseStore):
def __init__(self, engine, **kw):
super(FtpStore, self).__init__(engine, **kw)
user = kw.get('user', 'anonymous')
password = kw.get('password', '')
spliturl = urlparse.urlsplit(engine)
# Set URL, path, and strip 'ftp://' off
base, path = spliturl[1], spliturl[2] + '/'
if '@' in base:
auth, base = base.split('@')
user, password = auth.split(':')
self._store = FTP(base, user, password)
# Change to remote path if it exits
try:
self._store.cwd(path)
except error_perm:
self._makedir(path)
self._base, self._user, self._password = base, user, password
self._updated, self ._keys = True, None
def __getitem__(self, key):
try:
local = StringIO()
# Download item
self._store.retrbinary('RETR %s' % key, local.write)
self._updated = False
return self.loads(local.getvalue())
except:
raise KeyError(key)
def __setitem__(self, key, value):
local = StringIO(self.dumps(value))
self._store.storbinary('STOR %s' % key, local)
self._updated = True
def __delitem__(self, key):
try:
self._store.delete(key)
self._updated = True
except:
raise KeyError(key)
def _makedir(self, path):
'''Makes remote paths on an FTP server.'''
paths = list(reversed([i for i in path.split('/') if i != '']))
while paths:
tpath = paths.pop()
self._store.mkd(tpath)
self._store.cwd(tpath)
def keys(self):
'''Returns a list of keys in a store.'''
if self._updated or self._keys is None:
rlist, nlist = list(), list()
# Remote directory listing
self._store.retrlines('LIST -a', rlist.append)
for rlisting in rlist:
# Split remote file based on whitespace
rfile = rlisting.split()
# Append tuple of remote item type & name
if rfile[-1] not in ('.', '..') and rfile[0].startswith('-'):
nlist.append(rfile[-1])
self._keys = nlist
return self._keys
__all__ = ['FtpStore']

34
lib/shove/store/hdf5.py
View file

@ -1,34 +0,0 @@
# -*- coding: utf-8 -*-
'''
HDF5 Database Store.
shove's psuedo-URL for HDF5 stores follows the form:
hdf5://<path>/<group>
Where <path> is a URL path to a HDF5 database. Alternatively, the native
pathname to a HDF5 database can be passed as the 'engine' parameter.
<group> is the name of the database.
'''
try:
import h5py
except ImportError:
raise ImportError('This store requires h5py library')
from shove.store import ClientStore
class HDF5Store(ClientStore):
'''LevelDB based store'''
init = 'hdf5://'
def __init__(self, engine, **kw):
super(HDF5Store, self).__init__(engine, **kw)
engine, group = self._engine.rsplit('/')
self._store = h5py.File(engine).require_group(group).attrs
__all__ = ['HDF5Store']

47
lib/shove/store/leveldbstore.py
View file

@ -1,47 +0,0 @@
# -*- coding: utf-8 -*-
'''
LevelDB Database Store.
shove's psuedo-URL for LevelDB stores follows the form:
leveldb://<path>
Where <path> is a URL path to a LevelDB database. Alternatively, the native
pathname to a LevelDB database can be passed as the 'engine' parameter.
'''
try:
import leveldb
except ImportError:
raise ImportError('This store requires py-leveldb library')
from shove.store import ClientStore
class LevelDBStore(ClientStore):
'''LevelDB based store'''
init = 'leveldb://'
def __init__(self, engine, **kw):
super(LevelDBStore, self).__init__(engine, **kw)
self._store = leveldb.LevelDB(self._engine)
def __getitem__(self, key):
item = self.loads(self._store.Get(key))
if item is not None:
return item
raise KeyError(key)
def __setitem__(self, key, value):
self._store.Put(key, self.dumps(value))
def __delitem__(self, key):
self._store.Delete(key)
def keys(self):
return list(k for k in self._store.RangeIter(include_value=False))
__all__ = ['LevelDBStore']

38
lib/shove/store/memory.py
View file

@ -1,38 +0,0 @@
# -*- coding: utf-8 -*-
'''
Thread-safe in-memory store.
The shove psuedo-URL for a memory store is:
memory://
'''
import copy
import threading
from shove import synchronized
from shove.store.simple import SimpleStore
class MemoryStore(SimpleStore):
'''Thread-safe in-memory store.'''
def __init__(self, engine, **kw):
super(MemoryStore, self).__init__(engine, **kw)
self._lock = threading.Condition()
@synchronized
def __getitem__(self, key):
return copy.deepcopy(super(MemoryStore, self).__getitem__(key))
@synchronized
def __setitem__(self, key, value):
super(MemoryStore, self).__setitem__(key, value)
@synchronized
def __delitem__(self, key):
super(MemoryStore, self).__delitem__(key)
__all__ = ['MemoryStore']

50
lib/shove/store/redisdb.py
View file

@ -1,50 +0,0 @@
# -*- coding: utf-8 -*-
'''
Redis-based object store
The shove psuedo-URL for a redis-based store is:
redis://<host>:<port>/<db>
'''
import urlparse
try:
import redis
except ImportError:
raise ImportError('This store requires the redis library')
from shove.store import ClientStore
class RedisStore(ClientStore):
'''Redis based store'''
init = 'redis://'
def __init__(self, engine, **kw):
super(RedisStore, self).__init__(engine, **kw)
spliturl = urlparse.urlsplit(engine)
host, port = spliturl[1].split(':')
db = spliturl[2].replace('/', '')
self._store = redis.Redis(host, int(port), db)
def __contains__(self, key):
return self._store.exists(key)
def clear(self):
self._store.flushdb()
def keys(self):
return self._store.keys()
def setdefault(self, key, default=None):
return self._store.getset(key, default)
def update(self, other=None, **kw):
args = kw if other is not None else other
self._store.mset(args)
__all__ = ['RedisStore']

91
lib/shove/store/s3.py
View file

@ -1,91 +0,0 @@
# -*- coding: utf-8 -*-
'''
S3-accessed stores
shove's psuedo-URL for stores found on Amazon.com's S3 web service follows this
form:
s3://<s3_key>:<s3_secret>@<bucket>
<s3_key> is the Access Key issued by Amazon
<s3_secret> is the Secret Access Key issued by Amazon
<bucket> is the name of the bucket accessed through the S3 service
'''
try:
from boto.s3.connection import S3Connection
from boto.s3.key import Key
except ImportError:
raise ImportError('Requires boto library')
from shove import BaseStore
class S3Store(BaseStore):
def __init__(self, engine=None, **kw):
super(S3Store, self).__init__(engine, **kw)
# key = Access Key, secret=Secret Access Key, bucket=bucket name
key, secret, bucket = kw.get('key'), kw.get('secret'), kw.get('bucket')
if engine is not None:
auth, bucket = engine.split('://')[1].split('@')
key, secret = auth.split(':')
# kw 'secure' = (True or False, use HTTPS)
self._conn = S3Connection(key, secret, kw.get('secure', False))
buckets = self._conn.get_all_buckets()
# Use bucket if it exists
for b in buckets:
if b.name == bucket:
self._store = b
break
# Create bucket if it doesn't exist
else:
self._store = self._conn.create_bucket(bucket)
# Set bucket permission ('private', 'public-read',
# 'public-read-write', 'authenticated-read'
self._store.set_acl(kw.get('acl', 'private'))
# Updated flag used for avoiding network calls
self._updated, self._keys = True, None
def __getitem__(self, key):
rkey = self._store.lookup(key)
if rkey is None:
raise KeyError(key)
# Fetch string
value = self.loads(rkey.get_contents_as_string())
# Flag that the store has not been updated
self._updated = False
return value
def __setitem__(self, key, value):
rkey = Key(self._store)
rkey.key = key
rkey.set_contents_from_string(self.dumps(value))
# Flag that the store has been updated
self._updated = True
def __delitem__(self, key):
try:
self._store.delete_key(key)
# Flag that the store has been updated
self._updated = True
except:
raise KeyError(key)
def keys(self):
'''Returns a list of keys in the store.'''
return list(i[0] for i in self.items())
def items(self):
'''Returns a list of items from the store.'''
if self._updated or self._keys is None:
self._keys = self._store.get_all_keys()
return list((str(k.key), k) for k in self._keys)
def iteritems(self):
'''Lazily returns items from the store.'''
for k in self.items():
yield (k.key, k)
__all__ = ['S3Store']

21
lib/shove/store/simple.py
View file

@ -1,21 +0,0 @@
# -*- coding: utf-8 -*-
'''
Single-process in-memory store.
The shove psuedo-URL for a simple store is:
simple://
'''
from shove import BaseStore, SimpleBase
class SimpleStore(SimpleBase, BaseStore):
'''Single-process in-memory store.'''
def __init__(self, engine, **kw):
super(SimpleStore, self).__init__(engine, **kw)
__all__ = ['SimpleStore']

110
lib/shove/store/svn.py
View file

@ -1,110 +0,0 @@
# -*- coding: utf-8 -*-
'''
subversion managed store.
The shove psuedo-URL used for a subversion store that is password protected is:
svn:<username><password>:<path>?url=<url>
or for non-password protected repositories:
svn://<path>?url=<url>
<path> is the local repository copy
<url> is the URL of the subversion repository
'''
import os
import urllib
import threading
try:
import pysvn
except ImportError:
raise ImportError('Requires Python Subversion library')
from shove import BaseStore, synchronized
class SvnStore(BaseStore):
'''Class for subversion store.'''
def __init__(self, engine=None, **kw):
super(SvnStore, self).__init__(engine, **kw)
# Get path, url from keywords if used
path, url = kw.get('path'), kw.get('url')
# Get username. password from keywords if used
user, password = kw.get('user'), kw.get('password')
# Process psuedo URL if used
if engine is not None:
path, query = engine.split('n://')[1].split('?')
url = query.split('=')[1]
# Check for username, password
if '@' in path:
auth, path = path.split('@')
user, password = auth.split(':')
path = urllib.url2pathname(path)
# Create subversion client
self._client = pysvn.Client()
# Assign username, password
if user is not None:
self._client.set_username(user)
if password is not None:
self._client.set_password(password)
# Verify that store exists in repository
try:
self._client.info2(url)
# Create store in repository if it doesn't exist
except pysvn.ClientError:
self._client.mkdir(url, 'Adding directory')
# Verify that local copy exists
try:
if self._client.info(path) is None:
self._client.checkout(url, path)
# Check it out if it doesn't exist
except pysvn.ClientError:
self._client.checkout(url, path)
self._path, self._url = path, url
# Lock
self._lock = threading.Condition()
@synchronized
def __getitem__(self, key):
try:
return self.loads(self._client.cat(self._key_to_file(key)))
except:
raise KeyError(key)
@synchronized
def __setitem__(self, key, value):
fname = self._key_to_file(key)
# Write value to file
open(fname, 'wb').write(self.dumps(value))
# Add to repository
if key not in self:
self._client.add(fname)
self._client.checkin([fname], 'Adding %s' % fname)
@synchronized
def __delitem__(self, key):
try:
fname = self._key_to_file(key)
self._client.remove(fname)
# Remove deleted value from repository
self._client.checkin([fname], 'Removing %s' % fname)
except:
raise KeyError(key)
def _key_to_file(self, key):
'''Gives the filesystem path for a key.'''
return os.path.join(self._path, urllib.quote_plus(key))
@synchronized
def keys(self):
'''Returns a list of keys in the subversion repository.'''
return list(str(i.name.split('/')[-1]) for i
in self._client.ls(self._path))
__all__ = ['SvnStore']

48
lib/shove/store/zodb.py
View file

@ -1,48 +0,0 @@
# -*- coding: utf-8 -*-
'''
Zope Object Database store frontend.
shove's psuedo-URL for ZODB stores follows the form:
zodb:<path>
Where the path is a URL path to a ZODB FileStorage database. Alternatively, a
native pathname to a ZODB database can be passed as the 'engine' argument.
'''
try:
import transaction
from ZODB import FileStorage, DB
except ImportError:
raise ImportError('Requires ZODB library')
from shove.store import SyncStore
class ZodbStore(SyncStore):
'''ZODB store front end.'''
init = 'zodb://'
def __init__(self, engine, **kw):
super(ZodbStore, self).__init__(engine, **kw)
# Handle psuedo-URL
self._storage = FileStorage.FileStorage(self._engine)
self._db = DB(self._storage)
self._connection = self._db.open()
self._store = self._connection.root()
# Keeps DB in synch through commits of transactions
self.sync = transaction.commit
def close(self):
'''Closes all open storage and connections.'''
self.sync()
super(ZodbStore, self).close()
self._connection.close()
self._db.close()
self._storage.close()
__all__ = ['ZodbStore']

1
lib/shove/tests/__init__.py
View file

@ -1 +0,0 @@
# -*- coding: utf-8 -*-

133
lib/shove/tests/test_bsddb_store.py
View file

@ -1,133 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestBsdbStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('bsddb://test.db', compress=True)
def tearDown(self):
import os
self.store.close()
os.remove('test.db')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

137
lib/shove/tests/test_cassandra_store.py
View file

@ -1,137 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestCassandraStore(unittest.TestCase):
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Foo', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Foo/shove')
def tearDown(self):
self.store.clear()
self.store.close()
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
system_manager.drop_column_family('Foo', 'shove')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
# def test_clear(self):
# self.store['max'] = 3
# self.store['min'] = 6
# self.store['pow'] = 7
# self.store.clear()
# self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
# def test_popitem(self):
# self.store['max'] = 3
# self.store['min'] = 6
# self.store['pow'] = 7
# item = self.store.popitem()
# self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
# self.store['pow'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store.setdefault('pow', 8), 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

54
lib/shove/tests/test_db_cache.py
View file

@ -1,54 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestDbCache(unittest.TestCase):
initstring = 'sqlite:///'
def setUp(self):
from shove.cache.db import DbCache
self.cache = DbCache(self.initstring)
def tearDown(self):
self.cache = None
def test_getitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_setitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_delitem(self):
self.cache['test'] = 'test'
del self.cache['test']
self.assertEqual('test' in self.cache, False)
def test_get(self):
self.assertEqual(self.cache.get('min'), None)
def test_timeout(self):
import time
from shove.cache.db import DbCache
cache = DbCache(self.initstring, timeout=1)
cache['test'] = 'test'
time.sleep(2)
def tmp():
cache['test']
self.assertRaises(KeyError, tmp)
def test_cull(self):
from shove.cache.db import DbCache
cache = DbCache(self.initstring, max_entries=1)
cache['test'] = 'test'
cache['test2'] = 'test'
cache['test2'] = 'test'
self.assertEquals(len(cache), 1)
if __name__ == '__main__':
unittest.main()

131
lib/shove/tests/test_db_store.py
View file

@ -1,131 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestDbStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('sqlite://', compress=True)
def tearDown(self):
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

136
lib/shove/tests/test_dbm_store.py
View file

@ -1,136 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestDbmStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('dbm://test.dbm', compress=True)
def tearDown(self):
import os
self.store.close()
try:
os.remove('test.dbm.db')
except OSError:
pass
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.setdefault('how', 8)
self.assertEqual(self.store['how'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

133
lib/shove/tests/test_durus_store.py
View file

@ -1,133 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestDurusStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('durus://test.durus', compress=True)
def tearDown(self):
import os
self.store.close()
os.remove('test.durus')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

58
lib/shove/tests/test_file_cache.py
View file

@ -1,58 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestFileCache(unittest.TestCase):
initstring = 'file://test'
def setUp(self):
from shove.cache.file import FileCache
self.cache = FileCache(self.initstring)
def tearDown(self):
import os
self.cache = None
for x in os.listdir('test'):
os.remove(os.path.join('test', x))
os.rmdir('test')
def test_getitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_setitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_delitem(self):
self.cache['test'] = 'test'
del self.cache['test']
self.assertEqual('test' in self.cache, False)
def test_get(self):
self.assertEqual(self.cache.get('min'), None)
def test_timeout(self):
import time
from shove.cache.file import FileCache
cache = FileCache(self.initstring, timeout=1)
cache['test'] = 'test'
time.sleep(2)
def tmp():
cache['test']
self.assertRaises(KeyError, tmp)
def test_cull(self):
from shove.cache.file import FileCache
cache = FileCache(self.initstring, max_entries=1)
cache['test'] = 'test'
cache['test2'] = 'test'
num = len(cache)
self.assertEquals(num, 1)
if __name__ == '__main__':
unittest.main()

140
lib/shove/tests/test_file_store.py
View file

@ -1,140 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestFileStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('file://test', compress=True)
def tearDown(self):
import os
self.store.close()
for x in os.listdir('test'):
os.remove(os.path.join('test', x))
os.rmdir('test')
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

149
lib/shove/tests/test_ftp_store.py
View file

@ -1,149 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestFtpStore(unittest.TestCase):
ftpstring = 'put ftp string here'
def setUp(self):
from shove import Shove
self.store = Shove(self.ftpstring, compress=True)
def tearDown(self):
self.store.clear()
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
self.assertEqual(len(self.store), 2)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
item = self.store.popitem()
self.store.sync()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.store.sync()
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.sync()
self.store.update(tstore)
self.store.sync()
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

135
lib/shove/tests/test_hdf5_store.py
View file

@ -1,135 +0,0 @@
# -*- coding: utf-8 -*-
import unittest2
class TestHDF5Store(unittest2.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('hdf5://test.hdf5/test')
def tearDown(self):
import os
self.store.close()
try:
os.remove('test.hdf5')
except OSError:
pass
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.setdefault('bow', 8)
self.assertEqual(self.store['bow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest2.main()

132
lib/shove/tests/test_leveldb_store.py
View file

@ -1,132 +0,0 @@
# -*- coding: utf-8 -*-
import unittest2
class TestLevelDBStore(unittest2.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('leveldb://test', compress=True)
def tearDown(self):
import shutil
shutil.rmtree('test')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.setdefault('bow', 8)
self.assertEqual(self.store['bow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest2.main()

46
lib/shove/tests/test_memcached_cache.py
View file

@ -1,46 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestMemcached(unittest.TestCase):
initstring = 'memcache://localhost:11211'
def setUp(self):
from shove.cache.memcached import MemCached
self.cache = MemCached(self.initstring)
def tearDown(self):
self.cache = None
def test_getitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_setitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_delitem(self):
self.cache['test'] = 'test'
del self.cache['test']
self.assertEqual('test' in self.cache, False)
def test_get(self):
self.assertEqual(self.cache.get('min'), None)
def test_timeout(self):
import time
from shove.cache.memcached import MemCached
cache = MemCached(self.initstring, timeout=1)
cache['test'] = 'test'
time.sleep(1)
def tmp():
cache['test']
self.assertRaises(KeyError, tmp)
if __name__ == '__main__':
unittest.main()

54
lib/shove/tests/test_memory_cache.py
View file

@ -1,54 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestMemoryCache(unittest.TestCase):
initstring = 'memory://'
def setUp(self):
from shove.cache.memory import MemoryCache
self.cache = MemoryCache(self.initstring)
def tearDown(self):
self.cache = None
def test_getitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_setitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_delitem(self):
self.cache['test'] = 'test'
del self.cache['test']
self.assertEqual('test' in self.cache, False)
def test_get(self):
self.assertEqual(self.cache.get('min'), None)
def test_timeout(self):
import time
from shove.cache.memory import MemoryCache
cache = MemoryCache(self.initstring, timeout=1)
cache['test'] = 'test'
time.sleep(1)
def tmp():
cache['test']
self.assertRaises(KeyError, tmp)
def test_cull(self):
from shove.cache.memory import MemoryCache
cache = MemoryCache(self.initstring, max_entries=1)
cache['test'] = 'test'
cache['test2'] = 'test'
cache['test2'] = 'test'
self.assertEquals(len(cache), 1)
if __name__ == '__main__':
unittest.main()

135
lib/shove/tests/test_memory_store.py
View file

@ -1,135 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestMemoryStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('memory://', compress=True)
def tearDown(self):
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

45
lib/shove/tests/test_redis_cache.py
View file

@ -1,45 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestRedisCache(unittest.TestCase):
initstring = 'redis://localhost:6379/0'
def setUp(self):
from shove.cache.redisdb import RedisCache
self.cache = RedisCache(self.initstring)
def tearDown(self):
self.cache = None
def test_getitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_setitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_delitem(self):
self.cache['test'] = 'test'
del self.cache['test']
self.assertEqual('test' in self.cache, False)
def test_get(self):
self.assertEqual(self.cache.get('min'), None)
def test_timeout(self):
import time
from shove.cache.redisdb import RedisCache
cache = RedisCache(self.initstring, timeout=1)
cache['test'] = 'test'
time.sleep(3)
def tmp(): #@IgnorePep8
return cache['test']
self.assertRaises(KeyError, tmp)
if __name__ == '__main__':
unittest.main()

128
lib/shove/tests/test_redis_store.py
View file

@ -1,128 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestRedisStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('redis://localhost:6379/0')
def tearDown(self):
self.store.clear()
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store.setdefault('pow', 8), 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

149
lib/shove/tests/test_s3_store.py
View file

@ -1,149 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestS3Store(unittest.TestCase):
s3string = 's3 test string here'
def setUp(self):
from shove import Shove
self.store = Shove(self.s3string, compress=True)
def tearDown(self):
self.store.clear()
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
self.assertEqual(len(self.store), 2)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
item = self.store.popitem()
self.store.sync()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.store.sync()
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.sync()
self.store.update(tstore)
self.store.sync()
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

54
lib/shove/tests/test_simple_cache.py
View file

@ -1,54 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestSimpleCache(unittest.TestCase):
initstring = 'simple://'
def setUp(self):
from shove.cache.simple import SimpleCache
self.cache = SimpleCache(self.initstring)
def tearDown(self):
self.cache = None
def test_getitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_setitem(self):
self.cache['test'] = 'test'
self.assertEqual(self.cache['test'], 'test')
def test_delitem(self):
self.cache['test'] = 'test'
del self.cache['test']
self.assertEqual('test' in self.cache, False)
def test_get(self):
self.assertEqual(self.cache.get('min'), None)
def test_timeout(self):
import time
from shove.cache.simple import SimpleCache
cache = SimpleCache(self.initstring, timeout=1)
cache['test'] = 'test'
time.sleep(1)
def tmp():
cache['test']
self.assertRaises(KeyError, tmp)
def test_cull(self):
from shove.cache.simple import SimpleCache
cache = SimpleCache(self.initstring, max_entries=1)
cache['test'] = 'test'
cache['test2'] = 'test'
cache['test2'] = 'test'
self.assertEquals(len(cache), 1)
if __name__ == '__main__':
unittest.main()

135
lib/shove/tests/test_simple_store.py
View file

@ -1,135 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestSimpleStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('simple://', compress=True)
def tearDown(self):
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

148
lib/shove/tests/test_svn_store.py
View file

@ -1,148 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestSvnStore(unittest.TestCase):
svnstring = 'SVN test string here'
def setUp(self):
from shove import Shove
self.store = Shove(self.svnstring, compress=True)
def tearDown(self):
self.store.clear()
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
self.assertEqual(len(self.store), 2)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
item = self.store.popitem()
self.store.sync()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.store.sync()
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.sync()
self.store.update(tstore)
self.store.sync()
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

138
lib/shove/tests/test_zodb_store.py
View file

@ -1,138 +0,0 @@
# -*- coding: utf-8 -*-
import unittest
class TestZodbStore(unittest.TestCase):
init = 'zodb://test.db'
def setUp(self):
from shove import Shove
self.store = Shove(self.init, compress=True)
def tearDown(self):
self.store.close()
import os
os.remove('test.db')
os.remove('test.db.index')
os.remove('test.db.tmp')
os.remove('test.db.lock')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if __name__ == '__main__':
unittest.main()

63
sickbeard/rssfeeds.py
View file

@ -1,56 +1,31 @@
from __future__ import with_statement
import os
import urllib import urllib
import urlparse import urlparse
import re import re
from feedparser import feedparser
import sickbeard import sickbeard
from sickbeard import logger from sickbeard import logger
from sickbeard import encodingKludge as ek
from contextlib import closing
from sickbeard.exceptions import ex from sickbeard.exceptions import ex
from lib.feedcache import cache
from shove import Shove
def getFeed(url, post_data=None, request_headers=None):
parsed = list(urlparse.urlparse(url))
parsed[2] = re.sub("/{2,}", "/", parsed[2]) # replace two or more / with one
class RSSFeeds: if post_data:
def __init__(self, db_name): url += urllib.urlencode(post_data)
self.db_name = ek.ek(os.path.join, sickbeard.CACHE_DIR, 'rss', db_name + '.db')
if not os.path.exists(os.path.dirname(self.db_name)):
sickbeard.helpers.makeDir(os.path.dirname(self.db_name))
def clearCache(self, age=None): try:
try: feed = feedparser.parse(url, False, False, request_headers)
with closing(Shove('sqlite:///' + self.db_name, compress=True)) as fs:
fc = cache.Cache(fs)
fc.purge(age)
except Exception as e:
logger.log(u"RSS error clearing cache: " + ex(e), logger.DEBUG)
def getFeed(self, url, post_data=None, request_headers=None): if feed:
parsed = list(urlparse.urlparse(url)) if 'entries' in feed:
parsed[2] = re.sub("/{2,}", "/", parsed[2]) # replace two or more / with one return feed
elif 'error' in feed.feed:
err_code = feed.feed['error']['code']
err_desc = feed.feed['error']['description']
logger.log(u'RSS ERROR:[%s] CODE:[%s]' % (err_desc, err_code), logger.DEBUG)
else:
logger.log(u'RSS error loading url: ' + url, logger.DEBUG)
if post_data: except Exception as e:
url += urllib.urlencode(post_data) logger.log(u'RSS error: ' + ex(e), logger.DEBUG)
try:
with closing(Shove('sqlite:///' + self.db_name, compress=True)) as fs:
fc = cache.Cache(fs)
feed = fc.fetch(url, False, False, request_headers)
if feed:
if 'entries' in feed:
return feed
elif 'error' in feed.feed:
err_code = feed.feed['error']['code']
err_desc = feed.feed['error']['description']
logger.log(
u"RSS ERROR:[%s] CODE:[%s]" % (err_desc, err_code), logger.DEBUG)
else:
logger.log(u"RSS error loading url: " + url, logger.DEBUG)
except Exception as e:
logger.log(u"RSS error: " + ex(e), logger.DEBUG)

4
sickbeard/tvcache.py
View file

@ -30,7 +30,7 @@ from sickbeard import helpers, show_name_helpers
from sickbeard.exceptions import MultipleShowObjectsException from sickbeard.exceptions import MultipleShowObjectsException
from sickbeard.exceptions import AuthException from sickbeard.exceptions import AuthException
from name_parser.parser import NameParser, InvalidNameException, InvalidShowException from name_parser.parser import NameParser, InvalidNameException, InvalidShowException
from sickbeard.rssfeeds import RSSFeeds from sickbeard.rssfeeds import getFeed
from sickbeard import clients from sickbeard import clients
import itertools import itertools
@ -104,7 +104,7 @@ class TVCache():
return [] return []
def getRSSFeed(self, url, post_data=None, request_headers=None): def getRSSFeed(self, url, post_data=None, request_headers=None):
return RSSFeeds(self.providerID).getFeed(url, post_data, request_headers) return getFeed(url, post_data, request_headers)
def _translateTitle(self, title): def _translateTitle(self, title):
return u'' + title.replace(' ', '.') return u'' + title.replace(' ', '.')