diff --git a/CHANGES.md b/CHANGES.md index 1ca3e40b..452ae2d5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,6 +27,7 @@ * Fix show list view when no shows exist and "Group show lists shows into" is set to anything other than "One Show List" * Add coverage testing and coveralls support * Update feedparser library 5.1.3 to 5.2.0 (8c62940) +* Remove feedcache implementation and library [develop changelog] * Update Requests library 2.7.0 (ab1f493) to 2.7.0 (8b5e457) diff --git a/lib/feedcache/__init__.py b/lib/feedcache/__init__.py deleted file mode 100644 index 96ebc102..00000000 --- a/lib/feedcache/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Doug Hellmann. -# -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Doug -# Hellmann not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -""" - -""" - -__module_id__ = "$Id$" - -# -# Import system modules -# - - -# -# Import local modules -# -from cache import Cache - -# -# Module -# diff --git a/lib/feedcache/cache.py b/lib/feedcache/cache.py deleted file mode 100644 index 6ec0a39a..00000000 --- a/lib/feedcache/cache.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Doug Hellmann. -# -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Doug -# Hellmann not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -""" - -""" - -__module_id__ = "$Id$" - -# -# Import system modules -# -from feedparser import feedparser - -import logging -import time - -# -# Import local modules -# - - -# -# Module -# - -logger = logging.getLogger('feedcache.cache') - - -class Cache: - """A class to wrap Mark Pilgrim's Universal Feed Parser module - (http://www.feedparser.org) so that parameters can be used to - cache the feed results locally instead of fetching the feed every - time it is requested. Uses both etag and modified times for - caching. - """ - - def __init__(self, storage, timeToLiveSeconds=300, userAgent='feedcache'): - """ - Arguments: - - storage -- Backing store for the cache. It should follow - the dictionary API, with URLs used as keys. It should - persist data. - - timeToLiveSeconds=300 -- The length of time content should - live in the cache before an update is attempted. - - userAgent='feedcache' -- User agent string to be used when - fetching feed contents. - - """ - self.storage = storage - self.time_to_live = timeToLiveSeconds - self.user_agent = userAgent - return - - def purge(self, olderThanSeconds): - """Remove cached data from the storage if the data is older than the - date given. If olderThanSeconds is None, the entire cache is purged. - """ - if olderThanSeconds is None: - logger.debug('purging the entire cache') - for key in self.storage.keys(): - del self.storage[key] - else: - now = time.time() - # Iterate over the keys and load each item one at a time - # to avoid having the entire cache loaded into memory - # at one time. - for url in self.storage.keys(): - (cached_time, cached_data) = self.storage[url] - age = now - cached_time - if age >= olderThanSeconds: - logger.debug('removing %s with age %d', url, age) - del self.storage[url] - return - - def fetch(self, url, force_update=False, offline=False, request_headers=None): - """Return the feed at url. - - url - The URL of the feed. - - force_update=False - When True, update the cache whether the - current contents have - exceeded their time-to-live - or not. - - offline=False - When True, only return data from the local - cache and never access the remote - URL. - - If there is data for that feed in the cache already, check - the expiration date before accessing the server. If the - cached data has not expired, return it without accessing the - server. - - In cases where the server is accessed, check for updates - before deciding what to return. If the server reports a - status of 304, the previously cached content is returned. - - The cache is only updated if the server returns a status of - 200, to avoid holding redirected data in the cache. - """ - logger.debug('url="%s"' % url) - - # Convert the URL to a value we can use - # as a key for the storage backend. - key = url - if isinstance(key, unicode): - key = key.encode('utf-8') - - modified = None - etag = None - now = time.time() - - cached_time, cached_content = self.storage.get(key, (None, None)) - - # Offline mode support (no networked requests) - # so return whatever we found in the storage. - # If there is nothing in the storage, we'll be returning None. - if offline: - logger.debug('offline mode') - return cached_content - - # Does the storage contain a version of the data - # which is older than the time-to-live? - logger.debug('cache modified time: %s' % str(cached_time)) - if cached_time is not None and not force_update: - if self.time_to_live: - age = now - cached_time - if age <= self.time_to_live: - logger.debug('cache contents still valid') - return cached_content - else: - logger.debug('cache contents older than TTL') - else: - logger.debug('no TTL value') - - # The cache is out of date, but we have - # something. Try to use the etag and modified_time - # values from the cached content. - etag = cached_content.get('etag') - modified = cached_content.get('modified') - logger.debug('cached etag=%s' % etag) - logger.debug('cached modified=%s' % str(modified)) - else: - logger.debug('nothing in the cache, or forcing update') - - # We know we need to fetch, so go ahead and do it. - logger.debug('fetching...') - parsed_result = feedparser.parse(url, - agent=self.user_agent, - modified=modified, - etag=etag, - request_headers=request_headers) - - status = parsed_result.get('status', None) - logger.debug('HTTP status=%s' % status) - if status == 304: - # No new data, based on the etag or modified values. - # We need to update the modified time in the - # storage, though, so we know that what we have - # stored is up to date. - self.storage[key] = (now, cached_content) - - # Return the data from the cache, since - # the parsed data will be empty. - parsed_result = cached_content - elif status == 200: - # There is new content, so store it unless there was an error. - error = parsed_result.get('bozo_exception') - if not error: - logger.debug('Updating stored data for %s' % url) - self.storage[key] = (now, parsed_result) - else: - logger.warning('Not storing data with exception: %s', - error) - else: - logger.warning('Not updating cache with HTTP status %s', status) - - return parsed_result diff --git a/lib/feedcache/cachestoragelock.py b/lib/feedcache/cachestoragelock.py deleted file mode 100644 index 05babde6..00000000 --- a/lib/feedcache/cachestoragelock.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Doug Hellmann. -# -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Doug -# Hellmann not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# -from __future__ import with_statement - -"""Lock wrapper for cache storage which do not permit multi-threaded access. - -""" - -__module_id__ = "$Id$" - -# -# Import system modules -# -import threading - -# -# Import local modules -# - - -# -# Module -# - -class CacheStorageLock: - """Lock wrapper for cache storage which do not permit multi-threaded access. - """ - - def __init__(self, shelf): - self.lock = threading.Lock() - self.shelf = shelf - return - - def __getitem__(self, key): - with self.lock: - return self.shelf[key] - - def get(self, key, default=None): - with self.lock: - try: - return self.shelf[key] - except KeyError: - return default - - def __setitem__(self, key, value): - with self.lock: - self.shelf[key] = value diff --git a/lib/feedcache/example.py b/lib/feedcache/example.py deleted file mode 100644 index 4df7ab68..00000000 --- a/lib/feedcache/example.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Doug Hellmann. -# -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Doug -# Hellmann not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -"""Example use of feedcache.Cache. - -""" - -__module_id__ = "$Id$" - -# -# Import system modules -# -import sys -import shelve - -# -# Import local modules -# -import cache - -# -# Module -# - -def main(urls=[]): - print 'Saving feed data to ./.feedcache' - storage = shelve.open('.feedcache') - try: - fc = cache.Cache(storage) - for url in urls: - parsed_data = fc.fetch(url) - print parsed_data.feed.title - for entry in parsed_data.entries: - print '\t', entry.title - finally: - storage.close() - return - -if __name__ == '__main__': - main(sys.argv[1:]) - diff --git a/lib/feedcache/example_threads.py b/lib/feedcache/example_threads.py deleted file mode 100644 index 2eb56d30..00000000 --- a/lib/feedcache/example_threads.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Doug Hellmann. -# -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Doug -# Hellmann not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -"""Example use of feedcache.Cache combined with threads. - -""" - -__module_id__ = "$Id$" - -# -# Import system modules -# -import Queue -import sys -import shove -import threading - -# -# Import local modules -# -import cache - -# -# Module -# - -MAX_THREADS=5 -OUTPUT_DIR='/tmp/feedcache_example' - - -def main(urls=[]): - - if not urls: - print 'Specify the URLs to a few RSS or Atom feeds on the command line.' - return - - # Decide how many threads to start - num_threads = min(len(urls), MAX_THREADS) - - # Add the URLs to a queue - url_queue = Queue.Queue() - for url in urls: - url_queue.put(url) - - # Add poison pills to the url queue to cause - # the worker threads to break out of their loops - for i in range(num_threads): - url_queue.put(None) - - # Track the entries in the feeds being fetched - entry_queue = Queue.Queue() - - print 'Saving feed data to', OUTPUT_DIR - storage = shove.Shove('file://' + OUTPUT_DIR) - try: - - # Start a few worker threads - worker_threads = [] - for i in range(num_threads): - t = threading.Thread(target=fetch_urls, - args=(storage, url_queue, entry_queue,)) - worker_threads.append(t) - t.setDaemon(True) - t.start() - - # Start a thread to print the results - printer_thread = threading.Thread(target=print_entries, args=(entry_queue,)) - printer_thread.setDaemon(True) - printer_thread.start() - - # Wait for all of the URLs to be processed - url_queue.join() - - # Wait for the worker threads to finish - for t in worker_threads: - t.join() - - # Poison the print thread and wait for it to exit - entry_queue.put((None,None)) - entry_queue.join() - printer_thread.join() - - finally: - storage.close() - return - - -def fetch_urls(storage, input_queue, output_queue): - """Thread target for fetching feed data. - """ - c = cache.Cache(storage) - - while True: - next_url = input_queue.get() - if next_url is None: # None causes thread to exit - input_queue.task_done() - break - - feed_data = c.fetch(next_url) - for entry in feed_data.entries: - output_queue.put( (feed_data.feed, entry) ) - input_queue.task_done() - return - - -def print_entries(input_queue): - """Thread target for printing the contents of the feeds. - """ - while True: - feed, entry = input_queue.get() - if feed is None: # None causes thread to exist - input_queue.task_done() - break - - print '%s: %s' % (feed.title, entry.title) - input_queue.task_done() - return - - -if __name__ == '__main__': - main(sys.argv[1:]) - diff --git a/lib/feedcache/test_cache.py b/lib/feedcache/test_cache.py deleted file mode 100644 index 2c1ac096..00000000 --- a/lib/feedcache/test_cache.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Doug Hellmann. -# -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Doug -# Hellmann not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -"""Unittests for feedcache.cache - -""" - -__module_id__ = "$Id$" - -import logging -logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(levelname)-8s %(name)s %(message)s', - ) -logger = logging.getLogger('feedcache.test_cache') - -# -# Import system modules -# -import copy -import time -import unittest -import UserDict - -# -# Import local modules -# -import cache -from test_server import HTTPTestBase, TestHTTPServer - -# -# Module -# - - -class CacheTestBase(HTTPTestBase): - - CACHE_TTL = 30 - - def setUp(self): - HTTPTestBase.setUp(self) - - self.storage = self.getStorage() - self.cache = cache.Cache(self.storage, - timeToLiveSeconds=self.CACHE_TTL, - userAgent='feedcache.test', - ) - return - - def getStorage(self): - "Return a cache storage for the test." - return {} - - -class CacheTest(CacheTestBase): - - CACHE_TTL = 30 - - def getServer(self): - "These tests do not want to use the ETag or If-Modified-Since headers" - return TestHTTPServer(applyModifiedHeaders=False) - - def testRetrieveNotInCache(self): - # Retrieve data not already in the cache. - feed_data = self.cache.fetch(self.TEST_URL) - self.failUnless(feed_data) - self.failUnlessEqual(feed_data.feed.title, 'CacheTest test data') - return - - def testRetrieveIsInCache(self): - # Retrieve data which is alread in the cache, - # and verify that the second copy is identitical - # to the first. - - # First fetch - feed_data = self.cache.fetch(self.TEST_URL) - - # Second fetch - feed_data2 = self.cache.fetch(self.TEST_URL) - - # Since it is the in-memory storage, we should have the - # exact same object. - self.failUnless(feed_data is feed_data2) - return - - def testExpireDataInCache(self): - # Retrieve data which is in the cache but which - # has expired and verify that the second copy - # is different from the first. - - # First fetch - feed_data = self.cache.fetch(self.TEST_URL) - - # Change the timeout and sleep to move the clock - self.cache.time_to_live = 0 - time.sleep(1) - - # Second fetch - feed_data2 = self.cache.fetch(self.TEST_URL) - - # Since we reparsed, the cache response should be different. - self.failIf(feed_data is feed_data2) - return - - def testForceUpdate(self): - # Force cache to retrieve data which is alread in the cache, - # and verify that the new data is different. - - # Pre-populate the storage with bad data - self.cache.storage[self.TEST_URL] = (time.time() + 100, self.id()) - - # Fetch the data - feed_data = self.cache.fetch(self.TEST_URL, force_update=True) - - self.failIfEqual(feed_data, self.id()) - return - - def testOfflineMode(self): - # Retrieve data which is alread in the cache, - # whether it is expired or not. - - # Pre-populate the storage with data - self.cache.storage[self.TEST_URL] = (0, self.id()) - - # Fetch it - feed_data = self.cache.fetch(self.TEST_URL, offline=True) - - self.failUnlessEqual(feed_data, self.id()) - return - - def testUnicodeURL(self): - # Pass in a URL which is unicode - - url = unicode(self.TEST_URL) - feed_data = self.cache.fetch(url) - - storage = self.cache.storage - key = unicode(self.TEST_URL).encode('UTF-8') - - # Verify that the storage has a key - self.failUnless(key in storage) - - # Now pull the data from the storage directly - storage_timeout, storage_data = self.cache.storage.get(key) - self.failUnlessEqual(feed_data, storage_data) - return - - -class SingleWriteMemoryStorage(UserDict.UserDict): - """Cache storage which only allows the cache value - for a URL to be updated one time. - """ - - def __setitem__(self, url, data): - if url in self.keys(): - modified, existing = self[url] - # Allow the modified time to change, - # but not the feed content. - if data[1] != existing: - raise AssertionError('Trying to update cache for %s to %s' \ - % (url, data)) - UserDict.UserDict.__setitem__(self, url, data) - return - - -class CacheConditionalGETTest(CacheTestBase): - - CACHE_TTL = 0 - - def getStorage(self): - return SingleWriteMemoryStorage() - - def testFetchOnceForEtag(self): - # Fetch data which has a valid ETag value, and verify - # that while we hit the server twice the response - # codes cause us to use the same data. - - # First fetch populates the cache - response1 = self.cache.fetch(self.TEST_URL) - self.failUnlessEqual(response1.feed.title, 'CacheTest test data') - - # Remove the modified setting from the cache so we know - # the next time we check the etag will be used - # to check for updates. Since we are using an in-memory - # cache, modifying response1 updates the cache storage - # directly. - response1['modified'] = None - - # This should result in a 304 status, and no data from - # the server. That means the cache won't try to - # update the storage, so our SingleWriteMemoryStorage - # should not raise and we should have the same - # response object. - response2 = self.cache.fetch(self.TEST_URL) - self.failUnless(response1 is response2) - - # Should have hit the server twice - self.failUnlessEqual(self.server.getNumRequests(), 2) - return - - def testFetchOnceForModifiedTime(self): - # Fetch data which has a valid Last-Modified value, and verify - # that while we hit the server twice the response - # codes cause us to use the same data. - - # First fetch populates the cache - response1 = self.cache.fetch(self.TEST_URL) - self.failUnlessEqual(response1.feed.title, 'CacheTest test data') - - # Remove the etag setting from the cache so we know - # the next time we check the modified time will be used - # to check for updates. Since we are using an in-memory - # cache, modifying response1 updates the cache storage - # directly. - response1['etag'] = None - - # This should result in a 304 status, and no data from - # the server. That means the cache won't try to - # update the storage, so our SingleWriteMemoryStorage - # should not raise and we should have the same - # response object. - response2 = self.cache.fetch(self.TEST_URL) - self.failUnless(response1 is response2) - - # Should have hit the server twice - self.failUnlessEqual(self.server.getNumRequests(), 2) - return - - -class CacheRedirectHandlingTest(CacheTestBase): - - def _test(self, response): - # Set up the server to redirect requests, - # then verify that the cache is not updated - # for the original or new URL and that the - # redirect status is fed back to us with - # the fetched data. - - self.server.setResponse(response, '/redirected') - - response1 = self.cache.fetch(self.TEST_URL) - - # The response should include the status code we set - self.failUnlessEqual(response1.get('status'), response) - - # The response should include the new URL, too - self.failUnlessEqual(response1.href, self.TEST_URL + 'redirected') - - # The response should not have been cached under either URL - self.failIf(self.TEST_URL in self.storage) - self.failIf(self.TEST_URL + 'redirected' in self.storage) - return - - def test301(self): - self._test(301) - - def test302(self): - self._test(302) - - def test303(self): - self._test(303) - - def test307(self): - self._test(307) - - -class CachePurgeTest(CacheTestBase): - - def testPurgeAll(self): - # Remove everything from the cache - - self.cache.fetch(self.TEST_URL) - self.failUnless(self.storage.keys(), - 'Have no data in the cache storage') - - self.cache.purge(None) - - self.failIf(self.storage.keys(), - 'Still have data in the cache storage') - return - - def testPurgeByAge(self): - # Remove old content from the cache - - self.cache.fetch(self.TEST_URL) - self.failUnless(self.storage.keys(), - 'have no data in the cache storage') - - time.sleep(1) - - remains = (time.time(), copy.deepcopy(self.storage[self.TEST_URL][1])) - self.storage['http://this.should.remain/'] = remains - - self.cache.purge(1) - - self.failUnlessEqual(self.storage.keys(), - ['http://this.should.remain/']) - return - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/feedcache/test_cachestoragelock.py b/lib/feedcache/test_cachestoragelock.py deleted file mode 100644 index 741a39ab..00000000 --- a/lib/feedcache/test_cachestoragelock.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Doug Hellmann. -# -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Doug -# Hellmann not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -"""Tests for shelflock. - -""" - -__module_id__ = "$Id$" - -# -# Import system modules -# -import os -import shelve -import tempfile -import threading -import unittest - -# -# Import local modules -# -from cache import Cache -from cachestoragelock import CacheStorageLock -from test_server import HTTPTestBase - -# -# Module -# - -class CacheShelveTest(HTTPTestBase): - - def setUp(self): - HTTPTestBase.setUp(self) - handle, self.shelve_filename = tempfile.mkstemp('.shelve') - os.close(handle) # we just want the file name, so close the open handle - os.unlink(self.shelve_filename) # remove the empty file - return - - def tearDown(self): - try: - os.unlink(self.shelve_filename) - except AttributeError: - pass - HTTPTestBase.tearDown(self) - return - - def test(self): - storage = shelve.open(self.shelve_filename) - locking_storage = CacheStorageLock(storage) - try: - fc = Cache(locking_storage) - - # First fetch the data through the cache - parsed_data = fc.fetch(self.TEST_URL) - self.failUnlessEqual(parsed_data.feed.title, 'CacheTest test data') - - # Now retrieve the same data directly from the shelf - modified, shelved_data = storage[self.TEST_URL] - - # The data should be the same - self.failUnlessEqual(parsed_data, shelved_data) - finally: - storage.close() - return - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/feedcache/test_server.py b/lib/feedcache/test_server.py deleted file mode 100644 index f48be105..00000000 --- a/lib/feedcache/test_server.py +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Doug Hellmann. -# -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Doug -# Hellmann not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -"""Simple HTTP server for testing the feed cache. - -""" - -__module_id__ = "$Id$" - -# -# Import system modules -# -import BaseHTTPServer -import logging -import md5 -import threading -import time -import unittest -import urllib - -# -# Import local modules -# - - -# -# Module -# -logger = logging.getLogger('feedcache.test_server') - - -def make_etag(data): - """Given a string containing data to be returned to the client, - compute an ETag value for the data. - """ - _md5 = md5.new() - _md5.update(data) - return _md5.hexdigest() - - -class TestHTTPHandler(BaseHTTPServer.BaseHTTPRequestHandler): - "HTTP request handler which serves the same feed data every time." - - FEED_DATA = """ - - - CacheTest test data - - - http://localhost/feedcache/ - 2006-10-14T11:00:36Z - - single test entry - - 2006-10-14T11:00:36Z - - author goes here - authoremail@example.com - - http://www.example.com/ - description goes here - - - -""" - - # The data does not change, so save the ETag and modified times - # as class attributes. - ETAG = make_etag(FEED_DATA) - # Calculated using email.utils.formatdate(usegmt=True) - MODIFIED_TIME = 'Sun, 08 Apr 2012 20:16:48 GMT' - - def do_GET(self): - "Handle GET requests." - logger.debug('GET %s', self.path) - - if self.path == '/shutdown': - # Shortcut to handle stopping the server - logger.debug('Stopping server') - self.server.stop() - self.send_response(200) - - else: - # Record the request for tests that count them - self.server.requests.append(self.path) - # Process the request - logger.debug('pre-defined response code: %d', self.server.response) - handler_method_name = 'do_GET_%d' % self.server.response - handler_method = getattr(self, handler_method_name) - handler_method() - return - - def do_GET_3xx(self): - "Handle redirects" - if self.path.endswith('/redirected'): - logger.debug('already redirected') - # We have already redirected, so return the data. - return self.do_GET_200() - new_path = self.server.new_path - logger.debug('redirecting to %s', new_path) - self.send_response(self.server.response) - self.send_header('Location', new_path) - return - - do_GET_301 = do_GET_3xx - do_GET_302 = do_GET_3xx - do_GET_303 = do_GET_3xx - do_GET_307 = do_GET_3xx - - def do_GET_200(self): - logger.debug('Etag: %s' % self.ETAG) - logger.debug('Last-Modified: %s' % self.MODIFIED_TIME) - - incoming_etag = self.headers.get('If-None-Match', None) - logger.debug('Incoming ETag: "%s"' % incoming_etag) - - incoming_modified = self.headers.get('If-Modified-Since', None) - logger.debug('Incoming If-Modified-Since: %s' % incoming_modified) - - send_data = True - - # Does the client have the same version of the data we have? - if self.server.apply_modified_headers: - if incoming_etag == self.ETAG: - logger.debug('Response 304, etag') - self.send_response(304) - send_data = False - - elif incoming_modified == self.MODIFIED_TIME: - logger.debug('Response 304, modified time') - self.send_response(304) - send_data = False - - # Now optionally send the data, if the client needs it - if send_data: - logger.debug('Response 200') - self.send_response(200) - - self.send_header('Content-Type', 'application/atom+xml') - - logger.debug('Outgoing Etag: %s' % self.ETAG) - self.send_header('ETag', self.ETAG) - - logger.debug('Outgoing modified time: %s' % self.MODIFIED_TIME) - self.send_header('Last-Modified', self.MODIFIED_TIME) - - self.end_headers() - - logger.debug('Sending data') - self.wfile.write(self.FEED_DATA) - return - - -class TestHTTPServer(BaseHTTPServer.HTTPServer): - """HTTP Server which counts the number of requests made - and can stop based on client instructions. - """ - - def __init__(self, applyModifiedHeaders=True, handler=TestHTTPHandler): - self.apply_modified_headers = applyModifiedHeaders - self.keep_serving = True - self.requests = [] - self.setResponse(200) - BaseHTTPServer.HTTPServer.__init__(self, ('', 9999), handler) - return - - def setResponse(self, newResponse, newPath=None): - """Sets the response code to use for future requests, and a new - path to be used as a redirect target, if necessary. - """ - self.response = newResponse - self.new_path = newPath - return - - def getNumRequests(self): - "Return the number of requests which have been made on the server." - return len(self.requests) - - def stop(self): - "Stop serving requests, after the next request." - self.keep_serving = False - return - - def serve_forever(self): - "Main loop for server" - while self.keep_serving: - self.handle_request() - logger.debug('exiting') - return - - -class HTTPTestBase(unittest.TestCase): - "Base class for tests that use a TestHTTPServer" - - TEST_URL = 'http://localhost:9999/' - - CACHE_TTL = 0 - - def setUp(self): - self.server = self.getServer() - self.server_thread = threading.Thread(target=self.server.serve_forever) - # set daemon flag so the tests don't hang if cleanup fails - self.server_thread.setDaemon(True) - self.server_thread.start() - return - - def getServer(self): - "Return a web server for the test." - s = TestHTTPServer() - s.setResponse(200) - return s - - def tearDown(self): - # Stop the server thread - urllib.urlretrieve(self.TEST_URL + 'shutdown') - time.sleep(1) - self.server.server_close() - self.server_thread.join() - return diff --git a/lib/feedcache/test_shovefilesystem.py b/lib/feedcache/test_shovefilesystem.py deleted file mode 100644 index 1a48dead..00000000 --- a/lib/feedcache/test_shovefilesystem.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2007 Doug Hellmann. -# -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Doug -# Hellmann not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# DOUG HELLMANN DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL DOUG HELLMANN BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -"""Tests with shove filesystem storage. - -""" - -__module_id__ = "$Id$" - -# -# Import system modules -# -import os -import shove -import tempfile -import threading -import unittest - -# -# Import local modules -# -from cache import Cache -from test_server import HTTPTestBase - -# -# Module -# - -class CacheShoveTest(HTTPTestBase): - - def setUp(self): - HTTPTestBase.setUp(self) - self.shove_dirname = tempfile.mkdtemp('shove') - return - - def tearDown(self): - try: - os.system('rm -rf %s' % self.storage_dirname) - except AttributeError: - pass - HTTPTestBase.tearDown(self) - return - - def test(self): - # First fetch the data through the cache - storage = shove.Shove('file://' + self.shove_dirname) - try: - fc = Cache(storage) - parsed_data = fc.fetch(self.TEST_URL) - self.failUnlessEqual(parsed_data.feed.title, 'CacheTest test data') - finally: - storage.close() - - # Now retrieve the same data directly from the shelf - storage = shove.Shove('file://' + self.shove_dirname) - try: - modified, shelved_data = storage[self.TEST_URL] - finally: - storage.close() - - # The data should be the same - self.failUnlessEqual(parsed_data, shelved_data) - return - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/__init__.py b/lib/shove/__init__.py deleted file mode 100644 index 3be119b4..00000000 --- a/lib/shove/__init__.py +++ /dev/null @@ -1,519 +0,0 @@ -# -*- coding: utf-8 -*- -'''Common object storage frontend.''' - -import os -import zlib -import urllib -try: - import cPickle as pickle -except ImportError: - import pickle -from collections import deque - -try: - # Import store and cache entry points if setuptools installed - import pkg_resources - stores = dict((_store.name, _store) for _store in - pkg_resources.iter_entry_points('shove.stores')) - caches = dict((_cache.name, _cache) for _cache in - pkg_resources.iter_entry_points('shove.caches')) - # Pass if nothing loaded - if not stores and not caches: - raise ImportError() -except ImportError: - # Static store backend registry - stores = dict( - bsddb='shove.store.bsdb:BsdStore', - cassandra='shove.store.cassandra:CassandraStore', - dbm='shove.store.dbm:DbmStore', - durus='shove.store.durusdb:DurusStore', - file='shove.store.file:FileStore', - firebird='shove.store.db:DbStore', - ftp='shove.store.ftp:FtpStore', - hdf5='shove.store.hdf5:HDF5Store', - leveldb='shove.store.leveldbstore:LevelDBStore', - memory='shove.store.memory:MemoryStore', - mssql='shove.store.db:DbStore', - mysql='shove.store.db:DbStore', - oracle='shove.store.db:DbStore', - postgres='shove.store.db:DbStore', - redis='shove.store.redisdb:RedisStore', - s3='shove.store.s3:S3Store', - simple='shove.store.simple:SimpleStore', - sqlite='shove.store.db:DbStore', - svn='shove.store.svn:SvnStore', - zodb='shove.store.zodb:ZodbStore', - ) - # Static cache backend registry - caches = dict( - bsddb='shove.cache.bsdb:BsdCache', - file='shove.cache.file:FileCache', - filelru='shove.cache.filelru:FileLRUCache', - firebird='shove.cache.db:DbCache', - memcache='shove.cache.memcached:MemCached', - memlru='shove.cache.memlru:MemoryLRUCache', - memory='shove.cache.memory:MemoryCache', - mssql='shove.cache.db:DbCache', - mysql='shove.cache.db:DbCache', - oracle='shove.cache.db:DbCache', - postgres='shove.cache.db:DbCache', - redis='shove.cache.redisdb:RedisCache', - simple='shove.cache.simple:SimpleCache', - simplelru='shove.cache.simplelru:SimpleLRUCache', - sqlite='shove.cache.db:DbCache', - ) - - -def getbackend(uri, engines, **kw): - ''' - Loads the right backend based on a URI. - - @param uri Instance or name string - @param engines A dictionary of scheme/class pairs - ''' - if isinstance(uri, basestring): - mod = engines[uri.split('://', 1)[0]] - # Load module if setuptools not present - if isinstance(mod, basestring): - # Isolate classname from dot path - module, klass = mod.split(':') - # Load module - mod = getattr(__import__(module, '', '', ['']), klass) - # Load appropriate class from setuptools entry point - else: - mod = mod.load() - # Return instance - return mod(uri, **kw) - # No-op for existing instances - return uri - - -def synchronized(func): - ''' - Decorator to lock and unlock a method (Phillip J. Eby). - - @param func Method to decorate - ''' - def wrapper(self, *__args, **__kw): - self._lock.acquire() - try: - return func(self, *__args, **__kw) - finally: - self._lock.release() - wrapper.__name__ = func.__name__ - wrapper.__dict__ = func.__dict__ - wrapper.__doc__ = func.__doc__ - return wrapper - - -class Base(object): - - '''Base Mapping class.''' - - def __init__(self, engine, **kw): - ''' - @keyword compress True, False, or an integer compression level (1-9). - ''' - self._compress = kw.get('compress', False) - self._protocol = kw.get('protocol', pickle.HIGHEST_PROTOCOL) - - def __getitem__(self, key): - raise NotImplementedError() - - def __setitem__(self, key, value): - raise NotImplementedError() - - def __delitem__(self, key): - raise NotImplementedError() - - def __contains__(self, key): - try: - value = self[key] - except KeyError: - return False - return True - - def get(self, key, default=None): - ''' - Fetch a given key from the mapping. If the key does not exist, - return the default. - - @param key Keyword of item in mapping. - @param default Default value (default: None) - ''' - try: - return self[key] - except KeyError: - return default - - def dumps(self, value): - '''Optionally serializes and compresses an object.''' - # Serialize everything but ASCII strings - value = pickle.dumps(value, protocol=self._protocol) - if self._compress: - level = 9 if self._compress is True else self._compress - value = zlib.compress(value, level) - return value - - def loads(self, value): - '''Deserializes and optionally decompresses an object.''' - if self._compress: - try: - value = zlib.decompress(value) - except zlib.error: - pass - value = pickle.loads(value) - return value - - -class BaseStore(Base): - - '''Base Store class (based on UserDict.DictMixin).''' - - def __init__(self, engine, **kw): - super(BaseStore, self).__init__(engine, **kw) - self._store = None - - def __cmp__(self, other): - if other is None: - return False - if isinstance(other, BaseStore): - return cmp(dict(self.iteritems()), dict(other.iteritems())) - - def __del__(self): - # __init__ didn't succeed, so don't bother closing - if not hasattr(self, '_store'): - return - self.close() - - def __iter__(self): - for k in self.keys(): - yield k - - def __len__(self): - return len(self.keys()) - - def __repr__(self): - return repr(dict(self.iteritems())) - - def close(self): - '''Closes internal store and clears object references.''' - try: - self._store.close() - except AttributeError: - pass - self._store = None - - def clear(self): - '''Removes all keys and values from a store.''' - for key in self.keys(): - del self[key] - - def items(self): - '''Returns a list with all key/value pairs in the store.''' - return list(self.iteritems()) - - def iteritems(self): - '''Lazily returns all key/value pairs in a store.''' - for k in self: - yield (k, self[k]) - - def iterkeys(self): - '''Lazy returns all keys in a store.''' - return self.__iter__() - - def itervalues(self): - '''Lazily returns all values in a store.''' - for _, v in self.iteritems(): - yield v - - def keys(self): - '''Returns a list with all keys in a store.''' - raise NotImplementedError() - - def pop(self, key, *args): - ''' - Removes and returns a value from a store. - - @param args Default to return if key not present. - ''' - if len(args) > 1: - raise TypeError('pop expected at most 2 arguments, got ' + repr( - 1 + len(args)) - ) - try: - value = self[key] - # Return default if key not in store - except KeyError: - if args: - return args[0] - del self[key] - return value - - def popitem(self): - '''Removes and returns a key, value pair from a store.''' - try: - k, v = self.iteritems().next() - except StopIteration: - raise KeyError('Store is empty.') - del self[k] - return (k, v) - - def setdefault(self, key, default=None): - ''' - Returns the value corresponding to an existing key or sets the - to key to the default and returns the default. - - @param default Default value (default: None) - ''' - try: - return self[key] - except KeyError: - self[key] = default - return default - - def update(self, other=None, **kw): - ''' - Adds to or overwrites the values in this store with values from - another store. - - other Another store - kw Additional keys and values to store - ''' - if other is None: - pass - elif hasattr(other, 'iteritems'): - for k, v in other.iteritems(): - self[k] = v - elif hasattr(other, 'keys'): - for k in other.keys(): - self[k] = other[k] - else: - for k, v in other: - self[k] = v - if kw: - self.update(kw) - - def values(self): - '''Returns a list with all values in a store.''' - return list(v for _, v in self.iteritems()) - - -class Shove(BaseStore): - - '''Common object frontend class.''' - - def __init__(self, store='simple://', cache='simple://', **kw): - super(Shove, self).__init__(store, **kw) - # Load store - self._store = getbackend(store, stores, **kw) - # Load cache - self._cache = getbackend(cache, caches, **kw) - # Buffer for lazy writing and setting for syncing frequency - self._buffer, self._sync = dict(), kw.get('sync', 2) - - def __getitem__(self, key): - '''Gets a item from shove.''' - try: - return self._cache[key] - except KeyError: - # Synchronize cache and store - self.sync() - value = self._store[key] - self._cache[key] = value - return value - - def __setitem__(self, key, value): - '''Sets an item in shove.''' - self._cache[key] = self._buffer[key] = value - # When the buffer reaches self._limit, writes the buffer to the store - if len(self._buffer) >= self._sync: - self.sync() - - def __delitem__(self, key): - '''Deletes an item from shove.''' - try: - del self._cache[key] - except KeyError: - pass - self.sync() - del self._store[key] - - def keys(self): - '''Returns a list of keys in shove.''' - self.sync() - return self._store.keys() - - def sync(self): - '''Writes buffer to store.''' - for k, v in self._buffer.iteritems(): - self._store[k] = v - self._buffer.clear() - - def close(self): - '''Finalizes and closes shove.''' - # If close has been called, pass - if self._store is not None: - try: - self.sync() - except AttributeError: - pass - self._store.close() - self._store = self._cache = self._buffer = None - - -class FileBase(Base): - - '''Base class for file based storage.''' - - def __init__(self, engine, **kw): - super(FileBase, self).__init__(engine, **kw) - if engine.startswith('file://'): - engine = urllib.url2pathname(engine.split('://')[1]) - self._dir = engine - # Create directory - if not os.path.exists(self._dir): - self._createdir() - - def __getitem__(self, key): - # (per Larry Meyn) - try: - item = open(self._key_to_file(key), 'rb') - data = item.read() - item.close() - return self.loads(data) - except: - raise KeyError(key) - - def __setitem__(self, key, value): - # (per Larry Meyn) - try: - item = open(self._key_to_file(key), 'wb') - item.write(self.dumps(value)) - item.close() - except (IOError, OSError): - raise KeyError(key) - - def __delitem__(self, key): - try: - os.remove(self._key_to_file(key)) - except (IOError, OSError): - raise KeyError(key) - - def __contains__(self, key): - return os.path.exists(self._key_to_file(key)) - - def __len__(self): - return len(os.listdir(self._dir)) - - def _createdir(self): - '''Creates the store directory.''' - try: - os.makedirs(self._dir) - except OSError: - raise EnvironmentError( - 'Cache directory "%s" does not exist and ' \ - 'could not be created' % self._dir - ) - - def _key_to_file(self, key): - '''Gives the filesystem path for a key.''' - return os.path.join(self._dir, urllib.quote_plus(key)) - - def keys(self): - '''Returns a list of keys in the store.''' - return [urllib.unquote_plus(name) for name in os.listdir(self._dir)] - - -class SimpleBase(Base): - - '''Single-process in-memory store base class.''' - - def __init__(self, engine, **kw): - super(SimpleBase, self).__init__(engine, **kw) - self._store = dict() - - def __getitem__(self, key): - try: - return self._store[key] - except: - raise KeyError(key) - - def __setitem__(self, key, value): - self._store[key] = value - - def __delitem__(self, key): - try: - del self._store[key] - except: - raise KeyError(key) - - def __len__(self): - return len(self._store) - - def keys(self): - '''Returns a list of keys in the store.''' - return self._store.keys() - - -class LRUBase(SimpleBase): - - def __init__(self, engine, **kw): - super(LRUBase, self).__init__(engine, **kw) - self._max_entries = kw.get('max_entries', 300) - self._hits = 0 - self._misses = 0 - self._queue = deque() - self._refcount = dict() - - def __getitem__(self, key): - try: - value = super(LRUBase, self).__getitem__(key) - self._hits += 1 - except KeyError: - self._misses += 1 - raise - self._housekeep(key) - return value - - def __setitem__(self, key, value): - super(LRUBase, self).__setitem__(key, value) - self._housekeep(key) - if len(self._store) > self._max_entries: - while len(self._store) > self._max_entries: - k = self._queue.popleft() - self._refcount[k] -= 1 - if not self._refcount[k]: - super(LRUBase, self).__delitem__(k) - del self._refcount[k] - - def _housekeep(self, key): - self._queue.append(key) - self._refcount[key] = self._refcount.get(key, 0) + 1 - if len(self._queue) > self._max_entries * 4: - self._purge_queue() - - def _purge_queue(self): - for i in [None] * len(self._queue): - k = self._queue.popleft() - if self._refcount[k] == 1: - self._queue.append(k) - else: - self._refcount[k] -= 1 - - -class DbBase(Base): - - '''Database common base class.''' - - def __init__(self, engine, **kw): - super(DbBase, self).__init__(engine, **kw) - - def __delitem__(self, key): - self._store.delete(self._store.c.key == key).execute() - - def __len__(self): - return self._store.count().execute().fetchone()[0] - - -__all__ = ['Shove'] diff --git a/lib/shove/cache/__init__.py b/lib/shove/cache/__init__.py deleted file mode 100644 index 40a96afc..00000000 --- a/lib/shove/cache/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/lib/shove/cache/db.py b/lib/shove/cache/db.py deleted file mode 100644 index 21fea01f..00000000 --- a/lib/shove/cache/db.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Database object cache. - -The shove psuedo-URL used for database object caches is the format used by -SQLAlchemy: - -://:@:/ - - is the database engine. The engines currently supported SQLAlchemy are -sqlite, mysql, postgres, oracle, mssql, and firebird. - is the database account user name - is the database accound password - is the database location - is the database port - is the name of the specific database - -For more information on specific databases see: - -http://www.sqlalchemy.org/docs/dbengine.myt#dbengine_supported -''' - -import time -import random -from datetime import datetime -try: - from sqlalchemy import ( - MetaData, Table, Column, String, Binary, DateTime, select, update, - insert, delete, - ) - from shove import DbBase -except ImportError: - raise ImportError('Requires SQLAlchemy >= 0.4') - -__all__ = ['DbCache'] - - -class DbCache(DbBase): - - '''database cache backend''' - - def __init__(self, engine, **kw): - super(DbCache, self).__init__(engine, **kw) - # Get table name - tablename = kw.get('tablename', 'cache') - # Bind metadata - self._metadata = MetaData(engine) - # Make cache table - self._store = Table(tablename, self._metadata, - Column('key', String(60), primary_key=True, nullable=False), - Column('value', Binary, nullable=False), - Column('expires', DateTime, nullable=False), - ) - # Create cache table if it does not exist - if not self._store.exists(): - self._store.create() - # Set maximum entries - self._max_entries = kw.get('max_entries', 300) - # Maximum number of entries to cull per call if cache is full - self._maxcull = kw.get('maxcull', 10) - # Set timeout - self.timeout = kw.get('timeout', 300) - - def __getitem__(self, key): - row = select( - [self._store.c.value, self._store.c.expires], - self._store.c.key == key - ).execute().fetchone() - if row is not None: - # Remove if item expired - if row.expires < datetime.now().replace(microsecond=0): - del self[key] - raise KeyError(key) - return self.loads(str(row.value)) - raise KeyError(key) - - def __setitem__(self, key, value): - timeout, value, cache = self.timeout, self.dumps(value), self._store - # Cull if too many items - if len(self) >= self._max_entries: - self._cull() - # Generate expiration time - expires = datetime.fromtimestamp( - time.time() + timeout - ).replace(microsecond=0) - # Update database if key already present - if key in self: - update( - cache, - cache.c.key == key, - dict(value=value, expires=expires), - ).execute() - # Insert new key if key not present - else: - insert( - cache, dict(key=key, value=value, expires=expires) - ).execute() - - def _cull(self): - '''Remove items in cache to make more room.''' - cache, maxcull = self._store, self._maxcull - # Remove items that have timed out - now = datetime.now().replace(microsecond=0) - delete(cache, cache.c.expires < now).execute() - # Remove any items over the maximum allowed number in the cache - if len(self) >= self._max_entries: - # Upper limit for key query - ul = maxcull * 2 - # Get list of keys - keys = [ - i[0] for i in select( - [cache.c.key], limit=ul - ).execute().fetchall() - ] - # Get some keys at random - delkeys = list(random.choice(keys) for i in xrange(maxcull)) - delete(cache, cache.c.key.in_(delkeys)).execute() diff --git a/lib/shove/cache/file.py b/lib/shove/cache/file.py deleted file mode 100644 index 7b9a4ae7..00000000 --- a/lib/shove/cache/file.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -''' -File-based cache - -shove's psuedo-URL for file caches follows the form: - -file:// - -Where the path is a URL path to a directory on a local filesystem. -Alternatively, a native pathname to the directory can be passed as the 'engine' -argument. -''' - -import time - -from shove import FileBase -from shove.cache.simple import SimpleCache - - -class FileCache(FileBase, SimpleCache): - - '''File-based cache backend''' - - def __init__(self, engine, **kw): - super(FileCache, self).__init__(engine, **kw) - - def __getitem__(self, key): - try: - exp, value = super(FileCache, self).__getitem__(key) - # Remove item if time has expired. - if exp < time.time(): - del self[key] - raise KeyError(key) - return value - except: - raise KeyError(key) - - def __setitem__(self, key, value): - if len(self) >= self._max_entries: - self._cull() - super(FileCache, self).__setitem__( - key, (time.time() + self.timeout, value) - ) - - -__all__ = ['FileCache'] diff --git a/lib/shove/cache/filelru.py b/lib/shove/cache/filelru.py deleted file mode 100644 index de076613..00000000 --- a/lib/shove/cache/filelru.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -''' -File-based LRU cache - -shove's psuedo-URL for file caches follows the form: - -file:// - -Where the path is a URL path to a directory on a local filesystem. -Alternatively, a native pathname to the directory can be passed as the 'engine' -argument. -''' - -from shove import FileBase -from shove.cache.simplelru import SimpleLRUCache - - -class FileCache(FileBase, SimpleLRUCache): - - '''File-based LRU cache backend''' - - -__all__ = ['FileCache'] diff --git a/lib/shove/cache/memcached.py b/lib/shove/cache/memcached.py deleted file mode 100644 index aedfe282..00000000 --- a/lib/shove/cache/memcached.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -''' -"memcached" cache. - -The shove psuedo-URL for a memcache cache is: - -memcache:// -''' - -try: - import memcache -except ImportError: - raise ImportError("Memcache cache requires the 'memcache' library") - -from shove import Base - - -class MemCached(Base): - - '''Memcached cache backend''' - - def __init__(self, engine, **kw): - super(MemCached, self).__init__(engine, **kw) - if engine.startswith('memcache://'): - engine = engine.split('://')[1] - self._store = memcache.Client(engine.split(';')) - # Set timeout - self.timeout = kw.get('timeout', 300) - - def __getitem__(self, key): - value = self._store.get(key) - if value is None: - raise KeyError(key) - return self.loads(value) - - def __setitem__(self, key, value): - self._store.set(key, self.dumps(value), self.timeout) - - def __delitem__(self, key): - self._store.delete(key) - - -__all__ = ['MemCached'] diff --git a/lib/shove/cache/memlru.py b/lib/shove/cache/memlru.py deleted file mode 100644 index 7db61ec5..00000000 --- a/lib/shove/cache/memlru.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Thread-safe in-memory cache using LRU. - -The shove psuedo-URL for a memory cache is: - -memlru:// -''' - -import copy -import threading - -from shove import synchronized -from shove.cache.simplelru import SimpleLRUCache - - -class MemoryLRUCache(SimpleLRUCache): - - '''Thread-safe in-memory cache backend using LRU.''' - - def __init__(self, engine, **kw): - super(MemoryLRUCache, self).__init__(engine, **kw) - self._lock = threading.Condition() - - @synchronized - def __setitem__(self, key, value): - super(MemoryLRUCache, self).__setitem__(key, value) - - @synchronized - def __getitem__(self, key): - return copy.deepcopy(super(MemoryLRUCache, self).__getitem__(key)) - - @synchronized - def __delitem__(self, key): - super(MemoryLRUCache, self).__delitem__(key) - - -__all__ = ['MemoryLRUCache'] diff --git a/lib/shove/cache/memory.py b/lib/shove/cache/memory.py deleted file mode 100644 index e70f9bbb..00000000 --- a/lib/shove/cache/memory.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Thread-safe in-memory cache. - -The shove psuedo-URL for a memory cache is: - -memory:// -''' - -import copy -import threading - -from shove import synchronized -from shove.cache.simple import SimpleCache - - -class MemoryCache(SimpleCache): - - '''Thread-safe in-memory cache backend.''' - - def __init__(self, engine, **kw): - super(MemoryCache, self).__init__(engine, **kw) - self._lock = threading.Condition() - - @synchronized - def __setitem__(self, key, value): - super(MemoryCache, self).__setitem__(key, value) - - @synchronized - def __getitem__(self, key): - return copy.deepcopy(super(MemoryCache, self).__getitem__(key)) - - @synchronized - def __delitem__(self, key): - super(MemoryCache, self).__delitem__(key) - - -__all__ = ['MemoryCache'] diff --git a/lib/shove/cache/redisdb.py b/lib/shove/cache/redisdb.py deleted file mode 100644 index c53536c1..00000000 --- a/lib/shove/cache/redisdb.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Redis-based object cache - -The shove psuedo-URL for a redis cache is: - -redis://:/ -''' - -import urlparse - -try: - import redis -except ImportError: - raise ImportError('This store requires the redis library') - -from shove import Base - - -class RedisCache(Base): - - '''Redis cache backend''' - - init = 'redis://' - - def __init__(self, engine, **kw): - super(RedisCache, self).__init__(engine, **kw) - spliturl = urlparse.urlsplit(engine) - host, port = spliturl[1].split(':') - db = spliturl[2].replace('/', '') - self._store = redis.Redis(host, int(port), db) - # Set timeout - self.timeout = kw.get('timeout', 300) - - def __getitem__(self, key): - return self.loads(self._store[key]) - - def __setitem__(self, key, value): - self._store.setex(key, self.dumps(value), self.timeout) - - def __delitem__(self, key): - self._store.delete(key) - - -__all__ = ['RedisCache'] diff --git a/lib/shove/cache/simple.py b/lib/shove/cache/simple.py deleted file mode 100644 index 6855603e..00000000 --- a/lib/shove/cache/simple.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Single-process in-memory cache. - -The shove psuedo-URL for a simple cache is: - -simple:// -''' - -import time -import random - -from shove import SimpleBase - - -class SimpleCache(SimpleBase): - - '''Single-process in-memory cache.''' - - def __init__(self, engine, **kw): - super(SimpleCache, self).__init__(engine, **kw) - # Get random seed - random.seed() - # Set maximum number of items to cull if over max - self._maxcull = kw.get('maxcull', 10) - # Set max entries - self._max_entries = kw.get('max_entries', 300) - # Set timeout - self.timeout = kw.get('timeout', 300) - - def __getitem__(self, key): - exp, value = super(SimpleCache, self).__getitem__(key) - # Delete if item timed out. - if exp < time.time(): - super(SimpleCache, self).__delitem__(key) - raise KeyError(key) - return value - - def __setitem__(self, key, value): - # Cull values if over max # of entries - if len(self) >= self._max_entries: - self._cull() - # Set expiration time and value - exp = time.time() + self.timeout - super(SimpleCache, self).__setitem__(key, (exp, value)) - - def _cull(self): - '''Remove items in cache to make room.''' - num, maxcull = 0, self._maxcull - # Cull number of items allowed (set by self._maxcull) - for key in self.keys(): - # Remove only maximum # of items allowed by maxcull - if num <= maxcull: - # Remove items if expired - try: - self[key] - except KeyError: - num += 1 - else: - break - # Remove any additional items up to max # of items allowed by maxcull - while len(self) >= self._max_entries and num <= maxcull: - # Cull remainder of allowed quota at random - del self[random.choice(self.keys())] - num += 1 - - -__all__ = ['SimpleCache'] diff --git a/lib/shove/cache/simplelru.py b/lib/shove/cache/simplelru.py deleted file mode 100644 index fbb6e446..00000000 --- a/lib/shove/cache/simplelru.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Single-process in-memory LRU cache. - -The shove psuedo-URL for a simple cache is: - -simplelru:// -''' - -from shove import LRUBase - - -class SimpleLRUCache(LRUBase): - - '''In-memory cache that purges based on least recently used item.''' - - -__all__ = ['SimpleLRUCache'] diff --git a/lib/shove/store/__init__.py b/lib/shove/store/__init__.py deleted file mode 100644 index 5d639a07..00000000 --- a/lib/shove/store/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -from urllib import url2pathname -from shove.store.simple import SimpleStore - - -class ClientStore(SimpleStore): - - '''Base class for stores where updates have to be committed.''' - - def __init__(self, engine, **kw): - super(ClientStore, self).__init__(engine, **kw) - if engine.startswith(self.init): - self._engine = url2pathname(engine.split('://')[1]) - - def __getitem__(self, key): - return self.loads(super(ClientStore, self).__getitem__(key)) - - def __setitem__(self, key, value): - super(ClientStore, self).__setitem__(key, self.dumps(value)) - - -class SyncStore(ClientStore): - - '''Base class for stores where updates have to be committed.''' - - def __getitem__(self, key): - return self.loads(super(SyncStore, self).__getitem__(key)) - - def __setitem__(self, key, value): - super(SyncStore, self).__setitem__(key, value) - try: - self.sync() - except AttributeError: - pass - - def __delitem__(self, key): - super(SyncStore, self).__delitem__(key) - try: - self.sync() - except AttributeError: - pass - - -__all__ = [ - 'bsdb', 'db', 'dbm', 'durusdb', 'file', 'ftp', 'memory', 's3', 'simple', - 'svn', 'zodb', 'redisdb', 'hdf5db', 'leveldbstore', 'cassandra', -] diff --git a/lib/shove/store/bsdb.py b/lib/shove/store/bsdb.py deleted file mode 100644 index d1f9c6dc..00000000 --- a/lib/shove/store/bsdb.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Berkeley Source Database Store. - -shove's psuedo-URL for BSDDB stores follows the form: - -bsddb:// - -Where the path is a URL path to a Berkeley database. Alternatively, the native -pathname to a Berkeley database can be passed as the 'engine' parameter. -''' -try: - import bsddb -except ImportError: - raise ImportError('requires bsddb library') - -import threading - -from shove import synchronized -from shove.store import SyncStore - - -class BsdStore(SyncStore): - - '''Class for Berkeley Source Database Store.''' - - init = 'bsddb://' - - def __init__(self, engine, **kw): - super(BsdStore, self).__init__(engine, **kw) - self._store = bsddb.hashopen(self._engine) - self._lock = threading.Condition() - self.sync = self._store.sync - - @synchronized - def __getitem__(self, key): - return super(BsdStore, self).__getitem__(key) - - @synchronized - def __setitem__(self, key, value): - super(BsdStore, self).__setitem__(key, value) - - @synchronized - def __delitem__(self, key): - super(BsdStore, self).__delitem__(key) - - -__all__ = ['BsdStore'] diff --git a/lib/shove/store/cassandra.py b/lib/shove/store/cassandra.py deleted file mode 100644 index 1f6532ee..00000000 --- a/lib/shove/store/cassandra.py +++ /dev/null @@ -1,72 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Cassandra-based object store - -The shove psuedo-URL for a cassandra-based store is: - -cassandra://:// -''' - -import urlparse - -try: - import pycassa -except ImportError: - raise ImportError('This store requires the pycassa library') - -from shove import BaseStore - - -class CassandraStore(BaseStore): - - '''Cassandra based store''' - - init = 'cassandra://' - - def __init__(self, engine, **kw): - super(CassandraStore, self).__init__(engine, **kw) - spliturl = urlparse.urlsplit(engine) - _, keyspace, column_family = spliturl[2].split('/') - try: - self._pool = pycassa.connect(keyspace, [spliturl[1]]) - self._store = pycassa.ColumnFamily(self._pool, column_family) - except pycassa.InvalidRequestException: - from pycassa.system_manager import SystemManager - system_manager = SystemManager(spliturl[1]) - system_manager.create_keyspace( - keyspace, - pycassa.system_manager.SIMPLE_STRATEGY, - {'replication_factor': str(kw.get('replication', 1))} - ) - system_manager.create_column_family(keyspace, column_family) - self._pool = pycassa.connect(keyspace, [spliturl[1]]) - self._store = pycassa.ColumnFamily(self._pool, column_family) - - def __getitem__(self, key): - try: - item = self._store.get(key).get(key) - if item is not None: - return self.loads(item) - raise KeyError(key) - except pycassa.NotFoundException: - raise KeyError(key) - - def __setitem__(self, key, value): - self._store.insert(key, dict(key=self.dumps(value))) - - def __delitem__(self, key): - # beware eventual consistency - try: - self._store.remove(key) - except pycassa.NotFoundException: - raise KeyError(key) - - def clear(self): - # beware eventual consistency - self._store.truncate() - - def keys(self): - return list(i[0] for i in self._store.get_range()) - - -__all__ = ['CassandraStore'] diff --git a/lib/shove/store/db.py b/lib/shove/store/db.py deleted file mode 100644 index 0004e6f8..00000000 --- a/lib/shove/store/db.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Database object store. - -The shove psuedo-URL used for database object stores is the format used by -SQLAlchemy: - -://:@:/ - - is the database engine. The engines currently supported SQLAlchemy are -sqlite, mysql, postgres, oracle, mssql, and firebird. - is the database account user name - is the database accound password - is the database location - is the database port - is the name of the specific database - -For more information on specific databases see: - -http://www.sqlalchemy.org/docs/dbengine.myt#dbengine_supported -''' - -try: - from sqlalchemy import MetaData, Table, Column, String, Binary, select - from shove import BaseStore, DbBase -except ImportError, e: - raise ImportError('Error: ' + e + ' Requires SQLAlchemy >= 0.4') - - -class DbStore(BaseStore, DbBase): - - '''Database cache backend.''' - - def __init__(self, engine, **kw): - super(DbStore, self).__init__(engine, **kw) - # Get tablename - tablename = kw.get('tablename', 'store') - # Bind metadata - self._metadata = MetaData(engine) - # Make store table - self._store = Table(tablename, self._metadata, - Column('key', String(255), primary_key=True, nullable=False), - Column('value', Binary, nullable=False), - ) - # Create store table if it does not exist - if not self._store.exists(): - self._store.create() - - def __getitem__(self, key): - row = select( - [self._store.c.value], self._store.c.key == key, - ).execute().fetchone() - if row is not None: - return self.loads(str(row.value)) - raise KeyError(key) - - def __setitem__(self, k, v): - v, store = self.dumps(v), self._store - # Update database if key already present - if k in self: - store.update(store.c.key == k).execute(value=v) - # Insert new key if key not present - else: - store.insert().execute(key=k, value=v) - - def keys(self): - '''Returns a list of keys in the store.''' - return list(i[0] for i in select( - [self._store.c.key] - ).execute().fetchall()) - - -__all__ = ['DbStore'] diff --git a/lib/shove/store/dbm.py b/lib/shove/store/dbm.py deleted file mode 100644 index 323d2484..00000000 --- a/lib/shove/store/dbm.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -''' -DBM Database Store. - -shove's psuedo-URL for DBM stores follows the form: - -dbm:// - -Where is a URL path to a DBM database. Alternatively, the native -pathname to a DBM database can be passed as the 'engine' parameter. -''' - -import anydbm - -from shove.store import SyncStore - - -class DbmStore(SyncStore): - - '''Class for variants of the DBM database.''' - - init = 'dbm://' - - def __init__(self, engine, **kw): - super(DbmStore, self).__init__(engine, **kw) - self._store = anydbm.open(self._engine, 'c') - try: - self.sync = self._store.sync - except AttributeError: - pass - - -__all__ = ['DbmStore'] diff --git a/lib/shove/store/durusdb.py b/lib/shove/store/durusdb.py deleted file mode 100644 index 8e27670e..00000000 --- a/lib/shove/store/durusdb.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Durus object database frontend. - -shove's psuedo-URL for Durus stores follows the form: - -durus:// - - -Where the path is a URL path to a durus FileStorage database. Alternatively, a -native pathname to a durus database can be passed as the 'engine' parameter. -''' - -try: - from durus.connection import Connection - from durus.file_storage import FileStorage -except ImportError: - raise ImportError('Requires Durus library') - -from shove.store import SyncStore - - -class DurusStore(SyncStore): - - '''Class for Durus object database frontend.''' - - init = 'durus://' - - def __init__(self, engine, **kw): - super(DurusStore, self).__init__(engine, **kw) - self._db = FileStorage(self._engine) - self._connection = Connection(self._db) - self.sync = self._connection.commit - self._store = self._connection.get_root() - - def close(self): - '''Closes all open storage and connections.''' - self.sync() - self._db.close() - super(DurusStore, self).close() - - -__all__ = ['DurusStore'] diff --git a/lib/shove/store/file.py b/lib/shove/store/file.py deleted file mode 100644 index e66e9c4f..00000000 --- a/lib/shove/store/file.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Filesystem-based object store - -shove's psuedo-URL for filesystem-based stores follows the form: - -file:// - -Where the path is a URL path to a directory on a local filesystem. -Alternatively, a native pathname to the directory can be passed as the 'engine' -argument. -''' - -from shove import BaseStore, FileBase - - -class FileStore(FileBase, BaseStore): - - '''File-based store.''' - - def __init__(self, engine, **kw): - super(FileStore, self).__init__(engine, **kw) - - -__all__ = ['FileStore'] diff --git a/lib/shove/store/ftp.py b/lib/shove/store/ftp.py deleted file mode 100644 index c2d4aec6..00000000 --- a/lib/shove/store/ftp.py +++ /dev/null @@ -1,88 +0,0 @@ -# -*- coding: utf-8 -*- -''' -FTP-accessed stores - -shove's URL for FTP accessed stores follows the standard form for FTP URLs -defined in RFC-1738: - -ftp://:@:/ -''' - -import urlparse -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO -from ftplib import FTP, error_perm - -from shove import BaseStore - - -class FtpStore(BaseStore): - - def __init__(self, engine, **kw): - super(FtpStore, self).__init__(engine, **kw) - user = kw.get('user', 'anonymous') - password = kw.get('password', '') - spliturl = urlparse.urlsplit(engine) - # Set URL, path, and strip 'ftp://' off - base, path = spliturl[1], spliturl[2] + '/' - if '@' in base: - auth, base = base.split('@') - user, password = auth.split(':') - self._store = FTP(base, user, password) - # Change to remote path if it exits - try: - self._store.cwd(path) - except error_perm: - self._makedir(path) - self._base, self._user, self._password = base, user, password - self._updated, self ._keys = True, None - - def __getitem__(self, key): - try: - local = StringIO() - # Download item - self._store.retrbinary('RETR %s' % key, local.write) - self._updated = False - return self.loads(local.getvalue()) - except: - raise KeyError(key) - - def __setitem__(self, key, value): - local = StringIO(self.dumps(value)) - self._store.storbinary('STOR %s' % key, local) - self._updated = True - - def __delitem__(self, key): - try: - self._store.delete(key) - self._updated = True - except: - raise KeyError(key) - - def _makedir(self, path): - '''Makes remote paths on an FTP server.''' - paths = list(reversed([i for i in path.split('/') if i != ''])) - while paths: - tpath = paths.pop() - self._store.mkd(tpath) - self._store.cwd(tpath) - - def keys(self): - '''Returns a list of keys in a store.''' - if self._updated or self._keys is None: - rlist, nlist = list(), list() - # Remote directory listing - self._store.retrlines('LIST -a', rlist.append) - for rlisting in rlist: - # Split remote file based on whitespace - rfile = rlisting.split() - # Append tuple of remote item type & name - if rfile[-1] not in ('.', '..') and rfile[0].startswith('-'): - nlist.append(rfile[-1]) - self._keys = nlist - return self._keys - - -__all__ = ['FtpStore'] diff --git a/lib/shove/store/hdf5.py b/lib/shove/store/hdf5.py deleted file mode 100644 index a9b618e5..00000000 --- a/lib/shove/store/hdf5.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -''' -HDF5 Database Store. - -shove's psuedo-URL for HDF5 stores follows the form: - -hdf5:/// - -Where is a URL path to a HDF5 database. Alternatively, the native -pathname to a HDF5 database can be passed as the 'engine' parameter. - is the name of the database. -''' - -try: - import h5py -except ImportError: - raise ImportError('This store requires h5py library') - -from shove.store import ClientStore - - -class HDF5Store(ClientStore): - - '''LevelDB based store''' - - init = 'hdf5://' - - def __init__(self, engine, **kw): - super(HDF5Store, self).__init__(engine, **kw) - engine, group = self._engine.rsplit('/') - self._store = h5py.File(engine).require_group(group).attrs - - -__all__ = ['HDF5Store'] diff --git a/lib/shove/store/leveldbstore.py b/lib/shove/store/leveldbstore.py deleted file mode 100644 index ca73a494..00000000 --- a/lib/shove/store/leveldbstore.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -''' -LevelDB Database Store. - -shove's psuedo-URL for LevelDB stores follows the form: - -leveldb:// - -Where is a URL path to a LevelDB database. Alternatively, the native -pathname to a LevelDB database can be passed as the 'engine' parameter. -''' - -try: - import leveldb -except ImportError: - raise ImportError('This store requires py-leveldb library') - -from shove.store import ClientStore - - -class LevelDBStore(ClientStore): - - '''LevelDB based store''' - - init = 'leveldb://' - - def __init__(self, engine, **kw): - super(LevelDBStore, self).__init__(engine, **kw) - self._store = leveldb.LevelDB(self._engine) - - def __getitem__(self, key): - item = self.loads(self._store.Get(key)) - if item is not None: - return item - raise KeyError(key) - - def __setitem__(self, key, value): - self._store.Put(key, self.dumps(value)) - - def __delitem__(self, key): - self._store.Delete(key) - - def keys(self): - return list(k for k in self._store.RangeIter(include_value=False)) - - -__all__ = ['LevelDBStore'] diff --git a/lib/shove/store/memory.py b/lib/shove/store/memory.py deleted file mode 100644 index 525ae69e..00000000 --- a/lib/shove/store/memory.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Thread-safe in-memory store. - -The shove psuedo-URL for a memory store is: - -memory:// -''' - -import copy -import threading - -from shove import synchronized -from shove.store.simple import SimpleStore - - -class MemoryStore(SimpleStore): - - '''Thread-safe in-memory store.''' - - def __init__(self, engine, **kw): - super(MemoryStore, self).__init__(engine, **kw) - self._lock = threading.Condition() - - @synchronized - def __getitem__(self, key): - return copy.deepcopy(super(MemoryStore, self).__getitem__(key)) - - @synchronized - def __setitem__(self, key, value): - super(MemoryStore, self).__setitem__(key, value) - - @synchronized - def __delitem__(self, key): - super(MemoryStore, self).__delitem__(key) - - -__all__ = ['MemoryStore'] diff --git a/lib/shove/store/redisdb.py b/lib/shove/store/redisdb.py deleted file mode 100644 index 67fa2ebd..00000000 --- a/lib/shove/store/redisdb.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Redis-based object store - -The shove psuedo-URL for a redis-based store is: - -redis://:/ -''' - -import urlparse - -try: - import redis -except ImportError: - raise ImportError('This store requires the redis library') - -from shove.store import ClientStore - - -class RedisStore(ClientStore): - - '''Redis based store''' - - init = 'redis://' - - def __init__(self, engine, **kw): - super(RedisStore, self).__init__(engine, **kw) - spliturl = urlparse.urlsplit(engine) - host, port = spliturl[1].split(':') - db = spliturl[2].replace('/', '') - self._store = redis.Redis(host, int(port), db) - - def __contains__(self, key): - return self._store.exists(key) - - def clear(self): - self._store.flushdb() - - def keys(self): - return self._store.keys() - - def setdefault(self, key, default=None): - return self._store.getset(key, default) - - def update(self, other=None, **kw): - args = kw if other is not None else other - self._store.mset(args) - - -__all__ = ['RedisStore'] diff --git a/lib/shove/store/s3.py b/lib/shove/store/s3.py deleted file mode 100644 index dbf12f21..00000000 --- a/lib/shove/store/s3.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -''' -S3-accessed stores - -shove's psuedo-URL for stores found on Amazon.com's S3 web service follows this -form: - -s3://:@ - - is the Access Key issued by Amazon - is the Secret Access Key issued by Amazon - is the name of the bucket accessed through the S3 service -''' - -try: - from boto.s3.connection import S3Connection - from boto.s3.key import Key -except ImportError: - raise ImportError('Requires boto library') - -from shove import BaseStore - - -class S3Store(BaseStore): - - def __init__(self, engine=None, **kw): - super(S3Store, self).__init__(engine, **kw) - # key = Access Key, secret=Secret Access Key, bucket=bucket name - key, secret, bucket = kw.get('key'), kw.get('secret'), kw.get('bucket') - if engine is not None: - auth, bucket = engine.split('://')[1].split('@') - key, secret = auth.split(':') - # kw 'secure' = (True or False, use HTTPS) - self._conn = S3Connection(key, secret, kw.get('secure', False)) - buckets = self._conn.get_all_buckets() - # Use bucket if it exists - for b in buckets: - if b.name == bucket: - self._store = b - break - # Create bucket if it doesn't exist - else: - self._store = self._conn.create_bucket(bucket) - # Set bucket permission ('private', 'public-read', - # 'public-read-write', 'authenticated-read' - self._store.set_acl(kw.get('acl', 'private')) - # Updated flag used for avoiding network calls - self._updated, self._keys = True, None - - def __getitem__(self, key): - rkey = self._store.lookup(key) - if rkey is None: - raise KeyError(key) - # Fetch string - value = self.loads(rkey.get_contents_as_string()) - # Flag that the store has not been updated - self._updated = False - return value - - def __setitem__(self, key, value): - rkey = Key(self._store) - rkey.key = key - rkey.set_contents_from_string(self.dumps(value)) - # Flag that the store has been updated - self._updated = True - - def __delitem__(self, key): - try: - self._store.delete_key(key) - # Flag that the store has been updated - self._updated = True - except: - raise KeyError(key) - - def keys(self): - '''Returns a list of keys in the store.''' - return list(i[0] for i in self.items()) - - def items(self): - '''Returns a list of items from the store.''' - if self._updated or self._keys is None: - self._keys = self._store.get_all_keys() - return list((str(k.key), k) for k in self._keys) - - def iteritems(self): - '''Lazily returns items from the store.''' - for k in self.items(): - yield (k.key, k) - - -__all__ = ['S3Store'] diff --git a/lib/shove/store/simple.py b/lib/shove/store/simple.py deleted file mode 100644 index 8f7ebb33..00000000 --- a/lib/shove/store/simple.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Single-process in-memory store. - -The shove psuedo-URL for a simple store is: - -simple:// -''' - -from shove import BaseStore, SimpleBase - - -class SimpleStore(SimpleBase, BaseStore): - - '''Single-process in-memory store.''' - - def __init__(self, engine, **kw): - super(SimpleStore, self).__init__(engine, **kw) - - -__all__ = ['SimpleStore'] diff --git a/lib/shove/store/svn.py b/lib/shove/store/svn.py deleted file mode 100644 index 5bb8c33e..00000000 --- a/lib/shove/store/svn.py +++ /dev/null @@ -1,110 +0,0 @@ -# -*- coding: utf-8 -*- -''' -subversion managed store. - -The shove psuedo-URL used for a subversion store that is password protected is: - -svn::?url= - -or for non-password protected repositories: - -svn://?url= - - is the local repository copy - is the URL of the subversion repository -''' - -import os -import urllib -import threading - -try: - import pysvn -except ImportError: - raise ImportError('Requires Python Subversion library') - -from shove import BaseStore, synchronized - - -class SvnStore(BaseStore): - - '''Class for subversion store.''' - - def __init__(self, engine=None, **kw): - super(SvnStore, self).__init__(engine, **kw) - # Get path, url from keywords if used - path, url = kw.get('path'), kw.get('url') - # Get username. password from keywords if used - user, password = kw.get('user'), kw.get('password') - # Process psuedo URL if used - if engine is not None: - path, query = engine.split('n://')[1].split('?') - url = query.split('=')[1] - # Check for username, password - if '@' in path: - auth, path = path.split('@') - user, password = auth.split(':') - path = urllib.url2pathname(path) - # Create subversion client - self._client = pysvn.Client() - # Assign username, password - if user is not None: - self._client.set_username(user) - if password is not None: - self._client.set_password(password) - # Verify that store exists in repository - try: - self._client.info2(url) - # Create store in repository if it doesn't exist - except pysvn.ClientError: - self._client.mkdir(url, 'Adding directory') - # Verify that local copy exists - try: - if self._client.info(path) is None: - self._client.checkout(url, path) - # Check it out if it doesn't exist - except pysvn.ClientError: - self._client.checkout(url, path) - self._path, self._url = path, url - # Lock - self._lock = threading.Condition() - - @synchronized - def __getitem__(self, key): - try: - return self.loads(self._client.cat(self._key_to_file(key))) - except: - raise KeyError(key) - - @synchronized - def __setitem__(self, key, value): - fname = self._key_to_file(key) - # Write value to file - open(fname, 'wb').write(self.dumps(value)) - # Add to repository - if key not in self: - self._client.add(fname) - self._client.checkin([fname], 'Adding %s' % fname) - - @synchronized - def __delitem__(self, key): - try: - fname = self._key_to_file(key) - self._client.remove(fname) - # Remove deleted value from repository - self._client.checkin([fname], 'Removing %s' % fname) - except: - raise KeyError(key) - - def _key_to_file(self, key): - '''Gives the filesystem path for a key.''' - return os.path.join(self._path, urllib.quote_plus(key)) - - @synchronized - def keys(self): - '''Returns a list of keys in the subversion repository.''' - return list(str(i.name.split('/')[-1]) for i - in self._client.ls(self._path)) - - -__all__ = ['SvnStore'] diff --git a/lib/shove/store/zodb.py b/lib/shove/store/zodb.py deleted file mode 100644 index 43768dde..00000000 --- a/lib/shove/store/zodb.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Zope Object Database store frontend. - -shove's psuedo-URL for ZODB stores follows the form: - -zodb: - - -Where the path is a URL path to a ZODB FileStorage database. Alternatively, a -native pathname to a ZODB database can be passed as the 'engine' argument. -''' - -try: - import transaction - from ZODB import FileStorage, DB -except ImportError: - raise ImportError('Requires ZODB library') - -from shove.store import SyncStore - - -class ZodbStore(SyncStore): - - '''ZODB store front end.''' - - init = 'zodb://' - - def __init__(self, engine, **kw): - super(ZodbStore, self).__init__(engine, **kw) - # Handle psuedo-URL - self._storage = FileStorage.FileStorage(self._engine) - self._db = DB(self._storage) - self._connection = self._db.open() - self._store = self._connection.root() - # Keeps DB in synch through commits of transactions - self.sync = transaction.commit - - def close(self): - '''Closes all open storage and connections.''' - self.sync() - super(ZodbStore, self).close() - self._connection.close() - self._db.close() - self._storage.close() - - -__all__ = ['ZodbStore'] diff --git a/lib/shove/tests/__init__.py b/lib/shove/tests/__init__.py deleted file mode 100644 index 40a96afc..00000000 --- a/lib/shove/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/lib/shove/tests/test_bsddb_store.py b/lib/shove/tests/test_bsddb_store.py deleted file mode 100644 index 3de7896e..00000000 --- a/lib/shove/tests/test_bsddb_store.py +++ /dev/null @@ -1,133 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestBsdbStore(unittest.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('bsddb://test.db', compress=True) - - def tearDown(self): - import os - self.store.close() - os.remove('test.db') - - def test__getitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_cassandra_store.py b/lib/shove/tests/test_cassandra_store.py deleted file mode 100644 index a5c60f6a..00000000 --- a/lib/shove/tests/test_cassandra_store.py +++ /dev/null @@ -1,137 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestCassandraStore(unittest.TestCase): - - def setUp(self): - from shove import Shove - from pycassa.system_manager import SystemManager - system_manager = SystemManager('localhost:9160') - try: - system_manager.create_column_family('Foo', 'shove') - except: - pass - self.store = Shove('cassandra://localhost:9160/Foo/shove') - - def tearDown(self): - self.store.clear() - self.store.close() - from pycassa.system_manager import SystemManager - system_manager = SystemManager('localhost:9160') - system_manager.drop_column_family('Foo', 'shove') - - def test__getitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - -# def test_clear(self): -# self.store['max'] = 3 -# self.store['min'] = 6 -# self.store['pow'] = 7 -# self.store.clear() -# self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - -# def test_popitem(self): -# self.store['max'] = 3 -# self.store['min'] = 6 -# self.store['pow'] = 7 -# item = self.store.popitem() -# self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 -# self.store['pow'] = 7 - self.store.setdefault('pow', 8) - self.assertEqual(self.store.setdefault('pow', 8), 8) - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_db_cache.py b/lib/shove/tests/test_db_cache.py deleted file mode 100644 index 9dd27a06..00000000 --- a/lib/shove/tests/test_db_cache.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestDbCache(unittest.TestCase): - - initstring = 'sqlite:///' - - def setUp(self): - from shove.cache.db import DbCache - self.cache = DbCache(self.initstring) - - def tearDown(self): - self.cache = None - - def test_getitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_setitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_delitem(self): - self.cache['test'] = 'test' - del self.cache['test'] - self.assertEqual('test' in self.cache, False) - - def test_get(self): - self.assertEqual(self.cache.get('min'), None) - - def test_timeout(self): - import time - from shove.cache.db import DbCache - cache = DbCache(self.initstring, timeout=1) - cache['test'] = 'test' - time.sleep(2) - - def tmp(): - cache['test'] - self.assertRaises(KeyError, tmp) - - def test_cull(self): - from shove.cache.db import DbCache - cache = DbCache(self.initstring, max_entries=1) - cache['test'] = 'test' - cache['test2'] = 'test' - cache['test2'] = 'test' - self.assertEquals(len(cache), 1) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_db_store.py b/lib/shove/tests/test_db_store.py deleted file mode 100644 index 1d9ad616..00000000 --- a/lib/shove/tests/test_db_store.py +++ /dev/null @@ -1,131 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestDbStore(unittest.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('sqlite://', compress=True) - - def tearDown(self): - self.store.close() - - def test__getitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_dbm_store.py b/lib/shove/tests/test_dbm_store.py deleted file mode 100644 index e64ac9e7..00000000 --- a/lib/shove/tests/test_dbm_store.py +++ /dev/null @@ -1,136 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestDbmStore(unittest.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('dbm://test.dbm', compress=True) - - def tearDown(self): - import os - self.store.close() - try: - os.remove('test.dbm.db') - except OSError: - pass - - def test__getitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.setdefault('how', 8) - self.assertEqual(self.store['how'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_durus_store.py b/lib/shove/tests/test_durus_store.py deleted file mode 100644 index 006fcc41..00000000 --- a/lib/shove/tests/test_durus_store.py +++ /dev/null @@ -1,133 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestDurusStore(unittest.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('durus://test.durus', compress=True) - - def tearDown(self): - import os - self.store.close() - os.remove('test.durus') - - def test__getitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_file_cache.py b/lib/shove/tests/test_file_cache.py deleted file mode 100644 index b288ce82..00000000 --- a/lib/shove/tests/test_file_cache.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestFileCache(unittest.TestCase): - - initstring = 'file://test' - - def setUp(self): - from shove.cache.file import FileCache - self.cache = FileCache(self.initstring) - - def tearDown(self): - import os - self.cache = None - for x in os.listdir('test'): - os.remove(os.path.join('test', x)) - os.rmdir('test') - - def test_getitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_setitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_delitem(self): - self.cache['test'] = 'test' - del self.cache['test'] - self.assertEqual('test' in self.cache, False) - - def test_get(self): - self.assertEqual(self.cache.get('min'), None) - - def test_timeout(self): - import time - from shove.cache.file import FileCache - cache = FileCache(self.initstring, timeout=1) - cache['test'] = 'test' - time.sleep(2) - - def tmp(): - cache['test'] - self.assertRaises(KeyError, tmp) - - def test_cull(self): - from shove.cache.file import FileCache - cache = FileCache(self.initstring, max_entries=1) - cache['test'] = 'test' - cache['test2'] = 'test' - num = len(cache) - self.assertEquals(num, 1) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_file_store.py b/lib/shove/tests/test_file_store.py deleted file mode 100644 index 35643ced..00000000 --- a/lib/shove/tests/test_file_store.py +++ /dev/null @@ -1,140 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestFileStore(unittest.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('file://test', compress=True) - - def tearDown(self): - import os - self.store.close() - for x in os.listdir('test'): - os.remove(os.path.join('test', x)) - os.rmdir('test') - - def test__getitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.store.sync() - tstore.sync() - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_ftp_store.py b/lib/shove/tests/test_ftp_store.py deleted file mode 100644 index 17679a2c..00000000 --- a/lib/shove/tests/test_ftp_store.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestFtpStore(unittest.TestCase): - - ftpstring = 'put ftp string here' - - def setUp(self): - from shove import Shove - self.store = Shove(self.ftpstring, compress=True) - - def tearDown(self): - self.store.clear() - self.store.close() - - def test__getitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.store.sync() - tstore.sync() - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store.sync() - self.assertEqual(len(self.store), 2) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store.sync() - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - item = self.store.popitem() - self.store.sync() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.store.sync() - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.sync() - self.store.update(tstore) - self.store.sync() - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = self.store.keys() - self.assertEqual('min' in slist, True) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_hdf5_store.py b/lib/shove/tests/test_hdf5_store.py deleted file mode 100644 index b1342ecf..00000000 --- a/lib/shove/tests/test_hdf5_store.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest2 - - -class TestHDF5Store(unittest2.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('hdf5://test.hdf5/test') - - def tearDown(self): - import os - self.store.close() - try: - os.remove('test.hdf5') - except OSError: - pass - - def test__getitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.setdefault('bow', 8) - self.assertEqual(self.store['bow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - -if __name__ == '__main__': - unittest2.main() diff --git a/lib/shove/tests/test_leveldb_store.py b/lib/shove/tests/test_leveldb_store.py deleted file mode 100644 index b3a3d177..00000000 --- a/lib/shove/tests/test_leveldb_store.py +++ /dev/null @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest2 - - -class TestLevelDBStore(unittest2.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('leveldb://test', compress=True) - - def tearDown(self): - import shutil - shutil.rmtree('test') - - def test__getitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.setdefault('bow', 8) - self.assertEqual(self.store['bow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - - -if __name__ == '__main__': - unittest2.main() diff --git a/lib/shove/tests/test_memcached_cache.py b/lib/shove/tests/test_memcached_cache.py deleted file mode 100644 index 98f0b96d..00000000 --- a/lib/shove/tests/test_memcached_cache.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestMemcached(unittest.TestCase): - - initstring = 'memcache://localhost:11211' - - def setUp(self): - from shove.cache.memcached import MemCached - self.cache = MemCached(self.initstring) - - def tearDown(self): - self.cache = None - - def test_getitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_setitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_delitem(self): - self.cache['test'] = 'test' - del self.cache['test'] - self.assertEqual('test' in self.cache, False) - - def test_get(self): - self.assertEqual(self.cache.get('min'), None) - - def test_timeout(self): - import time - from shove.cache.memcached import MemCached - cache = MemCached(self.initstring, timeout=1) - cache['test'] = 'test' - time.sleep(1) - - def tmp(): - cache['test'] - self.assertRaises(KeyError, tmp) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_memory_cache.py b/lib/shove/tests/test_memory_cache.py deleted file mode 100644 index 87749cdb..00000000 --- a/lib/shove/tests/test_memory_cache.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestMemoryCache(unittest.TestCase): - - initstring = 'memory://' - - def setUp(self): - from shove.cache.memory import MemoryCache - self.cache = MemoryCache(self.initstring) - - def tearDown(self): - self.cache = None - - def test_getitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_setitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_delitem(self): - self.cache['test'] = 'test' - del self.cache['test'] - self.assertEqual('test' in self.cache, False) - - def test_get(self): - self.assertEqual(self.cache.get('min'), None) - - def test_timeout(self): - import time - from shove.cache.memory import MemoryCache - cache = MemoryCache(self.initstring, timeout=1) - cache['test'] = 'test' - time.sleep(1) - - def tmp(): - cache['test'] - self.assertRaises(KeyError, tmp) - - def test_cull(self): - from shove.cache.memory import MemoryCache - cache = MemoryCache(self.initstring, max_entries=1) - cache['test'] = 'test' - cache['test2'] = 'test' - cache['test2'] = 'test' - self.assertEquals(len(cache), 1) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_memory_store.py b/lib/shove/tests/test_memory_store.py deleted file mode 100644 index 12e505dd..00000000 --- a/lib/shove/tests/test_memory_store.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestMemoryStore(unittest.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('memory://', compress=True) - - def tearDown(self): - self.store.close() - - def test__getitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.store.sync() - tstore.sync() - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_redis_cache.py b/lib/shove/tests/test_redis_cache.py deleted file mode 100644 index c8e9b8db..00000000 --- a/lib/shove/tests/test_redis_cache.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestRedisCache(unittest.TestCase): - - initstring = 'redis://localhost:6379/0' - - def setUp(self): - from shove.cache.redisdb import RedisCache - self.cache = RedisCache(self.initstring) - - def tearDown(self): - self.cache = None - - def test_getitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_setitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_delitem(self): - self.cache['test'] = 'test' - del self.cache['test'] - self.assertEqual('test' in self.cache, False) - - def test_get(self): - self.assertEqual(self.cache.get('min'), None) - - def test_timeout(self): - import time - from shove.cache.redisdb import RedisCache - cache = RedisCache(self.initstring, timeout=1) - cache['test'] = 'test' - time.sleep(3) - def tmp(): #@IgnorePep8 - return cache['test'] - self.assertRaises(KeyError, tmp) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_redis_store.py b/lib/shove/tests/test_redis_store.py deleted file mode 100644 index 06b1e0e9..00000000 --- a/lib/shove/tests/test_redis_store.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestRedisStore(unittest.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('redis://localhost:6379/0') - - def tearDown(self): - self.store.clear() - self.store.close() - - def test__getitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.assertEqual(self.store.setdefault('pow', 8), 8) - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_s3_store.py b/lib/shove/tests/test_s3_store.py deleted file mode 100644 index 8a0f08d7..00000000 --- a/lib/shove/tests/test_s3_store.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestS3Store(unittest.TestCase): - - s3string = 's3 test string here' - - def setUp(self): - from shove import Shove - self.store = Shove(self.s3string, compress=True) - - def tearDown(self): - self.store.clear() - self.store.close() - - def test__getitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.store.sync() - tstore.sync() - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store.sync() - self.assertEqual(len(self.store), 2) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store.sync() - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - item = self.store.popitem() - self.store.sync() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.store.sync() - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.sync() - self.store.update(tstore) - self.store.sync() - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = self.store.keys() - self.assertEqual('min' in slist, True) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_simple_cache.py b/lib/shove/tests/test_simple_cache.py deleted file mode 100644 index 8cd1830c..00000000 --- a/lib/shove/tests/test_simple_cache.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestSimpleCache(unittest.TestCase): - - initstring = 'simple://' - - def setUp(self): - from shove.cache.simple import SimpleCache - self.cache = SimpleCache(self.initstring) - - def tearDown(self): - self.cache = None - - def test_getitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_setitem(self): - self.cache['test'] = 'test' - self.assertEqual(self.cache['test'], 'test') - - def test_delitem(self): - self.cache['test'] = 'test' - del self.cache['test'] - self.assertEqual('test' in self.cache, False) - - def test_get(self): - self.assertEqual(self.cache.get('min'), None) - - def test_timeout(self): - import time - from shove.cache.simple import SimpleCache - cache = SimpleCache(self.initstring, timeout=1) - cache['test'] = 'test' - time.sleep(1) - - def tmp(): - cache['test'] - self.assertRaises(KeyError, tmp) - - def test_cull(self): - from shove.cache.simple import SimpleCache - cache = SimpleCache(self.initstring, max_entries=1) - cache['test'] = 'test' - cache['test2'] = 'test' - cache['test2'] = 'test' - self.assertEquals(len(cache), 1) - - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_simple_store.py b/lib/shove/tests/test_simple_store.py deleted file mode 100644 index d2431ec5..00000000 --- a/lib/shove/tests/test_simple_store.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestSimpleStore(unittest.TestCase): - - def setUp(self): - from shove import Shove - self.store = Shove('simple://', compress=True) - - def tearDown(self): - self.store.close() - - def test__getitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.store.sync() - tstore.sync() - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_svn_store.py b/lib/shove/tests/test_svn_store.py deleted file mode 100644 index b3103816..00000000 --- a/lib/shove/tests/test_svn_store.py +++ /dev/null @@ -1,148 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestSvnStore(unittest.TestCase): - - svnstring = 'SVN test string here' - - def setUp(self): - from shove import Shove - self.store = Shove(self.svnstring, compress=True) - - def tearDown(self): - self.store.clear() - self.store.close() - - def test__getitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.store.sync() - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.store.sync() - tstore.sync() - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store.sync() - self.assertEqual(len(self.store), 2) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store.sync() - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - item = self.store.popitem() - self.store.sync() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.store.sync() - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.sync() - self.store.update(tstore) - self.store.sync() - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.sync() - slist = self.store.keys() - self.assertEqual('min' in slist, True) - -if __name__ == '__main__': - unittest.main() diff --git a/lib/shove/tests/test_zodb_store.py b/lib/shove/tests/test_zodb_store.py deleted file mode 100644 index 9d979fea..00000000 --- a/lib/shove/tests/test_zodb_store.py +++ /dev/null @@ -1,138 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class TestZodbStore(unittest.TestCase): - - init = 'zodb://test.db' - - def setUp(self): - from shove import Shove - self.store = Shove(self.init, compress=True) - - def tearDown(self): - self.store.close() - import os - os.remove('test.db') - os.remove('test.db.index') - os.remove('test.db.tmp') - os.remove('test.db.lock') - - def test__getitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__setitem__(self): - self.store['max'] = 3 - self.assertEqual(self.store['max'], 3) - - def test__delitem__(self): - self.store['max'] = 3 - del self.store['max'] - self.assertEqual('max' in self.store, False) - - def test_get(self): - self.store['max'] = 3 - self.assertEqual(self.store.get('min'), None) - - def test__cmp__(self): - from shove import Shove - tstore = Shove() - self.store['max'] = 3 - tstore['max'] = 3 - self.assertEqual(self.store, tstore) - - def test__len__(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.assertEqual(len(self.store), 2) - - def test_close(self): - self.store.close() - self.assertEqual(self.store, None) - - def test_clear(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - self.store.clear() - self.assertEqual(len(self.store), 0) - - def test_items(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.items()) - self.assertEqual(('min', 6) in slist, True) - - def test_iteritems(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iteritems()) - self.assertEqual(('min', 6) in slist, True) - - def test_iterkeys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.iterkeys()) - self.assertEqual('min' in slist, True) - - def test_itervalues(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = list(self.store.itervalues()) - self.assertEqual(6 in slist, True) - - def test_pop(self): - self.store['max'] = 3 - self.store['min'] = 6 - item = self.store.pop('min') - self.assertEqual(item, 6) - - def test_popitem(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - item = self.store.popitem() - self.assertEqual(len(item) + len(self.store), 4) - - def test_setdefault(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['powl'] = 7 - self.store.setdefault('pow', 8) - self.assertEqual(self.store['pow'], 8) - - def test_update(self): - from shove import Shove - tstore = Shove() - tstore['max'] = 3 - tstore['min'] = 6 - tstore['pow'] = 7 - self.store['max'] = 2 - self.store['min'] = 3 - self.store['pow'] = 7 - self.store.update(tstore) - self.assertEqual(self.store['min'], 6) - - def test_values(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.values() - self.assertEqual(6 in slist, True) - - def test_keys(self): - self.store['max'] = 3 - self.store['min'] = 6 - self.store['pow'] = 7 - slist = self.store.keys() - self.assertEqual('min' in slist, True) - - -if __name__ == '__main__': - unittest.main() diff --git a/sickbeard/rssfeeds.py b/sickbeard/rssfeeds.py index 6bbc975c..0bdf81e2 100644 --- a/sickbeard/rssfeeds.py +++ b/sickbeard/rssfeeds.py @@ -1,56 +1,31 @@ -from __future__ import with_statement - -import os import urllib import urlparse import re +from feedparser import feedparser import sickbeard from sickbeard import logger -from sickbeard import encodingKludge as ek -from contextlib import closing from sickbeard.exceptions import ex -from lib.feedcache import cache -from shove import Shove +def getFeed(url, post_data=None, request_headers=None): + parsed = list(urlparse.urlparse(url)) + parsed[2] = re.sub("/{2,}", "/", parsed[2]) # replace two or more / with one -class RSSFeeds: - def __init__(self, db_name): - self.db_name = ek.ek(os.path.join, sickbeard.CACHE_DIR, 'rss', db_name + '.db') - if not os.path.exists(os.path.dirname(self.db_name)): - sickbeard.helpers.makeDir(os.path.dirname(self.db_name)) + if post_data: + url += urllib.urlencode(post_data) - def clearCache(self, age=None): - try: - with closing(Shove('sqlite:///' + self.db_name, compress=True)) as fs: - fc = cache.Cache(fs) - fc.purge(age) - except Exception as e: - logger.log(u"RSS error clearing cache: " + ex(e), logger.DEBUG) + try: + feed = feedparser.parse(url, False, False, request_headers) - def getFeed(self, url, post_data=None, request_headers=None): - parsed = list(urlparse.urlparse(url)) - parsed[2] = re.sub("/{2,}", "/", parsed[2]) # replace two or more / with one + if feed: + if 'entries' in feed: + return feed + elif 'error' in feed.feed: + err_code = feed.feed['error']['code'] + err_desc = feed.feed['error']['description'] + logger.log(u'RSS ERROR:[%s] CODE:[%s]' % (err_desc, err_code), logger.DEBUG) + else: + logger.log(u'RSS error loading url: ' + url, logger.DEBUG) - if post_data: - url += urllib.urlencode(post_data) - - try: - with closing(Shove('sqlite:///' + self.db_name, compress=True)) as fs: - fc = cache.Cache(fs) - feed = fc.fetch(url, False, False, request_headers) - - if feed: - if 'entries' in feed: - return feed - elif 'error' in feed.feed: - err_code = feed.feed['error']['code'] - err_desc = feed.feed['error']['description'] - - logger.log( - u"RSS ERROR:[%s] CODE:[%s]" % (err_desc, err_code), logger.DEBUG) - else: - logger.log(u"RSS error loading url: " + url, logger.DEBUG) - - except Exception as e: - logger.log(u"RSS error: " + ex(e), logger.DEBUG) \ No newline at end of file + except Exception as e: + logger.log(u'RSS error: ' + ex(e), logger.DEBUG) \ No newline at end of file diff --git a/sickbeard/tvcache.py b/sickbeard/tvcache.py index af9b4591..25083d87 100644 --- a/sickbeard/tvcache.py +++ b/sickbeard/tvcache.py @@ -30,7 +30,7 @@ from sickbeard import helpers, show_name_helpers from sickbeard.exceptions import MultipleShowObjectsException from sickbeard.exceptions import AuthException from name_parser.parser import NameParser, InvalidNameException, InvalidShowException -from sickbeard.rssfeeds import RSSFeeds +from sickbeard.rssfeeds import getFeed from sickbeard import clients import itertools @@ -104,7 +104,7 @@ class TVCache(): return [] def getRSSFeed(self, url, post_data=None, request_headers=None): - return RSSFeeds(self.providerID).getFeed(url, post_data, request_headers) + return getFeed(url, post_data, request_headers) def _translateTitle(self, title): return u'' + title.replace(' ', '.')