monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2024-12-01 00:43:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

AniDB now updates its xml lists daily.

Browse source 
Added writeback cache for both rss feeds and name parser.
This commit is contained in:
echel0n 2014-07-14 23:55:52 -07:00
parent fefcfa0952
commit 4a061d4dd2
6 changed files with 71 additions and 85685 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
lib/adba/aniDBAbstracter.py
View file

@ -15,6 +15,8 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with aDBa. If not, see <http://www.gnu.org/licenses/>. # along with aDBa. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
from time import time, sleep from time import time, sleep
import aniDBfileInfo as fileInfo import aniDBfileInfo as fileInfo
import xml.etree.cElementTree as etree import xml.etree.cElementTree as etree
@ -22,11 +24,10 @@ import os, re, string
from aniDBmaper import AniDBMaper from aniDBmaper import AniDBMaper
from aniDBtvDBmaper import TvDBMap from aniDBtvDBmaper import TvDBMap
from aniDBerrors import * from aniDBerrors import *
from aniDBfileInfo import read_anidb_xml, read_tvdb_map_xml
class aniDBabstractObject(object): class aniDBabstractObject(object):
def __init__(self, aniDB, load=False): def __init__(self, aniDB, load=False):
self.laoded = False self.laoded = False
self.set_connection(aniDB) self.set_connection(aniDB)
@ -98,7 +99,7 @@ class aniDBabstractObject(object):
priority - low = 0, medium = 1, high = 2 (unconfirmed) priority - low = 0, medium = 1, high = 2 (unconfirmed)
""" """
if(self.aid): if (self.aid):
self.aniDB.notifyadd(aid=self.aid, type=1, priority=1) self.aniDB.notifyadd(aid=self.aid, type=1, priority=1)
@ -157,18 +158,19 @@ class Anime(aniDBabstractObject):
self.rawData = self.aniDB.groupstatus(aid=self.aid) self.rawData = self.aniDB.groupstatus(aid=self.aid)
self.release_groups = [] self.release_groups = []
for line in self.rawData.datalines: for line in self.rawData.datalines:
self.release_groups.append({"name":unicode(line["name"], "utf-8"), self.release_groups.append({"name": unicode(line["name"], "utf-8"),
"rating":line["rating"], "rating": line["rating"],
"range":line["episode_range"] "range": line["episode_range"]
}) })
return self.release_groups return self.release_groups
#TODO: refactor and use the new functions in anidbFileinfo # TODO: refactor and use the new functions in anidbFileinfo
def _get_aid_from_xml(self, name): def _get_aid_from_xml(self, name):
if not self.allAnimeXML: if not self.allAnimeXML:
self.allAnimeXML = self._read_animetitels_xml() self.allAnimeXML = read_anidb_xml()
regex = re.compile('( \(\d{4}\))|[%s]' % re.escape(string.punctuation)) # remove any punctuation and e.g. ' (2011)' regex = re.compile(
'( \(\d{4}\))|[%s]' % re.escape(string.punctuation)) # remove any punctuation and e.g. ' (2011)'
#regex = re.compile('[%s]' % re.escape(string.punctuation)) # remove any punctuation and e.g. ' (2011)' #regex = re.compile('[%s]' % re.escape(string.punctuation)) # remove any punctuation and e.g. ' (2011)'
name = regex.sub('', name.lower()) name = regex.sub('', name.lower())
lastAid = 0 lastAid = 0
@ -185,7 +187,7 @@ class Anime(aniDBabstractObject):
#TODO: refactor and use the new functions in anidbFileinfo #TODO: refactor and use the new functions in anidbFileinfo
def _get_name_from_xml(self, aid, onlyMain=True): def _get_name_from_xml(self, aid, onlyMain=True):
if not self.allAnimeXML: if not self.allAnimeXML:
self.allAnimeXML = self._read_animetitels_xml() self.allAnimeXML = read_anidb_xml()
for anime in self.allAnimeXML.findall("anime"): for anime in self.allAnimeXML.findall("anime"):
if int(anime.get("aid", False)) == aid: if int(anime.get("aid", False)) == aid:
@ -196,15 +198,6 @@ class Anime(aniDBabstractObject):
return title.text return title.text
return "" return ""
def _read_animetitels_xml(self, path=None):
if not path:
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "animetitles.xml")
f = open(path, "r")
allAnimeXML = etree.ElementTree(file=f)
return allAnimeXML
def _builPreSequal(self): def _builPreSequal(self):
if self.related_aid_list and self.related_aid_type: if self.related_aid_list and self.related_aid_type:
try: try:
@ -220,10 +213,9 @@ class Anime(aniDBabstractObject):
self.__dict__["sequal"] = self.related_aid_list self.__dict__["sequal"] = self.related_aid_list
class Episode(aniDBabstractObject): class Episode(aniDBabstractObject):
def __init__(self, aniDB, number=None, epid=None, filePath=None, fid=None, epno=None, paramsA=None, paramsF=None,
def __init__(self, aniDB, number=None, epid=None, filePath=None, fid=None, epno=None, paramsA=None, paramsF=None, load=False, calculate=False): load=False, calculate=False):
self.maper = AniDBMaper() self.maper = AniDBMaper()
self.epid = epid self.epid = epid
self.filePath = filePath self.filePath = filePath
@ -232,7 +224,6 @@ class Episode(aniDBabstractObject):
if calculate: if calculate:
(self.ed2k, self.size) = self._calculate_file_stuff(self.filePath) (self.ed2k, self.size) = self._calculate_file_stuff(self.filePath)
if not paramsA: if not paramsA:
self.bitCodeA = "C000F0C0" self.bitCodeA = "C000F0C0"
self.paramsA = self.maper.getFileCodesA(self.bitCodeA) self.paramsA = self.maper.getFileCodesA(self.bitCodeA)
@ -254,7 +245,8 @@ class Episode(aniDBabstractObject):
if self.filePath and not (self.ed2k or self.size): if self.filePath and not (self.ed2k or self.size):
(self.ed2k, self.size) = self._calculate_file_stuff(self.filePath) (self.ed2k, self.size) = self._calculate_file_stuff(self.filePath)
self.rawData = self.aniDB.file(fid=self.fid, size=self.size, ed2k=self.ed2k, aid=self.aid, aname=None, gid=None, gname=None, epno=self.epno, fmask=self.bitCodeF, amask=self.bitCodeA) self.rawData = self.aniDB.file(fid=self.fid, size=self.size, ed2k=self.ed2k, aid=self.aid, aname=None, gid=None,
gname=None, epno=self.epno, fmask=self.bitCodeF, amask=self.bitCodeA)
self._fill(self.rawData.datalines[0]) self._fill(self.rawData.datalines[0])
self._build_names() self._build_names()
self.laoded = True self.laoded = True
@ -273,7 +265,7 @@ class Episode(aniDBabstractObject):
try: try:
self.aniDB.mylistadd(size=self.size, ed2k=self.ed2k, state=status) self.aniDB.mylistadd(size=self.size, ed2k=self.ed2k, state=status)
except Exception, e : except Exception, e:
self.log(u"exception msg: " + str(e)) self.log(u"exception msg: " + str(e))
else: else:
# TODO: add the name or something # TODO: add the name or something

56
lib/adba/aniDBfileInfo.py
View file

@ -16,12 +16,16 @@
# along with aDBa. If not, see <http://www.gnu.org/licenses/>. # along with aDBa. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement from __future__ import with_statement
import hashlib import hashlib
import os import os
import xml.etree.cElementTree as etree import xml.etree.cElementTree as etree
import time
# http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/ # http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/
import requests
def get_file_hash(filePath): def get_file_hash(filePath):
""" Returns the ed2k hash of a given file.""" """ Returns the ed2k hash of a given file."""
if not filePath: if not filePath:
@ -31,8 +35,10 @@ def get_file_hash(filePath):
def gen(f): def gen(f):
while True: while True:
x = f.read(9728000) x = f.read(9728000)
if x: yield x if x:
else: return yield x
else:
return
def md4_hash(data): def md4_hash(data):
m = md4() m = md4()
@ -44,7 +50,8 @@ def get_file_hash(filePath):
hashes = [md4_hash(data).digest() for data in a] hashes = [md4_hash(data).digest() for data in a]
if len(hashes) == 1: if len(hashes) == 1:
return hashes[0].encode("hex") return hashes[0].encode("hex")
else: return md4_hash(reduce(lambda a,d: a + d, hashes, "")).hexdigest() else:
return md4_hash(reduce(lambda a, d: a + d, hashes, "")).hexdigest()
def get_file_size(path): def get_file_size(path):
@ -52,16 +59,43 @@ def get_file_size(path):
return size return size
def get_anime_titles_xml(path):
daily_dump = requests.get("http://raw.githubusercontent.com/ScudLee/anime-lists/master/animetitles.xml")
with open(path, "wb") as f:
f.write(daily_dump.content)
def read_anidb_xml(filePath):
def get_anime_list_xml(path):
daily_dump = requests.get("http://raw.githubusercontent.com/ScudLee/anime-lists/master/anime-list.xml")
with open(path, "wb") as f:
f.write(daily_dump.content)
def read_anidb_xml(filePath=None):
if not filePath: if not filePath:
filePath = os.path.join(os.path.dirname(os.path.abspath( __file__ )), "animetitles.xml") filePath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "animetitles.xml")
if not os.path.isfile(filePath):
get_anime_titles_xml(filePath)
else:
mtime = os.path.getmtime(filePath)
if time.time() > mtime + 24 * 60 * 60:
get_anime_titles_xml(filePath)
return read_xml_into_etree(filePath) return read_xml_into_etree(filePath)
def read_tvdb_map_xml(filePath): def read_tvdb_map_xml(filePath=None):
if not filePath: if not filePath:
filePath = os.path.join(os.path.dirname(os.path.abspath( __file__ )), "anime-list.xml") filePath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "anime-list.xml")
if not os.path.isfile(filePath):
get_anime_list_xml(filePath)
else:
mtime = os.path.getmtime(filePath)
if time.time() > mtime + 24 * 60 * 60:
get_anime_list_xml(filePath)
return read_xml_into_etree(filePath) return read_xml_into_etree(filePath)
@ -69,7 +103,5 @@ def read_xml_into_etree(filePath):
if not filePath: if not filePath:
return None return None
f = open(filePath,"r") with open(filePath, "r") as f:
xmlASetree = etree.ElementTree(file = f) return etree.ElementTree(file=f)
return xmlASetree

23766
lib/adba/anime-list.xml
View file

File diff suppressed because it is too large Load diff

61872
lib/adba/animetitles.xml
View file

File diff suppressed because it is too large Load diff

4
sickbeard/name_parser/parser.py
View file

@ -591,7 +591,7 @@ class NameParserCache:
name = name.encode('utf-8', 'ignore') name = name.encode('utf-8', 'ignore')
try : try :
with closing(shelve.open(self.db_name)) as npc: with closing(shelve.open(self.db_name, writeback=True)) as npc:
npc[str(name)] = parse_result npc[str(name)] = parse_result
while len(npc.items()) > self.npc_cache_size: while len(npc.items()) > self.npc_cache_size:
@ -604,7 +604,7 @@ class NameParserCache:
name = name.encode('utf-8', 'ignore') name = name.encode('utf-8', 'ignore')
try: try:
with closing(shelve.open(ek.ek(os.path.join, sickbeard.CACHE_DIR, 'name_parser_cache'))) as npc: with closing(shelve.open(self.db_name, writeback=True)) as npc:
parse_result = npc.get(str(name), None) parse_result = npc.get(str(name), None)
except Exception as e: except Exception as e:
logger.log(u"NameParser cache error: " + ex(e), logger.ERROR) logger.log(u"NameParser cache error: " + ex(e), logger.ERROR)

4
sickbeard/rssfeeds.py
View file

@ -20,7 +20,7 @@ class RSSFeeds:
def clearCache(self, age=None): def clearCache(self, age=None):
try: try:
with closing(shelve.open(ek.ek(os.path.join, sickbeard.CACHE_DIR, self.db_name))) as fs: with closing(shelve.open(self.db_name, writeback=True)) as fs:
fc = cache.Cache(fs) fc = cache.Cache(fs)
fc.purge(age) fc.purge(age)
except Exception as e: except Exception as e:
@ -35,7 +35,7 @@ class RSSFeeds:
url += urllib.urlencode(post_data) url += urllib.urlencode(post_data)
try: try:
with closing(shelve.open(self.db_name)) as fs: with closing(shelve.open(self.db_name, writeback=True)) as fs:
fc = cache.Cache(fs) fc = cache.Cache(fs)
feed = fc.fetch(url, False, False, request_headers) feed = fc.fetch(url, False, False, request_headers)
except Exception as e: except Exception as e: