monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2025-03-15 09:07:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Change provider KAT remove dead url.

Browse source 
Change provider KAT to use mediaExtensions from common instead of private list.
Change provider KAT provider PEP8 and code convention cleanup.
This commit is contained in:
JackDandy 2015-06-09 00:35:09 +01:00
parent e15c75984e
commit 6c0638c262
2 changed files with 83 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGES.md
View file

@ -20,6 +20,9 @@
* Add new tld variants to provider TPB * Add new tld variants to provider TPB
* Add test for authenticity to provider TPB to notify of 3rd party block * Add test for authenticity to provider TPB to notify of 3rd party block
* Change provider Womble's use SSL * Change provider Womble's use SSL
* Change provider KAT remove dead url
* Change provider KAT to use mediaExtensions from common instead of private list
* Change provider KAT provider PEP8 and code convention cleanup
* Change refactor and code simplification for torrent providers * Change refactor and code simplification for torrent providers
* Add Rarbg torrent provider * Add Rarbg torrent provider
* Fix getManualSearchStatus: object has no attribute 'segment' * Fix getManualSearchStatus: object has no attribute 'segment'

164
sickbeard/providers/kat.py
View file

@ -1,6 +1,4 @@
# coding=utf-8 # coding=utf-8
# Author: Mr_Orange <mr_orange@hotmail.it>
# URL: http://code.google.com/p/sickbeard/
# #
# This file is part of SickGear. # This file is part of SickGear.
# #
@ -28,9 +26,9 @@ import urlparse
import sickbeard import sickbeard
import generic import generic
from sickbeard.common import Quality from sickbeard.common import Quality, mediaExtensions
from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
from sickbeard import logger,tvcache,helpers,db,classes from sickbeard import logger, tvcache, helpers, db, classes
from sickbeard.show_name_helpers import allPossibleShowNames, sanitizeSceneName from sickbeard.show_name_helpers import allPossibleShowNames, sanitizeSceneName
from sickbeard.bs4_parser import BS4Parser from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode from lib.unidecode import unidecode
@ -39,20 +37,25 @@ from lib.unidecode import unidecode
class KATProvider(generic.TorrentProvider): class KATProvider(generic.TorrentProvider):
def __init__(self): def __init__(self):
generic.TorrentProvider.__init__(self, 'KickAssTorrents', True, False) generic.TorrentProvider.__init__(self, 'KickAssTorrents', True, False)
self.confirmed = False self.confirmed = False
self.ratio = None self.ratio = None
self.minseed = None self.minseed = None
self.minleech = None self.minleech = None
self.urls = ['https://kat.ph/', 'http://katproxy.com/']
self.url = self.urls[0]
self.cache = KATCache(self) self.cache = KATCache(self)
self.urls = ['https://kat.ph/', 'http://katproxy.com/', 'http://www.kickmirror.com/']
self.url = 'https://kat.ph/'
def getQuality(self, item, anime=False): def getQuality(self, item, anime=False):
quality = Quality.sceneQuality(item[0], anime) quality = Quality.sceneQuality(item[0], anime)
return quality return quality
def _reverseQuality(self, quality): @staticmethod
def _reverse_quality(quality):
quality_string = '' quality_string = ''
@ -80,68 +83,63 @@ class KATProvider(generic.TorrentProvider):
def _find_season_quality(self, title, torrent_link, ep_number): def _find_season_quality(self, title, torrent_link, ep_number):
""" Return the modified title of a Season Torrent with the quality found inspecting torrent file list """ """ Return the modified title of a Season Torrent with the quality found inspecting torrent file list """
mediaExtensions = ['avi', 'mkv', 'wmv', 'divx',
'vob', 'dvr-ms', 'wtv', 'ts'
'ogv', 'rar', 'zip', 'mp4']
quality = Quality.UNKNOWN quality = Quality.UNKNOWN
fileName = None file_name = None
data = self.getURL(torrent_link) data = self.getURL(torrent_link)
if not data: if not data:
return None return None
try: try:
with BS4Parser(data, features=["html5lib", "permissive"]) as soup: with BS4Parser(data, features=['html5lib', 'permissive']) as soup:
file_table = soup.find('table', attrs={'class': 'torrentFileList'}) file_table = soup.find('table', attrs={'class': 'torrentFileList'})
if not file_table: if not file_table:
return None return None
files = [x.text for x in file_table.find_all('td', attrs={'class': 'torFileName'})] files = [x.text for x in file_table.find_all('td', attrs={'class': 'torFileName'})]
videoFiles = filter(lambda x: x.rpartition(".")[2].lower() in mediaExtensions, files) video_files = filter(lambda i: i.rpartition('.')[2].lower() in mediaExtensions, files)
#Filtering SingleEpisode/MultiSeason Torrent # Filtering SingleEpisode/MultiSeason Torrent
if len(videoFiles) < ep_number or len(videoFiles) > float(ep_number * 1.1): if len(video_files) < ep_number or len(video_files) > float(ep_number * 1.1):
logger.log(u"Result " + title + " have " + str( logger.log(u'Result %s lists %s episodes with %s episodes retrieved in torrent'
ep_number) + " episode and episodes retrived in torrent are " + str(len(videoFiles)), logger.DEBUG) % (title, ep_number, len(video_files)), logger.DEBUG)
logger.log( logger.log(u'Result %s seem to be a single episode or multi-season torrent, skipping result...'
u"Result " + title + " Seem to be a Single Episode or MultiSeason torrent, skipping result...", % title, logger.DEBUG)
logger.DEBUG)
return None return None
if Quality.sceneQuality(title) != Quality.UNKNOWN: if Quality.UNKNOWN != Quality.sceneQuality(title):
return title return title
for fileName in videoFiles: for file_name in video_files:
quality = Quality.sceneQuality(os.path.basename(fileName)) quality = Quality.sceneQuality(os.path.basename(file_name))
if quality != Quality.UNKNOWN: break if Quality.UNKNOWN != quality:
break
if fileName is not None and quality == Quality.UNKNOWN: if None is not file_name and Quality.UNKNOWN == quality:
quality = Quality.assumeQuality(os.path.basename(fileName)) quality = Quality.assumeQuality(os.path.basename(file_name))
if quality == Quality.UNKNOWN: if Quality.UNKNOWN == quality:
logger.log(u"Unable to obtain a Season Quality for " + title, logger.DEBUG) logger.log(u'Unable to obtain a Season Quality for ' + title, logger.DEBUG)
return None return None
try: try:
myParser = NameParser(showObj=self.show) my_parser = NameParser(showObj=self.show)
parse_result = myParser.parse(fileName) parse_result = my_parser.parse(file_name)
except (InvalidNameException, InvalidShowException): except (InvalidNameException, InvalidShowException):
return None return None
logger.log(u"Season quality for " + title + " is " + Quality.qualityStrings[quality], logger.DEBUG) logger.log(u'Season quality for %s is %s' % (title, Quality.qualityStrings[quality]), logger.DEBUG)
if parse_result.series_name and parse_result.season_number: if parse_result.series_name and parse_result.season_number:
title = parse_result.series_name + ' S%02d' % int( title = parse_result.series_name + ' S%02d %s' % (int(parse_result.season_number),
parse_result.season_number) + ' ' + self._reverseQuality(quality) self._reverse_quality(quality))
return title return title
except Exception as e: except Exception:
logger.log(u"Failed parsing " + self.name + " Traceback: " + traceback.format_exc(), logger.ERROR) logger.log(u'Failed to quality parse ' + self.name + ' Traceback: ' + traceback.format_exc(), logger.ERROR)
def _get_season_search_strings(self, ep_obj): def _get_season_search_strings(self, ep_obj):
search_string = {'Season': []} search_string = {'Season': []}
@ -153,11 +151,11 @@ class KATProvider(generic.TorrentProvider):
ep_string = show_name + ' Season ' + str(ep_obj.airdate).split('-')[0] ep_string = show_name + ' Season ' + str(ep_obj.airdate).split('-')[0]
search_string['Season'].append(ep_string) search_string['Season'].append(ep_string)
elif ep_obj.show.anime: elif ep_obj.show.anime:
ep_string = show_name + ' ' + "%02d" % ep_obj.scene_absolute_number ep_string = show_name + ' ' + '%02d' % ep_obj.scene_absolute_number
search_string['Season'].append(ep_string) search_string['Season'].append(ep_string)
else: else:
ep_string = show_name + ' S%02d' % int(ep_obj.scene_season) + ' -S%02d' % int( ep_string = show_name + ' S%02d' % int(ep_obj.scene_season) + ' -S%02d' % int(
ep_obj.scene_season) + 'E' + ' category:tv' #1) showName SXX -SXXE ep_obj.scene_season) + 'E' + ' category:tv' # 1) showName SXX -SXXE
search_string['Season'].append(ep_string) search_string['Season'].append(ep_string)
ep_string = show_name + ' Season ' + str( ep_string = show_name + ' Season ' + str(
ep_obj.scene_season) + ' -Ep*' + ' category:tv' # 2) showName Season X ep_obj.scene_season) + ' -Ep*' + ' category:tv' # 2) showName Season X
@ -171,31 +169,30 @@ class KATProvider(generic.TorrentProvider):
if self.show.air_by_date: if self.show.air_by_date:
for show_name in set(allPossibleShowNames(self.show)): for show_name in set(allPossibleShowNames(self.show)):
ep_string = sanitizeSceneName(show_name) + ' ' + \ ep_string = sanitizeSceneName(show_name) + ' ' + \
str(ep_obj.airdate).replace('-', ' ') str(ep_obj.airdate).replace('-', ' ')
search_string['Episode'].append(ep_string) search_string['Episode'].append(ep_string)
elif self.show.sports: elif self.show.sports:
for show_name in set(allPossibleShowNames(self.show)): for show_name in set(allPossibleShowNames(self.show)):
ep_string = sanitizeSceneName(show_name) + ' ' + \ ep_string = sanitizeSceneName(show_name) + ' ' + \
str(ep_obj.airdate).replace('-', '|') + '|' + \ str(ep_obj.airdate).replace('-', '|') + '|' + \
ep_obj.airdate.strftime('%b') ep_obj.airdate.strftime('%b')
search_string['Episode'].append(ep_string) search_string['Episode'].append(ep_string)
elif self.show.anime: elif self.show.anime:
for show_name in set(allPossibleShowNames(self.show)): for show_name in set(allPossibleShowNames(self.show)):
ep_string = sanitizeSceneName(show_name) + ' ' + \ ep_string = sanitizeSceneName(show_name) + ' ' + \
"%02i" % int(ep_obj.scene_absolute_number) '%02i' % int(ep_obj.scene_absolute_number)
search_string['Episode'].append(ep_string) search_string['Episode'].append(ep_string)
else: else:
for show_name in set(allPossibleShowNames(self.show)): for show_name in set(allPossibleShowNames(self.show)):
ep_string = sanitizeSceneName(show_name) + ' ' + \ ep_string = sanitizeSceneName(show_name) + ' ' + \
sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season, sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season,
'episodenumber': ep_obj.scene_episode} + '|' + \ 'episodenumber': ep_obj.scene_episode} + '|' + \
sickbeard.config.naming_ep_type[0] % {'seasonnumber': ep_obj.scene_season, sickbeard.config.naming_ep_type[0] % {'seasonnumber': ep_obj.scene_season,
'episodenumber': ep_obj.scene_episode} + ' %s category:tv' % add_string 'episodenumber': ep_obj.scene_episode} + ' %s category:tv' % add_string
search_string['Episode'].append(re.sub('\s+', ' ', ep_string)) search_string['Episode'].append(re.sub('\s+', ' ', ep_string))
return [search_string] return [search_string]
def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0): def _doSearch(self, search_params, search_mode='eponly', epcount=0, age=0):
results = [] results = []
@ -205,14 +202,15 @@ class KATProvider(generic.TorrentProvider):
for search_string in search_params[mode]: for search_string in search_params[mode]:
for url in self.urls: for url in self.urls:
if mode != 'RSS': search_url = url
searchURL = url + 'usearch/%s/?field=seeders&sorder=desc' % (urllib.quote(unidecode(search_string))) if 'RSS' == mode:
logger.log(u"Search string: " + searchURL, logger.DEBUG) search_url += 'tv/?field=time_add&sorder=desc'
logger.log(u'KAT cache update URL: ' + search_url, logger.DEBUG)
else: else:
searchURL = url + 'tv/?field=time_add&sorder=desc' search_url += 'usearch/%s/?field=seeders&sorder=desc' % (urllib.quote(unidecode(search_string)))
logger.log(u"KAT cache update URL: " + searchURL, logger.DEBUG) logger.log(u'Search string: ' + search_url, logger.DEBUG)
html = self.getURL(searchURL) html = self.getURL(search_url)
if html: if html:
self.url = url self.url = url
break break
@ -221,13 +219,13 @@ class KATProvider(generic.TorrentProvider):
continue continue
try: try:
with BS4Parser(html, features=["html5lib", "permissive"]) as soup: with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
torrent_table = soup.find('table', attrs={'class': 'data'}) torrent_table = soup.find('table', attrs={'class': 'data'})
torrent_rows = torrent_table.find_all('tr') if torrent_table else [] torrent_rows = torrent_table.find_all('tr') if torrent_table else []
#Continue only if one Release is found # Continue only if one Release is found
if len(torrent_rows) < 2: if len(torrent_rows) < 2:
logger.log(u"The data returned from " + self.name + " does not contain any torrents", logger.log(u'The data returned from ' + self.name + ' does not contain any torrents',
logger.WARNING) logger.WARNING)
continue continue
@ -235,43 +233,43 @@ class KATProvider(generic.TorrentProvider):
try: try:
link = urlparse.urljoin(self.url, link = urlparse.urljoin(self.url,
(tr.find('div', {'class': 'torrentname'}).find_all('a')[1])['href']) (tr.find('div', {'class': 'torrentname'}).find_all('a')[1])['href'])
id = tr.get('id')[-7:] tid = tr.get('id')[-7:]
title = (tr.find('div', {'class': 'torrentname'}).find_all('a')[1]).text \ title = (tr.find('div', {'class': 'torrentname'}).find_all('a')[1]).text \
or (tr.find('div', {'class': 'torrentname'}).find_all('a')[2]).text or (tr.find('div', {'class': 'torrentname'}).find_all('a')[2]).text
url = tr.find('a', 'imagnet')['href'] url = tr.find('a', 'imagnet')['href']
verified = True if tr.find('a', 'iverify') else False verified = True if tr.find('a', 'iverify') else False
trusted = True if tr.find('img', {'alt': 'verified'}) else False # trusted = True if tr.find('img', {'alt': 'verified'}) else False
seeders = int(tr.find_all('td')[-2].text) seeders = int(tr.find_all('td')[-2].text)
leechers = int(tr.find_all('td')[-1].text) leechers = int(tr.find_all('td')[-1].text)
except (AttributeError, TypeError): except (AttributeError, TypeError):
continue continue
if mode != 'RSS' and (seeders < self.minseed or leechers < self.minleech): if 'RSS' != mode and (seeders < self.minseed or leechers < self.minleech):
continue continue
if self.confirmed and not verified: if self.confirmed and not verified:
logger.log( logger.log(
u"KAT Provider found result " + title + " but that doesn't seem like a verified result so I'm ignoring it", u'KAT Provider found result ' + title + ' but that doesn\'t seem like a verified result so I\'m ignoring it',
logger.DEBUG) logger.DEBUG)
continue continue
#Check number video files = episode in season and find the real Quality for full season torrent analyzing files in torrent # Check number video files = episode in season and find the real Quality for full season torrent analyzing files in torrent
if mode == 'Season' and search_mode == 'sponly': if 'Season' == mode and 'sponly' == search_mode:
ep_number = int(epcount / len(set(allPossibleShowNames(self.show)))) ep_number = int(epcount / len(set(allPossibleShowNames(self.show))))
title = self._find_season_quality(title, link, ep_number) title = self._find_season_quality(title, link, ep_number)
if not title or not url: if not title or not url:
continue continue
item = title, url, id, seeders, leechers item = title, url, tid, seeders, leechers
items[mode].append(item) items[mode].append(item)
except Exception as e: except Exception:
logger.log(u"Failed to parsing " + self.name + " Traceback: " + traceback.format_exc(), logger.log(u'Failed to parse ' + self.name + ' Traceback: ' + traceback.format_exc(),
logger.ERROR) logger.ERROR)
#For each search mode sort all the items by seeders # For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[3], reverse=True) items[mode].sort(key=lambda tup: tup[3], reverse=True)
results += items[mode] results += items[mode]
@ -280,23 +278,22 @@ class KATProvider(generic.TorrentProvider):
def _get_title_and_url(self, item): def _get_title_and_url(self, item):
title, url, id, seeders, leechers = item title, url, tid, seeders, leechers = item
if title: if title:
title = u'' + title title = u'' + title.replace(' ', '.')
title = title.replace(' ', '.')
if url: if url:
url = url.replace('&amp;', '&') url = url.replace('&amp;', '&')
return (title, url) return title, url
def findPropers(self, search_date=datetime.datetime.today()): def findPropers(self, search_date=datetime.datetime.today()):
results = [] results = []
myDB = db.DBConnection() my_db = db.DBConnection()
sqlResults = myDB.select( sql_results = my_db.select(
'SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.airdate, s.indexer FROM tv_episodes AS e' + 'SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.airdate, s.indexer FROM tv_episodes AS e' +
' INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id)' + ' INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id)' +
' WHERE e.airdate >= ' + str(search_date.toordinal()) + ' WHERE e.airdate >= ' + str(search_date.toordinal()) +
@ -304,17 +301,17 @@ class KATProvider(generic.TorrentProvider):
' OR (e.status IN (' + ','.join([str(x) for x in Quality.SNATCHED]) + ')))' ' OR (e.status IN (' + ','.join([str(x) for x in Quality.SNATCHED]) + ')))'
) )
if not sqlResults: if not sql_results:
return [] return []
for sqlshow in sqlResults: for sqlshow in sql_results:
self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow["showid"])) self.show = helpers.findCertainShow(sickbeard.showList, int(sqlshow['showid']))
if self.show: if self.show:
curEp = self.show.getEpisode(int(sqlshow["season"]), int(sqlshow["episode"])) cur_ep = self.show.getEpisode(int(sqlshow['season']), int(sqlshow['episode']))
searchString = self._get_episode_search_strings(curEp, add_string='PROPER|REPACK') search_string = self._get_episode_search_strings(cur_ep, add_string='PROPER|REPACK')
for item in self._doSearch(searchString[0]): for item in self._doSearch(search_string[0]):
title, url = self._get_title_and_url(item) title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, datetime.datetime.today(), self.show)) results.append(classes.Proper(title, url, datetime.datetime.today(), self.show))
@ -325,12 +322,11 @@ class KATProvider(generic.TorrentProvider):
class KATCache(tvcache.TVCache): class KATCache(tvcache.TVCache):
def __init__(self, provider): def __init__(self, this_provider):
tvcache.TVCache.__init__(self, provider) tvcache.TVCache.__init__(self, this_provider)
# only poll ThePirateBay every 10 minutes max self.minTime = 20 # cache update frequency
self.minTime = 20
def _getRSSData(self): def _getRSSData(self):
search_params = {'RSS': ['rss']} search_params = {'RSS': ['rss']}