monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2025-01-05 17:43:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #549 from JackDandy/feature/ChangeProviders

Browse source 
Change refactor and tidy providers.
This commit is contained in:
JackDandy 2015-12-02 01:39:10 +00:00
commit 5f5b3ba5ee
83 changed files with 3197 additions and 3557 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
CHANGES.md
View file

@ -42,8 +42,12 @@
* Change default de-referrer url to blank
* Change javascript urls in templates to allow proper caching
* Change downloads to prevent cache misfiring with "Result is not a valid torrent file"
* Add BitMeTV torrent provider
* Add Torrenting provider
* Add FunFile torrent provider
* Add TVChaosUK torrent provider
* Add HD-Space torrent provider
* Add Shazbat torrent provider
* Remove unnecessary call to indexers during nameparsing
* Change disable ToTV due to non-deletable yet reported hacker BTC inbox scam and also little to no new content listings
* Fix Episode View KeyError: 'state-title' failure for shows without a runtime
@ -53,8 +57,7 @@
* Fix add existing shows from folders that contain a plus char
* Fix post process issue where items in history were processed out of turn
* Change increase frequency of updating show data
* Remove FreshOnTV (TvT) torrent provider
* Remove Strike torrent provider
* Remove Animenzb provider
* Change increase the scope and number of non release group text that is identified and removed
* Add a general config setting to allow adding incomplete show data
* Change to throttle connection rate on thread initiation for adba library
@ -74,12 +77,21 @@
* Add IMDb Popular to Add Show page
* Add version to anime renaming pattern
* Add Code Climate configuration files
* Change to move init-scripts to single folder
* Change move init-scripts to single folder
* Change sickbeard variables to sickgear variables in init-scripts
* Change improve the use of multiple plex servers
* Change to move JS code out of home template and into dedicated file
* Change to remove branch from window title
* Change to move JS code out of inc_top template and into dedicated file
* Change move JS code out of home template and into dedicated file
* Change remove branch from window title
* Change move JS code out of inc_top template and into dedicated file
* Change cleanup torrent providers
* Change utilise tvdbid for searching usenet providers
* Add setting to provider BTN to Reject Blu-ray M2TS releases
* Remove jsonrpclib library
* Change consolidate global and per show ignore and require words functions
* Change "Require word" title and notes on Config Search page to properly describe its functional logic
* Add regular expression capability to ignore and require words by starting wordlist with "regex:"
* Add list shows with custom ignore and require words under the global counterparts on the Search Settings page
* Fix failure to search for more than one selected wanted episode
* Add notice for users with Python 2.7.8 or below to update to latest Python
[develop changelog]
@ -87,6 +99,8 @@
* Add ability to parse command line output from unix unrar version 4 and below
* Fix show search box on non-poster show list views
* Fix removal of non-release groups such that anime qualities are not trimmed from name
* Change readd Strike torrent provider
* Change readd FreshOnTV (TvT) torrent provider
### 0.10.0 (2015-08-06 11:05:00 UTC)

6
SickBeard.py
View file

@ -340,7 +340,7 @@ class SickGear(object):
logger.ERROR)
if sickbeard.LAUNCH_BROWSER and not self.runAsDaemon:
logger.log(u'Launching browser and exiting', logger.ERROR)
sickbeard.launchBrowser(self.startPort)
sickbeard.launch_browser(self.startPort)
os._exit(1)
# Check if we need to perform a restore first
@ -377,7 +377,7 @@ class SickGear(object):
# Launch browser
if sickbeard.LAUNCH_BROWSER and not (self.noLaunch or self.runAsDaemon):
sickbeard.launchBrowser(self.startPort)
sickbeard.launch_browser(self.startPort)
# main loop
while True:
@ -488,7 +488,7 @@ class SickGear(object):
sickbeard.halt()
# save all shows to DB
sickbeard.saveAll()
sickbeard.save_all()
# shutdown web server
if self.webserver:

BIN
gui/slick/images/providers/animenzb.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 726 B

BIN
gui/slick/images/providers/bitmetv.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.3 KiB

BIN
gui/slick/images/providers/freshontv.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 3.5 KiB

BIN
gui/slick/images/providers/hdspace.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 752 B

BIN
gui/slick/images/providers/shazbat.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 557 B

BIN
gui/slick/images/providers/strike.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 417 B

BIN
gui/slick/images/providers/tvchaosuk.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1,002 B

36
gui/slick/interfaces/default/config_providers.tmpl
View file

@ -11,9 +11,9 @@
#import os.path
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_top.tmpl')
#if $varExists('header')
#if $varExists('header')
<h1 class="header">$header</h1>
#else
#else
<h1 class="title">$title</h1>
#end if
@ -281,7 +281,7 @@
<span class="component-desc">
#set $field_name = curNzbProvider.get_id() + '_api_key'
<input type="text" name="$field_name" value="<%= starify(curNzbProvider.api_key) %>" class="form-control input-sm input350" />
#if callable(getattr(curNzbProvider, 'ui_string'))
#if callable(getattr(curNzbProvider, 'ui_string', None))
<div class="clear-left"><p>${curNzbProvider.ui_string($field_name)}</p></div>
#end if
</span>
@ -347,6 +347,17 @@
##
#for $curTorrentProvider in [$curProvider for $curProvider in $sickbeard.providers.sortedProviderList() if $curProvider.providerType == $GenericProvider.TORRENT]:
<div class="providerDiv" id="${curTorrentProvider.get_id()}Div">
#if callable(getattr(curTorrentProvider, 'ui_string', None))
#set $field_name = curTorrentProvider.get_id() + '_tip'
#set $tip_text = curTorrentProvider.ui_string($field_name)
#if $tip_text
<div class="field-pair">
<span class="component-desc" style="margin:0;width:100%">
<div class="clear-left"><p class="grey-text"><span class="red-text">Important! ${curTorrentProvider.name}</span> $tip_text</p></div>
</span>
</div>
#end if
#end if
#if $hasattr($curTorrentProvider, 'api_key'):
<div class="field-pair">
<label for="${curTorrentProvider.get_id()}_api_key">
@ -364,7 +375,7 @@
<span class="component-desc">
#set $field_name = curTorrentProvider.get_id() + '_digest'
<input type="text" name="$field_name" id="$field_name" value="<%= starify(curTorrentProvider.digest) %>" class="form-control input-sm input350" />
#if callable(getattr(curTorrentProvider, 'ui_string'))
#if callable(getattr(curTorrentProvider, 'ui_string', None))
<div class="clear-left"><p>${curTorrentProvider.ui_string($field_name)}</p></div>
#end if
</span>
@ -396,7 +407,7 @@
<label for="${curTorrentProvider.get_id()}_password">
<span class="component-title">Password:</span>
<span class="component-desc">
<input type="password" name="${curTorrentProvider.get_id()}_password" id="${curTorrentProvider.get_id()}_password" value="#echo '*' * len($curTorrentProvider.password)#" class="form-control input-sm input350" />
<input type="password" name="${curTorrentProvider.get_id()}_password" id="${curTorrentProvider.get_id()}_password" value="#echo $curTorrentProvider.password and '*' * len($curTorrentProvider.password) or ''#" class="form-control input-sm input350" />
</span>
</label>
</div>
@ -419,7 +430,7 @@
<span class="component-desc">
<input type="number" name="${curTorrentProvider.get_id()}_ratio" id="${curTorrentProvider.get_id()}_ratio" value="$curTorrentProvider._seed_ratio" class="form-control input-sm input75" />
<p>this ratio is requested of each item sent to $torrent_method_text[$sickbeard.TORRENT_METHOD]</p>
<div class="clear-left"><p>(set -1 to seed forever, or leave blank for the $torrent_method_text[$sickbeard.TORRENT_METHOD] setting)</p></div>
<div class="clear-left"><p>(#if 'Transmission' in $torrent_method_text[$sickbeard.TORRENT_METHOD]#set -1 to seed forever, or #end if#leave blank for the $torrent_method_text[$sickbeard.TORRENT_METHOD] setting)</p></div>
</span>
</label>
</div>
@ -507,6 +518,17 @@
</label>
</div>
#end if
#if $hasattr($curTorrentProvider, 'reject_m2ts'):
<div class="field-pair">
<label for="${curTorrentProvider.get_id()}_reject_m2ts">
<span class="component-title">Reject Blu-ray M2TS releases</span>
<span class="component-desc">
<input type="checkbox" name="${curTorrentProvider.get_id()}_reject_m2ts" id="${curTorrentProvider.get_id()}_reject_m2ts" <%= html_checked if curTorrentProvider.reject_m2ts else '' %>/>
<p>enable to ignore Blu-ray MPEG-2 Transport Stream container releases</p>
</span>
</label>
</div>
#end if
#if $hasattr($curTorrentProvider, 'enable_recentsearch') and $curTorrentProvider.supportsBacklog:
<div class="field-pair">
<label for="${curTorrentProvider.get_id()}_enable_recentsearch">
@ -725,4 +747,4 @@
//-->
</script>
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')

32
gui/slick/interfaces/default/config_search.tmpl
View file

@ -113,18 +113,42 @@
<label>
<span class="component-title">Ignore result with any word</span>
<span class="component-desc">
<input type="text" name="ignore_words" value="$sickbeard.IGNORE_WORDS" class="form-control input-sm input350">
<input type="text" name="ignore_words" value="$sickbeard.IGNORE_WORDS" class="form-control input-sm input350"><p>(opt: start "regex:")</p>
<p class="clear-left note">ignore search result <em class="grey-text">if its title contains any</em> of these comma seperated words</p>
</span>
<span class="component-title">Shows with custom ignores</span>
<span class="component-desc">
#set $shows = []
#for $show in $using_rls_ignore_words
#set void = $shows.append('<a href="%s/home/editShow?show=%s" style="vertical-align:middle">%s</a>' % ($sbRoot, $show[0], $show[1]))
#end for
#if len($using_rls_ignore_words)
<p style="line-height:1.2em;margin-top:6px">#echo ', '.join($shows)#</p>
#else
<p style="line-height:1.2em;margin-top:7px">...will list here when in use</p>
#end if
</span>
</label>
</div>
<div class="field-pair">
<label>
<span class="component-title">Require at least one word</span>
<span class="component-title">Require all these words</span>
<span class="component-desc">
<input type="text" name="require_words" value="$sickbeard.REQUIRE_WORDS" class="form-control input-sm input350">
<p class="clear-left note">ignore search result <em class="grey-text">unless its title contains one</em> of these comma seperated words</p>
<input type="text" name="require_words" value="$sickbeard.REQUIRE_WORDS" class="form-control input-sm input350"><p>(opt: start "regex:")</p>
<p class="clear-left note">ignore search result <em class="grey-text">unless its title contains all</em> of these comma seperated words</p>
</span>
<span class="component-title">Shows with custom requires</span>
<span class="component-desc">
#set $shows = []
#for $show in $using_rls_require_words
#set void = $shows.append('<a href="%s/home/editShow?show=%s" style="vertical-align:middle">%s</a>' % ($sbRoot, $show[0], $show[1]))
#end for
#if len($using_rls_require_words)
<p style="line-height:1.2em;margin-top:6px">#echo ', '.join($shows)#</p>
#else
<p style="line-height:1.2em;margin-top:7px">...will list here when in use</p>
#end if
</span>
</label>
</div>

14
gui/slick/interfaces/default/editShow.tmpl
View file

@ -125,8 +125,8 @@
<span class="component-title">Ignore result with any word</span>
<span class="component-desc">
<input type="text" name="rls_ignore_words" id="rls_ignore_words" value="$show.rls_ignore_words" class="form-control form-control-inline input-sm input350">
<p>e.g. [word1,word2, ... ,word_n]</p>
<p class="note">ignore search result <em class="grey-text">if its title contains any</em> of these comma seperated words</p>
<p>e.g. [[regex:]word1, word2, ..., word_n, regex_n]</p>
<p class="note">ignore search result <em class="grey-text">if its title contains any</em> of these comma seperated words or regular expressions</p>
</span>
</label>
</div>
@ -136,8 +136,8 @@
<span class="component-title">Require at least one word</span>
<span class="component-desc">
<input type="text" name="rls_require_words" id="rls_require_words" value="$show.rls_require_words" class="form-control form-control-inline input-sm input350">
<p>e.g. [word1,word2, ... ,word_n]</p>
<p class="note">ignore search result <em class="grey-text">unless its title contains one</em> of these comma seperated words</p>
<p>e.g. [[regex:]word1, word2, ..., word_n, regex_n]</p>
<p class="note">ignore search result <em class="grey-text">unless its title contains one</em> of these comma seperated words or regular expressions</p>
</span>
</label>
</div>
@ -276,7 +276,7 @@
var scene_ex = \$('#SceneName').val()
var option = \$('<option>')
all_exceptions = []
\$('#exceptions_list option').each ( function() {
all_exceptions.push( \$(this).val() )
});
@ -295,7 +295,7 @@
\$('#removeSceneName').click(function() {
\$('#exceptions_list option:selected').remove();
\$(this).toggle_SceneException()
});
@ -317,4 +317,4 @@
</script>
</div>
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')

10
gui/slick/interfaces/default/manage_subtitleMissed.tmpl
View file

@ -3,8 +3,8 @@
#from lib import subliminal
#from sickbeard import common
##
#set global $title = 'Episode Overview'
#set global $header = 'Episode Overview'
#set global $title = 'Missing Subtitles'
#set global $header = 'Missing Subtitles'
#set global $sbPath = '..'
#set global $topmenu = 'manage'
##
@ -12,9 +12,9 @@
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_top.tmpl')
<div id="content960">
#if $varExists('header')
#if $varExists('header')
<h1 class="header">$header</h1>
#else
#else
<h1 class="title">$title</h1>
#end if
##
@ -64,4 +64,4 @@
</form>
#end if
</div>
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')

229
lib/jsonrpclib/SimpleJSONRPCServer.py
View file

@ -1,229 +0,0 @@
import jsonrpclib
from jsonrpclib import Fault
from jsonrpclib.jsonrpc import USE_UNIX_SOCKETS
import SimpleXMLRPCServer
import SocketServer
import socket
import logging
import os
import types
import traceback
import sys
try:
import fcntl
except ImportError:
# For Windows
fcntl = None
def get_version(request):
# must be a dict
if 'jsonrpc' in request.keys():
return 2.0
if 'id' in request.keys():
return 1.0
return None
def validate_request(request):
if type(request) is not types.DictType:
fault = Fault(
-32600, 'Request must be {}, not %s.' % type(request)
)
return fault
rpcid = request.get('id', None)
version = get_version(request)
if not version:
fault = Fault(-32600, 'Request %s invalid.' % request, rpcid=rpcid)
return fault
request.setdefault('params', [])
method = request.get('method', None)
params = request.get('params')
param_types = (types.ListType, types.DictType, types.TupleType)
if not method or type(method) not in types.StringTypes or \
type(params) not in param_types:
fault = Fault(
-32600, 'Invalid request parameters or method.', rpcid=rpcid
)
return fault
return True
class SimpleJSONRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher):
def __init__(self, encoding=None):
SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self,
allow_none=True,
encoding=encoding)
def _marshaled_dispatch(self, data, dispatch_method = None):
response = None
try:
request = jsonrpclib.loads(data)
except Exception, e:
fault = Fault(-32700, 'Request %s invalid. (%s)' % (data, e))
response = fault.response()
return response
if not request:
fault = Fault(-32600, 'Request invalid -- no request data.')
return fault.response()
if type(request) is types.ListType:
# This SHOULD be a batch, by spec
responses = []
for req_entry in request:
result = validate_request(req_entry)
if type(result) is Fault:
responses.append(result.response())
continue
resp_entry = self._marshaled_single_dispatch(req_entry)
if resp_entry is not None:
responses.append(resp_entry)
if len(responses) > 0:
response = '[%s]' % ','.join(responses)
else:
response = ''
else:
result = validate_request(request)
if type(result) is Fault:
return result.response()
response = self._marshaled_single_dispatch(request)
return response
def _marshaled_single_dispatch(self, request):
# TODO - Use the multiprocessing and skip the response if
# it is a notification
# Put in support for custom dispatcher here
# (See SimpleXMLRPCServer._marshaled_dispatch)
method = request.get('method')
params = request.get('params')
try:
response = self._dispatch(method, params)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
fault = Fault(-32603, '%s:%s' % (exc_type, exc_value))
return fault.response()
if 'id' not in request.keys() or request['id'] == None:
# It's a notification
return None
try:
response = jsonrpclib.dumps(response,
methodresponse=True,
rpcid=request['id']
)
return response
except:
exc_type, exc_value, exc_tb = sys.exc_info()
fault = Fault(-32603, '%s:%s' % (exc_type, exc_value))
return fault.response()
def _dispatch(self, method, params):
func = None
try:
func = self.funcs[method]
except KeyError:
if self.instance is not None:
if hasattr(self.instance, '_dispatch'):
return self.instance._dispatch(method, params)
else:
try:
func = SimpleXMLRPCServer.resolve_dotted_attribute(
self.instance,
method,
True
)
except AttributeError:
pass
if func is not None:
try:
if type(params) is types.ListType:
response = func(*params)
else:
response = func(**params)
return response
except TypeError:
return Fault(-32602, 'Invalid parameters.')
except:
err_lines = traceback.format_exc().splitlines()
trace_string = '%s | %s' % (err_lines[-3], err_lines[-1])
fault = jsonrpclib.Fault(-32603, 'Server error: %s' %
trace_string)
return fault
else:
return Fault(-32601, 'Method %s not supported.' % method)
class SimpleJSONRPCRequestHandler(
SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
def do_POST(self):
if not self.is_rpc_path_valid():
self.report_404()
return
try:
max_chunk_size = 10*1024*1024
size_remaining = int(self.headers["content-length"])
L = []
while size_remaining:
chunk_size = min(size_remaining, max_chunk_size)
L.append(self.rfile.read(chunk_size))
size_remaining -= len(L[-1])
data = ''.join(L)
response = self.server._marshaled_dispatch(data)
self.send_response(200)
except Exception, e:
self.send_response(500)
err_lines = traceback.format_exc().splitlines()
trace_string = '%s | %s' % (err_lines[-3], err_lines[-1])
fault = jsonrpclib.Fault(-32603, 'Server error: %s' % trace_string)
response = fault.response()
if response == None:
response = ''
self.send_header("Content-type", "application/json-rpc")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
self.wfile.flush()
self.connection.shutdown(1)
class SimpleJSONRPCServer(SocketServer.TCPServer, SimpleJSONRPCDispatcher):
allow_reuse_address = True
def __init__(self, addr, requestHandler=SimpleJSONRPCRequestHandler,
logRequests=True, encoding=None, bind_and_activate=True,
address_family=socket.AF_INET):
self.logRequests = logRequests
SimpleJSONRPCDispatcher.__init__(self, encoding)
# TCPServer.__init__ has an extra parameter on 2.6+, so
# check Python version and decide on how to call it
vi = sys.version_info
self.address_family = address_family
if USE_UNIX_SOCKETS and address_family == socket.AF_UNIX:
# Unix sockets can't be bound if they already exist in the
# filesystem. The convention of e.g. X11 is to unlink
# before binding again.
if os.path.exists(addr):
try:
os.unlink(addr)
except OSError:
logging.warning("Could not unlink socket %s", addr)
# if python 2.5 and lower
if vi[0] < 3 and vi[1] < 6:
SocketServer.TCPServer.__init__(self, addr, requestHandler)
else:
SocketServer.TCPServer.__init__(self, addr, requestHandler,
bind_and_activate)
if fcntl is not None and hasattr(fcntl, 'FD_CLOEXEC'):
flags = fcntl.fcntl(self.fileno(), fcntl.F_GETFD)
flags |= fcntl.FD_CLOEXEC
fcntl.fcntl(self.fileno(), fcntl.F_SETFD, flags)
class CGIJSONRPCRequestHandler(SimpleJSONRPCDispatcher):
def __init__(self, encoding=None):
SimpleJSONRPCDispatcher.__init__(self, encoding)
def handle_jsonrpc(self, request_text):
response = self._marshaled_dispatch(request_text)
print 'Content-Type: application/json-rpc'
print 'Content-Length: %d' % len(response)
print
sys.stdout.write(response)
handle_xmlrpc = handle_jsonrpc

6
lib/jsonrpclib/__init__.py
View file

@ -1,6 +0,0 @@
from jsonrpclib.config import Config
config = Config.instance()
from jsonrpclib.history import History
history = History.instance()
from jsonrpclib.jsonrpc import Server, MultiCall, Fault
from jsonrpclib.jsonrpc import ProtocolError, loads, dumps

38
lib/jsonrpclib/config.py
View file

@ -1,38 +0,0 @@
import sys
class LocalClasses(dict):
def add(self, cls):
self[cls.__name__] = cls
class Config(object):
"""
This is pretty much used exclusively for the 'jsonclass'
functionality... set use_jsonclass to False to turn it off.
You can change serialize_method and ignore_attribute, or use
the local_classes.add(class) to include "local" classes.
"""
use_jsonclass = True
# Change to False to keep __jsonclass__ entries raw.
serialize_method = '_serialize'
# The serialize_method should be a string that references the
# method on a custom class object which is responsible for
# returning a tuple of the constructor arguments and a dict of
# attributes.
ignore_attribute = '_ignore'
# The ignore attribute should be a string that references the
# attribute on a custom class object which holds strings and / or
# references of the attributes the class translator should ignore.
classes = LocalClasses()
# The list of classes to use for jsonclass translation.
version = 2.0
# Version of the JSON-RPC spec to support
user_agent = 'jsonrpclib/0.1 (Python %s)' % \
'.'.join([str(ver) for ver in sys.version_info[0:3]])
# User agent to use for calls.
_instance = None
@classmethod
def instance(cls):
if not cls._instance:
cls._instance = cls()
return cls._instance

40
lib/jsonrpclib/history.py
View file

@ -1,40 +0,0 @@
class History(object):
"""
This holds all the response and request objects for a
session. A server using this should call "clear" after
each request cycle in order to keep it from clogging
memory.
"""
requests = []
responses = []
_instance = None
@classmethod
def instance(cls):
if not cls._instance:
cls._instance = cls()
return cls._instance
def add_response(self, response_obj):
self.responses.append(response_obj)
def add_request(self, request_obj):
self.requests.append(request_obj)
@property
def request(self):
if len(self.requests) == 0:
return None
else:
return self.requests[-1]
@property
def response(self):
if len(self.responses) == 0:
return None
else:
return self.responses[-1]
def clear(self):
del self.requests[:]
del self.responses[:]

152
lib/jsonrpclib/jsonclass.py
View file

@ -1,152 +0,0 @@
import types
import inspect
import re
import traceback
from jsonrpclib import config
iter_types = [
types.DictType,
types.ListType,
types.TupleType
]
string_types = [
types.StringType,
types.UnicodeType
]
numeric_types = [
types.IntType,
types.LongType,
types.FloatType
]
value_types = [
types.BooleanType,
types.NoneType
]
supported_types = iter_types+string_types+numeric_types+value_types
invalid_module_chars = r'[^a-zA-Z0-9\_\.]'
class TranslationError(Exception):
pass
def dump(obj, serialize_method=None, ignore_attribute=None, ignore=[]):
if not serialize_method:
serialize_method = config.serialize_method
if not ignore_attribute:
ignore_attribute = config.ignore_attribute
obj_type = type(obj)
# Parse / return default "types"...
if obj_type in numeric_types+string_types+value_types:
return obj
if obj_type in iter_types:
if obj_type in (types.ListType, types.TupleType):
new_obj = []
for item in obj:
new_obj.append(dump(item, serialize_method,
ignore_attribute, ignore))
if obj_type is types.TupleType:
new_obj = tuple(new_obj)
return new_obj
# It's a dict...
else:
new_obj = {}
for key, value in obj.iteritems():
new_obj[key] = dump(value, serialize_method,
ignore_attribute, ignore)
return new_obj
# It's not a standard type, so it needs __jsonclass__
module_name = inspect.getmodule(obj).__name__
class_name = obj.__class__.__name__
json_class = class_name
if module_name not in ['', '__main__']:
json_class = '%s.%s' % (module_name, json_class)
return_obj = {"__jsonclass__":[json_class,]}
# If a serialization method is defined..
if serialize_method in dir(obj):
# Params can be a dict (keyword) or list (positional)
# Attrs MUST be a dict.
serialize = getattr(obj, serialize_method)
params, attrs = serialize()
return_obj['__jsonclass__'].append(params)
return_obj.update(attrs)
return return_obj
# Otherwise, try to figure it out
# Obviously, we can't assume to know anything about the
# parameters passed to __init__
return_obj['__jsonclass__'].append([])
attrs = {}
ignore_list = getattr(obj, ignore_attribute, [])+ignore
for attr_name, attr_value in obj.__dict__.iteritems():
if type(attr_value) in supported_types and \
attr_name not in ignore_list and \
attr_value not in ignore_list:
attrs[attr_name] = dump(attr_value, serialize_method,
ignore_attribute, ignore)
return_obj.update(attrs)
return return_obj
def load(obj):
if type(obj) in string_types+numeric_types+value_types:
return obj
if type(obj) is types.ListType:
return_list = []
for entry in obj:
return_list.append(load(entry))
return return_list
# Othewise, it's a dict type
if '__jsonclass__' not in obj.keys():
return_dict = {}
for key, value in obj.iteritems():
new_value = load(value)
return_dict[key] = new_value
return return_dict
# It's a dict, and it's a __jsonclass__
orig_module_name = obj['__jsonclass__'][0]
params = obj['__jsonclass__'][1]
if orig_module_name == '':
raise TranslationError('Module name empty.')
json_module_clean = re.sub(invalid_module_chars, '', orig_module_name)
if json_module_clean != orig_module_name:
raise TranslationError('Module name %s has invalid characters.' %
orig_module_name)
json_module_parts = json_module_clean.split('.')
json_class = None
if len(json_module_parts) == 1:
# Local class name -- probably means it won't work
if json_module_parts[0] not in config.classes.keys():
raise TranslationError('Unknown class or module %s.' %
json_module_parts[0])
json_class = config.classes[json_module_parts[0]]
else:
json_class_name = json_module_parts.pop()
json_module_tree = '.'.join(json_module_parts)
try:
temp_module = __import__(json_module_tree)
except ImportError:
raise TranslationError('Could not import %s from module %s.' %
(json_class_name, json_module_tree))
# The returned class is the top-level module, not the one we really
# want. (E.g., if we import a.b.c, we now have a.) Walk through other
# path components to get to b and c.
for i in json_module_parts[1:]:
temp_module = getattr(temp_module, i)
json_class = getattr(temp_module, json_class_name)
# Creating the object...
new_obj = None
if type(params) is types.ListType:
new_obj = json_class(*params)
elif type(params) is types.DictType:
new_obj = json_class(**params)
else:
raise TranslationError('Constructor args must be a dict or list.')
for key, value in obj.iteritems():
if key == '__jsonclass__':
continue
setattr(new_obj, key, value)
return new_obj

561
lib/jsonrpclib/jsonrpc.py
View file

@ -1,561 +0,0 @@
"""
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
============================
JSONRPC Library (jsonrpclib)
============================
This library is a JSON-RPC v.2 (proposed) implementation which
follows the xmlrpclib API for portability between clients. It
uses the same Server / ServerProxy, loads, dumps, etc. syntax,
while providing features not present in XML-RPC like:
* Keyword arguments
* Notifications
* Versioning
* Batches and batch notifications
Eventually, I'll add a SimpleXMLRPCServer compatible library,
and other things to tie the thing off nicely. :)
For a quick-start, just open a console and type the following,
replacing the server address, method, and parameters
appropriately.
>>> import jsonrpclib
>>> server = jsonrpclib.Server('http://localhost:8181')
>>> server.add(5, 6)
11
>>> server._notify.add(5, 6)
>>> batch = jsonrpclib.MultiCall(server)
>>> batch.add(3, 50)
>>> batch.add(2, 3)
>>> batch._notify.add(3, 5)
>>> batch()
[53, 5]
See http://code.google.com/p/jsonrpclib/ for more info.
"""
import types
import sys
from xmlrpclib import Transport as XMLTransport
from xmlrpclib import SafeTransport as XMLSafeTransport
from xmlrpclib import ServerProxy as XMLServerProxy
from xmlrpclib import _Method as XML_Method
import time
import string
import random
# Library includes
import jsonrpclib
from jsonrpclib import config
from jsonrpclib import history
# JSON library importing
cjson = None
json = None
try:
import cjson
except ImportError:
try:
import json
except ImportError:
try:
import simplejson as json
except ImportError:
raise ImportError(
'You must have the cjson, json, or simplejson ' +
'module(s) available.'
)
IDCHARS = string.ascii_lowercase+string.digits
class UnixSocketMissing(Exception):
"""
Just a properly named Exception if Unix Sockets usage is
attempted on a platform that doesn't support them (Windows)
"""
pass
#JSON Abstractions
def jdumps(obj, encoding='utf-8'):
# Do 'serialize' test at some point for other classes
global cjson
if cjson:
return cjson.encode(obj)
else:
return json.dumps(obj, encoding=encoding)
def jloads(json_string):
global cjson
if cjson:
return cjson.decode(json_string)
else:
return json.loads(json_string)
# XMLRPClib re-implementations
class ProtocolError(Exception):
pass
class TransportMixIn(object):
""" Just extends the XMLRPC transport where necessary. """
user_agent = config.user_agent
# for Python 2.7 support
_connection = None
def send_content(self, connection, request_body):
connection.putheader("Content-Type", "application/json-rpc")
connection.putheader("Content-Length", str(len(request_body)))
connection.endheaders()
if request_body:
connection.send(request_body)
def getparser(self):
target = JSONTarget()
return JSONParser(target), target
class JSONParser(object):
def __init__(self, target):
self.target = target
def feed(self, data):
self.target.feed(data)
def close(self):
pass
class JSONTarget(object):
def __init__(self):
self.data = []
def feed(self, data):
self.data.append(data)
def close(self):
return ''.join(self.data)
class Transport(TransportMixIn, XMLTransport):
def __init__(self):
TransportMixIn.__init__(self)
XMLTransport.__init__(self)
class SafeTransport(TransportMixIn, XMLSafeTransport):
def __init__(self):
TransportMixIn.__init__(self)
XMLSafeTransport.__init__(self)
from httplib import HTTP, HTTPConnection
from socket import socket
USE_UNIX_SOCKETS = False
try:
from socket import AF_UNIX, SOCK_STREAM
USE_UNIX_SOCKETS = True
except ImportError:
pass
if (USE_UNIX_SOCKETS):
class UnixHTTPConnection(HTTPConnection):
def connect(self):
self.sock = socket(AF_UNIX, SOCK_STREAM)
self.sock.connect(self.host)
class UnixHTTP(HTTP):
_connection_class = UnixHTTPConnection
class UnixTransport(TransportMixIn, XMLTransport):
def make_connection(self, host):
import httplib
host, extra_headers, x509 = self.get_host_info(host)
return UnixHTTP(host)
class ServerProxy(XMLServerProxy):
"""
Unfortunately, much more of this class has to be copied since
so much of it does the serialization.
"""
def __init__(self, uri, transport=None, encoding=None,
verbose=0, version=None):
import urllib
if not version:
version = config.version
self.__version = version
schema, uri = urllib.splittype(uri)
if schema not in ('http', 'https', 'unix'):
raise IOError('Unsupported JSON-RPC protocol.')
if schema == 'unix':
if not USE_UNIX_SOCKETS:
# Don't like the "generic" Exception...
raise UnixSocketMissing("Unix sockets not available.")
self.__host = uri
self.__handler = '/'
else:
self.__host, self.__handler = urllib.splithost(uri)
if not self.__handler:
# Not sure if this is in the JSON spec?
#self.__handler = '/'
self.__handler == '/'
if transport is None:
if schema == 'unix':
transport = UnixTransport()
elif schema == 'https':
transport = SafeTransport()
else:
transport = Transport()
self.__transport = transport
self.__encoding = encoding
self.__verbose = verbose
def _request(self, methodname, params, rpcid=None):
request = dumps(params, methodname, encoding=self.__encoding,
rpcid=rpcid, version=self.__version)
response = self._run_request(request)
check_for_errors(response)
return response['result']
def _request_notify(self, methodname, params, rpcid=None):
request = dumps(params, methodname, encoding=self.__encoding,
rpcid=rpcid, version=self.__version, notify=True)
response = self._run_request(request, notify=True)
check_for_errors(response)
return
def _run_request(self, request, notify=None):
history.add_request(request)
response = self.__transport.request(
self.__host,
self.__handler,
request,
verbose=self.__verbose
)
# Here, the XMLRPC library translates a single list
# response to the single value -- should we do the
# same, and require a tuple / list to be passed to
# the response object, or expect the Server to be
# outputting the response appropriately?
history.add_response(response)
if not response:
return None
return_obj = loads(response)
return return_obj
def __getattr__(self, name):
# Same as original, just with new _Method reference
return _Method(self._request, name)
@property
def _notify(self):
# Just like __getattr__, but with notify namespace.
return _Notify(self._request_notify)
class _Method(XML_Method):
def __call__(self, *args, **kwargs):
if len(args) > 0 and len(kwargs) > 0:
raise ProtocolError('Cannot use both positional ' +
'and keyword arguments (according to JSON-RPC spec.)')
if len(args) > 0:
return self.__send(self.__name, args)
else:
return self.__send(self.__name, kwargs)
def __getattr__(self, name):
self.__name = '%s.%s' % (self.__name, name)
return self
# The old method returned a new instance, but this seemed wasteful.
# The only thing that changes is the name.
#return _Method(self.__send, "%s.%s" % (self.__name, name))
class _Notify(object):
def __init__(self, request):
self._request = request
def __getattr__(self, name):
return _Method(self._request, name)
# Batch implementation
class MultiCallMethod(object):
def __init__(self, method, notify=False):
self.method = method
self.params = []
self.notify = notify
def __call__(self, *args, **kwargs):
if len(kwargs) > 0 and len(args) > 0:
raise ProtocolError('JSON-RPC does not support both ' +
'positional and keyword arguments.')
if len(kwargs) > 0:
self.params = kwargs
else:
self.params = args
def request(self, encoding=None, rpcid=None):
return dumps(self.params, self.method, version=2.0,
encoding=encoding, rpcid=rpcid, notify=self.notify)
def __repr__(self):
return '%s' % self.request()
def __getattr__(self, method):
new_method = '%s.%s' % (self.method, method)
self.method = new_method
return self
class MultiCallNotify(object):
def __init__(self, multicall):
self.multicall = multicall
def __getattr__(self, name):
new_job = MultiCallMethod(name, notify=True)
self.multicall._job_list.append(new_job)
return new_job
class MultiCallIterator(object):
def __init__(self, results):
self.results = results
def __iter__(self):
for i in range(0, len(self.results)):
yield self[i]
raise StopIteration
def __getitem__(self, i):
item = self.results[i]
check_for_errors(item)
return item['result']
def __len__(self):
return len(self.results)
class MultiCall(object):
def __init__(self, server):
self._server = server
self._job_list = []
def _request(self):
if len(self._job_list) < 1:
# Should we alert? This /is/ pretty obvious.
return
request_body = '[ %s ]' % ','.join([job.request() for
job in self._job_list])
responses = self._server._run_request(request_body)
del self._job_list[:]
if not responses:
responses = []
return MultiCallIterator(responses)
@property
def _notify(self):
return MultiCallNotify(self)
def __getattr__(self, name):
new_job = MultiCallMethod(name)
self._job_list.append(new_job)
return new_job
__call__ = _request
# These lines conform to xmlrpclib's "compatibility" line.
# Not really sure if we should include these, but oh well.
Server = ServerProxy
class Fault(object):
# JSON-RPC error class
def __init__(self, code=-32000, message='Server error', rpcid=None):
self.faultCode = code
self.faultString = message
self.rpcid = rpcid
def error(self):
return {'code':self.faultCode, 'message':self.faultString}
def response(self, rpcid=None, version=None):
if not version:
version = config.version
if rpcid:
self.rpcid = rpcid
return dumps(
self, methodresponse=True, rpcid=self.rpcid, version=version
)
def __repr__(self):
return '<Fault %s: %s>' % (self.faultCode, self.faultString)
def random_id(length=8):
return_id = ''
for i in range(length):
return_id += random.choice(IDCHARS)
return return_id
class Payload(dict):
def __init__(self, rpcid=None, version=None):
if not version:
version = config.version
self.id = rpcid
self.version = float(version)
def request(self, method, params=[]):
if type(method) not in types.StringTypes:
raise ValueError('Method name must be a string.')
if not self.id:
self.id = random_id()
request = { 'id':self.id, 'method':method }
if params:
request['params'] = params
if self.version >= 2:
request['jsonrpc'] = str(self.version)
return request
def notify(self, method, params=[]):
request = self.request(method, params)
if self.version >= 2:
del request['id']
else:
request['id'] = None
return request
def response(self, result=None):
response = {'result':result, 'id':self.id}
if self.version >= 2:
response['jsonrpc'] = str(self.version)
else:
response['error'] = None
return response
def error(self, code=-32000, message='Server error.'):
error = self.response()
if self.version >= 2:
del error['result']
else:
error['result'] = None
error['error'] = {'code':code, 'message':message}
return error
def dumps(params=[], methodname=None, methodresponse=None,
encoding=None, rpcid=None, version=None, notify=None):
"""
This differs from the Python implementation in that it implements
the rpcid argument since the 2.0 spec requires it for responses.
"""
if not version:
version = config.version
valid_params = (types.TupleType, types.ListType, types.DictType)
if methodname in types.StringTypes and \
type(params) not in valid_params and \
not isinstance(params, Fault):
"""
If a method, and params are not in a listish or a Fault,
error out.
"""
raise TypeError('Params must be a dict, list, tuple or Fault ' +
'instance.')
# Begin parsing object
payload = Payload(rpcid=rpcid, version=version)
if not encoding:
encoding = 'utf-8'
if type(params) is Fault:
response = payload.error(params.faultCode, params.faultString)
return jdumps(response, encoding=encoding)
if type(methodname) not in types.StringTypes and methodresponse != True:
raise ValueError('Method name must be a string, or methodresponse '+
'must be set to True.')
if config.use_jsonclass == True:
from jsonrpclib import jsonclass
params = jsonclass.dump(params)
if methodresponse is True:
if rpcid is None:
raise ValueError('A method response must have an rpcid.')
response = payload.response(params)
return jdumps(response, encoding=encoding)
request = None
if notify == True:
request = payload.notify(methodname, params)
else:
request = payload.request(methodname, params)
return jdumps(request, encoding=encoding)
def loads(data):
"""
This differs from the Python implementation, in that it returns
the request structure in Dict format instead of the method, params.
It will return a list in the case of a batch request / response.
"""
if data == '':
# notification
return None
result = jloads(data)
# if the above raises an error, the implementing server code
# should return something like the following:
# { 'jsonrpc':'2.0', 'error': fault.error(), id: None }
if config.use_jsonclass == True:
from jsonrpclib import jsonclass
result = jsonclass.load(result)
return result
def check_for_errors(result):
if not result:
# Notification
return result
if type(result) is not types.DictType:
raise TypeError('Response is not a dict.')
if 'jsonrpc' in result.keys() and float(result['jsonrpc']) > 2.0:
raise NotImplementedError('JSON-RPC version not yet supported.')
if 'result' not in result.keys() and 'error' not in result.keys():
raise ValueError('Response does not have a result or error key.')
if 'error' in result.keys() and result['error'] != None:
code = result['error']['code']
message = result['error']['message']
raise ProtocolError((code, message))
return result
def isbatch(result):
if type(result) not in (types.ListType, types.TupleType):
return False
if len(result) < 1:
return False
if type(result[0]) is not types.DictType:
return False
if 'jsonrpc' not in result[0].keys():
return False
try:
version = float(result[0]['jsonrpc'])
except ValueError:
raise ProtocolError('"jsonrpc" key must be a float(able) value.')
if version < 2:
return False
return True
def isnotification(request):
if 'id' not in request.keys():
# 2.0 notification
return True
if request['id'] == None:
# 1.0 notification
return True
return False

3
readme.md
View file

@ -55,7 +55,7 @@ Some of our innovative features;
</table>
## Required software
* Python 2.6+
* Python 2.7.9+ (earlier versions must be used with a current version of openssl)
* Cheetah 2.1.0+
## Howto
@ -90,7 +90,6 @@ Finally, a massive thanks to all those that remain in the shadows, the quiet one
* Support
* Please note that, aside from bug reports, we do *not* offer support. We can offer some help, but we really need you to understand the basics of your Linux or Windows OS. If you do not understand basics such as locating a database file, not running as root, setting up file permissions, or claiming a user derp, then we really cannot help you!
* IRC: `irc.freenode.net` channel `#SickGear`
* Webchat IRC: [webchat link](http://webchat.freenode.net/?channels=SickGear)
## Screenies
<table><thead></thead><tbody>

55
sickbeard/__init__.py
View file

@ -32,21 +32,16 @@ import os.path
import uuid
import base64
sys.path.insert(1, os.path.abspath('../lib'))
from sickbeard import providers, metadata, config, webserveInit, searchBacklog, showUpdater, versionChecker, \
autoPostProcesser, subtitles, traktChecker, helpers, db, exceptions, show_queue, search_queue, scheduler, \
show_name_helpers, logger, naming, searchRecent, searchProper, scene_numbering, scene_exceptions, name_cache
from sickbeard.providers.generic import GenericProvider
from providers import btn, newznab, womble, thepiratebay, torrentleech, kat, iptorrents, grabtheinfo, scenetime, pretome, \
omgwtfnzbs, scc, torrentday, hdbits, speedcd, nyaatorrents, torrentbytes, beyondhd, gftracker, transmithe_net, \
bitsoup, tokyotoshokan, animenzb, rarbg, morethan, alpharatio, pisexy, torrentshack, torrenting, funfile
from sickbeard.config import CheckSection, check_setting_int, check_setting_str, check_setting_float, ConfigMigrator, \
naming_ep_type, minimax
from sickbeard import helpers, logger, db, naming, metadata, providers, scene_exceptions, scene_numbering, \
scheduler, auto_post_processer, search_queue, search_propers, search_recent, search_backlog, \
show_queue, show_updater, subtitles, traktChecker, version_checker
from sickbeard.config import CheckSection, check_setting_int, check_setting_str, ConfigMigrator, minimax
from sickbeard.common import SD, SKIPPED
from sickbeard.databases import mainDB, cache_db, failed_db
from indexers.indexer_api import indexerApi
from indexers.indexer_exceptions import indexer_shownotfound, indexer_exception, indexer_error, \
indexer_episodenotfound, indexer_attributenotfound, indexer_seasonnotfound, indexer_userabort, indexerExcepts
from sickbeard.common import SD, SKIPPED, NAMING_REPEAT
from sickbeard.databases import mainDB, cache_db, failed_db
from sickbeard.providers.generic import GenericProvider
from lib.configobj import ConfigObj
PID = None
@ -1020,7 +1015,7 @@ def initialize(consoleLogging=True):
if hasattr(torrent_prov, 'options'):
torrent_prov.options = check_setting_str(CFG, prov_id_uc, prov_id + '_options', '')
if hasattr(torrent_prov, '_seed_ratio'):
torrent_prov._seed_ratio = check_setting_str(CFG, prov_id_uc, prov_id + '_seed_ratio', '')
torrent_prov._seed_ratio = check_setting_str(CFG, prov_id_uc, prov_id + '_seed_ratio', '').replace('None', '')
if hasattr(torrent_prov, 'seed_time'):
torrent_prov.seed_time = check_setting_int(CFG, prov_id_uc, prov_id + '_seed_time', '')
if hasattr(torrent_prov, 'minseed'):
@ -1029,6 +1024,8 @@ def initialize(consoleLogging=True):
torrent_prov.minleech = check_setting_int(CFG, prov_id_uc, prov_id + '_minleech', 0)
if hasattr(torrent_prov, 'freeleech'):
torrent_prov.freeleech = bool(check_setting_int(CFG, prov_id_uc, prov_id + '_freeleech', 0))
if hasattr(torrent_prov, 'reject_m2ts'):
torrent_prov.reject_m2ts = bool(check_setting_int(CFG, prov_id_uc, prov_id + '_reject_m2ts', 0))
if hasattr(torrent_prov, 'enable_recentsearch'):
torrent_prov.enable_recentsearch = bool(check_setting_int(CFG, prov_id_uc,
prov_id + '_enable_recentsearch', 1))
@ -1105,7 +1102,7 @@ def initialize(consoleLogging=True):
# initialize schedulers
# updaters
update_now = datetime.timedelta(minutes=0)
versionCheckScheduler = scheduler.Scheduler(versionChecker.CheckVersion(),
versionCheckScheduler = scheduler.Scheduler(version_checker.CheckVersion(),
cycleTime=datetime.timedelta(hours=UPDATE_FREQUENCY),
threadName='CHECKVERSION',
silent=False)
@ -1114,7 +1111,7 @@ def initialize(consoleLogging=True):
cycleTime=datetime.timedelta(seconds=3),
threadName='SHOWQUEUE')
showUpdateScheduler = scheduler.Scheduler(showUpdater.ShowUpdater(),
showUpdateScheduler = scheduler.Scheduler(show_updater.ShowUpdater(),
cycleTime=datetime.timedelta(hours=1),
threadName='SHOWUPDATER',
start_time=datetime.time(hour=SHOW_UPDATE_HOUR),
@ -1126,19 +1123,19 @@ def initialize(consoleLogging=True):
threadName='SEARCHQUEUE')
update_interval = datetime.timedelta(minutes=RECENTSEARCH_FREQUENCY)
recentSearchScheduler = scheduler.Scheduler(searchRecent.RecentSearcher(),
recentSearchScheduler = scheduler.Scheduler(search_recent.RecentSearcher(),
cycleTime=update_interval,
threadName='RECENTSEARCHER',
run_delay=update_now if RECENTSEARCH_STARTUP
else datetime.timedelta(minutes=5),
prevent_cycle_run=searchQueueScheduler.action.is_recentsearch_in_progress)
backlogSearchScheduler = searchBacklog.BacklogSearchScheduler(searchBacklog.BacklogSearcher(),
cycleTime=datetime.timedelta(minutes=get_backlog_cycle_time()),
threadName='BACKLOG',
run_delay=update_now if BACKLOG_STARTUP
else datetime.timedelta(minutes=10),
prevent_cycle_run=searchQueueScheduler.action.is_standard_backlog_in_progress)
backlogSearchScheduler = search_backlog.BacklogSearchScheduler(search_backlog.BacklogSearcher(),
cycleTime=datetime.timedelta(minutes=get_backlog_cycle_time()),
threadName='BACKLOG',
run_delay=update_now if BACKLOG_STARTUP
else datetime.timedelta(minutes=10),
prevent_cycle_run=searchQueueScheduler.action.is_standard_backlog_in_progress)
search_intervals = {'15m': 15, '45m': 45, '90m': 90, '4h': 4 * 60, 'daily': 24 * 60}
if CHECK_PROPERS_INTERVAL in search_intervals:
@ -1148,7 +1145,7 @@ def initialize(consoleLogging=True):
update_interval = datetime.timedelta(hours=1)
run_at = datetime.time(hour=1) # 1 AM
properFinderScheduler = scheduler.Scheduler(searchProper.ProperSearcher(),
properFinderScheduler = scheduler.Scheduler(search_propers.ProperSearcher(),
cycleTime=update_interval,
threadName='FINDPROPERS',
start_time=run_at,
@ -1156,7 +1153,7 @@ def initialize(consoleLogging=True):
prevent_cycle_run=searchQueueScheduler.action.is_propersearch_in_progress)
# processors
autoPostProcesserScheduler = scheduler.Scheduler(autoPostProcesser.PostProcesser(),
autoPostProcesserScheduler = scheduler.Scheduler(auto_post_processer.PostProcesser(),
cycleTime=datetime.timedelta(
minutes=AUTOPOSTPROCESSER_FREQUENCY),
threadName='POSTPROCESSER',
@ -1335,12 +1332,12 @@ def halt():
def sig_handler(signum=None, frame=None):
if type(signum) != type(None):
if isinstance(signum, type(None)):
logger.log(u'Signal %i caught, saving and exiting...' % int(signum))
events.put(events.SystemEvent.SHUTDOWN)
def saveAll():
def save_all():
global showList
# write all shows
@ -1356,7 +1353,7 @@ def saveAll():
def restart(soft=True):
if soft:
halt()
saveAll()
save_all()
logger.log(u'Re-initializing all data')
initialize()
else:
@ -1515,6 +1512,8 @@ def save_config():
new_config[prov_id_uc][prov_id + '_minleech'] = int(torrent_prov.minleech)
if hasattr(torrent_prov, 'freeleech'):
new_config[prov_id_uc][prov_id + '_freeleech'] = int(torrent_prov.freeleech)
if hasattr(torrent_prov, 'reject_m2ts'):
new_config[prov_id_uc][prov_id + '_reject_m2ts'] = int(torrent_prov.reject_m2ts)
if hasattr(torrent_prov, 'enable_recentsearch'):
new_config[prov_id_uc][prov_id + '_enable_recentsearch'] = int(torrent_prov.enable_recentsearch)
if hasattr(torrent_prov, 'enable_backlog'):
@ -1799,7 +1798,7 @@ def save_config():
new_config.write()
def launchBrowser(start_port=None):
def launch_browser(start_port=None):
if not start_port:
start_port = WEB_PORT
if ENABLE_HTTPS:

2
sickbeard/autoPostProcesser.py → sickbeard/auto_post_processer.py
View file

@ -45,4 +45,4 @@ class PostProcesser():
processTV.processDir(sickbeard.TV_DOWNLOAD_DIR)
self.amActive = False
self.amActive = False

6
sickbeard/bs4_parser.py
View file

@ -1,4 +1,5 @@
from bs4 import BeautifulSoup
import re
class BS4Parser:
@ -12,6 +13,11 @@ class BS4Parser:
kwargs_new[k] = v
tag, attr = [x in kwargs_new and kwargs_new.pop(x) or y for (x, y) in [('tag', 'table'), ('attr', '')]]
if attr:
args = (re.sub(r'(?is).*(<%(tag)s[^>]+%(attr)s[^>]*>.*</%(tag)s>).*' % {'tag': tag, 'attr': attr},
r'<html><head></head><body>\1</body></html>', args[0]).strip(),) + args[1:]
self.soup = BeautifulSoup(*args, **kwargs_new)
def __enter__(self):

11
sickbeard/clients/rtorrent.py
View file

@ -16,12 +16,9 @@
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
from base64 import b64encode
import sickbeard
from sickbeard.clients.generic import GenericClient
from lib.rtorrent import RTorrent
from lib.rtorrent.err import MethodError
class rTorrentAPI(GenericClient):
@ -45,7 +42,6 @@ class rTorrentAPI(GenericClient):
return self.auth
def _add_torrent_uri(self, result):
filedata = None
if not self.auth:
return False
@ -62,7 +58,7 @@ class rTorrentAPI(GenericClient):
# Set label
if sickbeard.TORRENT_LABEL:
torrent.set_custom(1, sickbeard.TORRENT_LABEL.lower())
torrent.set_custom(1, sickbeard.TORRENT_LABEL)
if sickbeard.TORRENT_PATH:
torrent.set_directory(sickbeard.TORRENT_PATH)
@ -76,7 +72,6 @@ class rTorrentAPI(GenericClient):
return False
def _add_torrent_file(self, result):
filedata = None
if not self.auth:
return False
@ -84,7 +79,7 @@ class rTorrentAPI(GenericClient):
if not result:
return False
# group_name = 'sb_test'.lower() ##### Use provider instead of _test
# group_name = 'sb_test' ##### Use provider instead of _test
# if not self._set_torrent_ratio(group_name):
# return False
@ -98,7 +93,7 @@ class rTorrentAPI(GenericClient):
# Set label
if sickbeard.TORRENT_LABEL:
torrent.set_custom(1, sickbeard.TORRENT_LABEL.lower())
torrent.set_custom(1, sickbeard.TORRENT_LABEL)
if sickbeard.TORRENT_PATH:
torrent.set_directory(sickbeard.TORRENT_PATH)

16
sickbeard/common.py
View file

@ -91,7 +91,7 @@ class Quality:
RAWHDTV = 1 << 3 # 8 -- 720p/1080i mpeg2 (trollhd releases)
FULLHDTV = 1 << 4 # 16 -- 1080p HDTV (QCF releases)
HDWEBDL = 1 << 5 # 32
FULLHDWEBDL = 1 << 6 # 64 -- 1080p web-dl
FULLHDWEBDL = 1 << 6 # 64 -- 1080p web-dl
HDBLURAY = 1 << 7 # 128
FULLHDBLURAY = 1 << 8 # 256
@ -171,7 +171,7 @@ class Quality:
@staticmethod
def sceneQuality(name, anime=False):
"""
Return The quality from the scene episode File
Return The quality from the scene episode File
"""
name = os.path.basename(name)
@ -205,15 +205,15 @@ class Quality:
else:
return Quality.UNKNOWN
if checkName(['(pdtv|hdtv|dsr|tvrip)([-]|.((aac|ac3|dd).?\d\.?\d.)*(xvid|x264|h.?264))'], all) and not checkName(['(720|1080)[pi]'], all) \
if checkName(['(pdtv|hdtv|dsr|tvrip)([-]|.((aac|ac3|dd).?\d\.?\d.)*(xvid|x264|h.?264))'], all) and not checkName(['(720|1080|2160)[pi]'], all) \
and not checkName(['hr.ws.pdtv.(x264|h.?264)'], any):
return Quality.SDTV
elif checkName(['web.?dl|web.?rip', 'xvid|x264|h.?264'], all) and not checkName(['(720|1080)[pi]'], all):
elif checkName(['web.?dl|web.?rip', 'xvid|x264|h.?264'], all) and not checkName(['(720|1080|2160)[pi]'], all):
return Quality.SDTV
elif checkName(['(dvd.?rip|b[r|d]rip)(.ws)?(.(xvid|divx|x264|h.?264))?'], any) and not checkName(['(720|1080)[pi]'], all):
elif checkName(['(dvd.?rip|b[r|d]rip)(.ws)?(.(xvid|divx|x264|h.?264))?'], any) and not checkName(['(720|1080|2160)[pi]'], all):
return Quality.SDDVD
elif checkName(['720p', 'hdtv', 'x264|h.?264'], all) or checkName(['hr.ws.pdtv.(x264|h.?264)'], any) \
and not checkName(['(1080)[pi]'], all):
and not checkName(['(1080|2160)[pi]'], all):
return Quality.HDTV
elif checkName(['720p|1080i', 'hdtv', 'mpeg-?2'], all) or checkName(['1080[pi].hdtv', 'h.?264'], all):
return Quality.RAWHDTV
@ -337,7 +337,7 @@ ANY = Quality.combineQualities(
[Quality.SDTV, Quality.SDDVD, Quality.HDTV, Quality.FULLHDTV, Quality.HDWEBDL, Quality.FULLHDWEBDL,
Quality.HDBLURAY, Quality.FULLHDBLURAY, Quality.UNKNOWN], []) # SD + HD
# legacy template, cant remove due to reference in mainDB upgrade?
# legacy template, cant remove due to reference in mainDB upgrade?
BEST = Quality.combineQualities([Quality.SDTV, Quality.HDTV, Quality.HDWEBDL], [Quality.HDTV])
qualityPresets = (SD, HD, HD720p, HD1080p, ANY)
@ -401,4 +401,4 @@ class Overview:
countryList = {'Australia': 'AU',
'Canada': 'CA',
'USA': 'US'}
'USA': 'US'}

3
sickbeard/config.py
View file

@ -22,8 +22,9 @@ import re
import urlparse
import sickbeard
import sickbeard.providers
from sickbeard import encodingKludge as ek
from sickbeard import helpers, logger, naming, db, providers
from sickbeard import helpers, logger, naming, db
naming_ep_type = ('%(seasonnumber)dx%(episodenumber)02d',

25
sickbeard/helpers.py
View file

@ -94,18 +94,17 @@ def remove_extension(name):
return name
def remove_non_release_groups(name, anime=False):
def remove_non_release_groups(name, is_anime=False):
"""
Remove non release groups from name
"""
if name:
rc = [re.compile(r'(?i)' + v) for v in [
'([\s\.\-_\[\{\(]*(no-rar|nzbgeek|ripsalot|rp|siklopentan)[\s\.\-_\]\}\)]*)$',
'(?<=\w)([\s\.\-_]*[\[\{\(][\s\.\-_]*(www\.\w+.\w+)[\s\.\-_]*[\]\}\)][\s\.\-_]*)$',
'(?<=\w)([\s\.\-_]*[\[\{\(]\s*(rar(bg|tv)|((e[tz]|v)tv))[\s\.\-_]*[\]\}\)][\s\.\-_]*)$'] +
(['(?<=\w)([\s\.\-_]*[\[\{\(][\s\.\-_]*[\w\s\.\-\_]+[\s\.\-_]*[\]\}\)][\s\.\-_]*)$'], [])[anime]
]
'([\s\.\-_\[\{\(]*(no-rar|nzbgeek|ripsalot|rp|siklopentan)[\s\.\-_\]\}\)]*)$',
'(?<=\w)([\s\.\-_]*[\[\{\(][\s\.\-_]*(www\.\w+.\w+)[\s\.\-_]*[\]\}\)][\s\.\-_]*)$',
'(?<=\w)([\s\.\-_]*[\[\{\(]\s*(rar(bg|tv)|((e[tz]|v)tv))[\s\.\-_]*[\]\}\)][\s\.\-_]*)$'] +
(['(?<=\w)([\s\.\-_]*[\[\{\(][\s\.\-_]*[\w\s\.\-\_]+[\s\.\-_]*[\]\}\)][\s\.\-_]*)$'], [])[is_anime]]
rename = name
while rename:
for regex in rc:
@ -1165,7 +1164,7 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N
logger.log(u'HTTP error %s while loading URL %s' % (e.errno, url), logger.WARNING)
return
except requests.exceptions.ConnectionError as e:
logger.log(u'Connection error msg:%s while loading URL %s' % (str(e.message), url), logger.WARNING)
logger.log(u'Internet connection error msg:%s while loading URL %s' % (str(e.message), url), logger.WARNING)
return
except requests.exceptions.ReadTimeout as e:
logger.log(u'Read timed out msg:%s while loading URL %s' % (str(e.message), url), logger.WARNING)
@ -1173,8 +1172,11 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N
except (requests.exceptions.Timeout, socket.timeout) as e:
logger.log(u'Connection timed out msg:%s while loading URL %s' % (str(e.message), url), logger.WARNING)
return
except Exception:
logger.log(u'Exception caught while loading URL %s Detail... %s' % (url, traceback.format_exc()), logger.WARNING)
except Exception as e:
if e.message:
logger.log(u'Exception caught while loading URL %s\r\nDetail... %s\r\n%s' % (url, str(e.message), traceback.format_exc()), logger.WARNING)
else:
logger.log(u'Unknown exception while loading URL %s\r\nDetail... %s' % (url, traceback.format_exc()), logger.WARNING)
return
if json:
@ -1410,6 +1412,7 @@ def clear_unused_providers():
myDB = db.DBConnection('cache.db')
myDB.action('DELETE FROM provider_cache WHERE provider NOT IN (%s)' % ','.join(['?'] * len(providers)), providers)
def make_search_segment_html_string(segment, max_eps=5):
seg_str = ''
if segment and not isinstance(segment, list):
@ -1427,3 +1430,7 @@ def make_search_segment_html_string(segment, max_eps=5):
episodes = ['S' + str(x.season).zfill(2) + 'E' + str(x.episode).zfill(2) for x in segment]
seg_str = u'Episode' + maybe_plural(len(episodes)) + ': ' + ', '.join(episodes)
return seg_str
def has_anime():
return False if not sickbeard.showList else any(filter(lambda show: show.is_anime, sickbeard.showList))

292
sickbeard/properFinder.py
View file

@ -32,22 +32,30 @@ from sickbeard import history
from sickbeard.common import DOWNLOADED, SNATCHED, SNATCHED_PROPER, Quality
from name_parser.parser import NameParser, InvalidNameException, InvalidShowException
from name_parser.parser import NameParser
def searchPropers():
def search_propers():
if not sickbeard.DOWNLOAD_PROPERS:
return
logger.log(u'Beginning the search for new propers')
logger.log(u'Beginning search for new propers')
propers = _getProperList()
age_shows, age_anime = 2, 14
aired_since_shows = datetime.datetime.today() - datetime.timedelta(days=age_shows)
aired_since_anime = datetime.datetime.today() - datetime.timedelta(days=age_anime)
recent_shows, recent_anime = _recent_history(aired_since_shows, aired_since_anime)
if recent_shows or recent_anime:
propers = _get_proper_list(aired_since_shows, recent_shows, recent_anime)
if propers:
_downloadPropers(propers)
if propers:
_download_propers(propers)
else:
logger.log(u'No downloads or snatches found for the last %s%s days to use for a propers search' %
(age_shows, ('', ' (%s for anime)' % age_anime)[helpers.has_anime()]))
_set_lastProperSearch(datetime.datetime.today().toordinal())
_set_last_proper_search(datetime.datetime.today().toordinal())
run_at = ''
if None is sickbeard.properFinderScheduler.start_time:
@ -59,226 +67,242 @@ def searchPropers():
logger.log(u'Completed the search for new propers%s' % run_at)
def _getProperList():
def _get_proper_list(aired_since_shows, recent_shows, recent_anime):
propers = {}
search_date = datetime.datetime.today() - datetime.timedelta(days=2)
# for each provider get a list of the
origThreadName = threading.currentThread().name
orig_thread_name = threading.currentThread().name
providers = [x for x in sickbeard.providers.sortedProviderList() if x.is_active()]
for curProvider in providers:
threading.currentThread().name = origThreadName + ' :: [' + curProvider.name + ']'
for cur_provider in providers:
if not recent_anime and cur_provider.anime_only:
continue
threading.currentThread().name = orig_thread_name + ' :: [' + cur_provider.name + ']'
logger.log(u'Searching for any new PROPER releases from ' + curProvider.name)
logger.log(u'Searching for new PROPER releases')
try:
curPropers = curProvider.find_propers(search_date)
found_propers = cur_provider.find_propers(search_date=aired_since_shows, shows=recent_shows, anime=recent_anime)
except exceptions.AuthException as e:
logger.log(u'Authentication error: ' + ex(e), logger.ERROR)
continue
except Exception as e:
logger.log(u'Error while searching ' + curProvider.name + ', skipping: ' + ex(e), logger.ERROR)
logger.log(u'Error while searching ' + cur_provider.name + ', skipping: ' + ex(e), logger.ERROR)
logger.log(traceback.format_exc(), logger.DEBUG)
continue
finally:
threading.currentThread().name = origThreadName
threading.currentThread().name = orig_thread_name
# if they haven't been added by a different provider than add the proper to the list
for x in curPropers:
name = _genericName(x.name)
if not name in propers:
logger.log(u'Found new proper: ' + x.name, logger.DEBUG)
x.provider = curProvider
propers[name] = x
count = 0
np = NameParser(False, try_scene_exceptions=True)
for x in found_propers:
name = _generic_name(x.name)
if name not in propers:
try:
parse_result = np.parse(x.title)
if parse_result.series_name and parse_result.episode_numbers and \
parse_result.show.indexerid in recent_shows + recent_anime:
logger.log(u'Found new proper: ' + x.name, logger.DEBUG)
x.show = parse_result.show.indexerid
x.provider = cur_provider
propers[name] = x
count += 1
except Exception:
continue
cur_provider.log_result('Propers', count, '%s' % cur_provider.name)
# take the list of unique propers and get it sorted by
sortedPropers = sorted(propers.values(), key=operator.attrgetter('date'), reverse=True)
finalPropers = []
sorted_propers = sorted(propers.values(), key=operator.attrgetter('date'), reverse=True)
verified_propers = []
for curProper in sortedPropers:
try:
myParser = NameParser(False)
parse_result = myParser.parse(curProper.name)
except InvalidNameException:
logger.log(u'Unable to parse the filename ' + curProper.name + ' into a valid episode', logger.DEBUG)
continue
except InvalidShowException:
logger.log(u'Unable to parse the filename ' + curProper.name + ' into a valid show', logger.DEBUG)
continue
if not parse_result.series_name:
continue
if not parse_result.episode_numbers:
logger.log(
u'Ignoring ' + curProper.name + ' because it\'s for a full season rather than specific episode',
logger.DEBUG)
continue
logger.log(
u'Successful match! Result ' + parse_result.original_name + ' matched to show ' + parse_result.show.name,
logger.DEBUG)
for cur_proper in sorted_propers:
# set the indexerid in the db to the show's indexerid
curProper.indexerid = parse_result.show.indexerid
cur_proper.indexerid = parse_result.show.indexerid
# set the indexer in the db to the show's indexer
curProper.indexer = parse_result.show.indexer
cur_proper.indexer = parse_result.show.indexer
# populate our Proper instance
curProper.season = parse_result.season_number if parse_result.season_number != None else 1
curProper.episode = parse_result.episode_numbers[0]
curProper.release_group = parse_result.release_group
curProper.version = parse_result.version
curProper.quality = Quality.nameQuality(curProper.name, parse_result.is_anime)
cur_proper.season = parse_result.season_number if None is not parse_result.season_number else 1
cur_proper.episode = parse_result.episode_numbers[0]
cur_proper.release_group = parse_result.release_group
cur_proper.version = parse_result.version
cur_proper.quality = Quality.nameQuality(cur_proper.name, parse_result.is_anime)
# only get anime proper if it has release group and version
if parse_result.is_anime:
if not curProper.release_group and curProper.version == -1:
logger.log(u'Proper ' + curProper.name + ' doesn\'t have a release group and version, ignoring it',
if not cur_proper.release_group and -1 == cur_proper.version:
logger.log(u'Proper %s doesn\'t have a release group and version, ignoring it' % cur_proper.name,
logger.DEBUG)
continue
if not show_name_helpers.filterBadReleases(curProper.name, parse=False):
logger.log(u'Proper ' + curProper.name + ' isn\'t a valid scene release that we want, ignoring it',
if not show_name_helpers.pass_wordlist_checks(cur_proper.name, parse=False):
logger.log(u'Proper %s isn\'t a valid scene release that we want, ignoring it' % cur_proper.name,
logger.DEBUG)
continue
if parse_result.show.rls_ignore_words and search.filter_release_name(curProper.name,
parse_result.show.rls_ignore_words):
logger.log(
u'Ignoring ' + curProper.name + ' based on ignored words filter: ' + parse_result.show.rls_ignore_words,
logger.MESSAGE)
re_extras = dict(re_prefix='.*', re_suffix='.*')
result = show_name_helpers.contains_any(cur_proper.name, parse_result.show.rls_ignore_words, **re_extras)
if None is not result and result:
logger.log(u'Ignored: %s for containing ignore word' % cur_proper.name)
continue
if parse_result.show.rls_require_words and not search.filter_release_name(curProper.name,
parse_result.show.rls_require_words):
logger.log(
u'Ignoring ' + curProper.name + ' based on required words filter: ' + parse_result.show.rls_require_words,
logger.MESSAGE)
result = show_name_helpers.contains_any(cur_proper.name, parse_result.show.rls_require_words, **re_extras)
if None is not result and not result:
logger.log(u'Ignored: %s for not containing any required word match' % cur_proper.name)
continue
# check if we actually want this proper (if it's the right quality)
myDB = db.DBConnection()
sqlResults = myDB.select('SELECT status FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?',
[curProper.indexerid, curProper.season, curProper.episode])
if not sqlResults:
my_db = db.DBConnection()
sql_results = my_db.select('SELECT status FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?',
[cur_proper.indexerid, cur_proper.season, cur_proper.episode])
if not sql_results:
continue
# only keep the proper if we have already retrieved the same quality ep (don't get better/worse ones)
oldStatus, oldQuality = Quality.splitCompositeStatus(int(sqlResults[0]['status']))
if oldStatus not in (DOWNLOADED, SNATCHED) or oldQuality != curProper.quality:
old_status, old_quality = Quality.splitCompositeStatus(int(sql_results[0]['status']))
if old_status not in (DOWNLOADED, SNATCHED) or cur_proper.quality != old_quality:
continue
# check if we actually want this proper (if it's the right release group and a higher version)
if parse_result.is_anime:
myDB = db.DBConnection()
sqlResults = myDB.select(
my_db = db.DBConnection()
sql_results = my_db.select(
'SELECT release_group, version FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?',
[curProper.indexerid, curProper.season, curProper.episode])
[cur_proper.indexerid, cur_proper.season, cur_proper.episode])
oldVersion = int(sqlResults[0]['version'])
oldRelease_group = (sqlResults[0]['release_group'])
old_version = int(sql_results[0]['version'])
old_release_group = (sql_results[0]['release_group'])
if oldVersion > -1 and oldVersion < curProper.version:
logger.log('Found new anime v' + str(curProper.version) + ' to replace existing v' + str(oldVersion))
if -1 < old_version < cur_proper.version:
logger.log(u'Found new anime v%s to replace existing v%s' % (cur_proper.version, old_version))
else:
continue
if oldRelease_group != curProper.release_group:
logger.log('Skipping proper from release group: ' + curProper.release_group + ', does not match existing release group: ' + oldRelease_group)
if cur_proper.release_group != old_release_group:
logger.log(u'Skipping proper from release group: %s, does not match existing release group: %s' %
(cur_proper.release_group, old_release_group))
continue
# if the show is in our list and there hasn't been a proper already added for that particular episode then add it to our list of propers
if curProper.indexerid != -1 and (curProper.indexerid, curProper.season, curProper.episode) not in map(
operator.attrgetter('indexerid', 'season', 'episode'), finalPropers):
logger.log(u'Found a proper that we need: ' + str(curProper.name))
finalPropers.append(curProper)
# if the show is in our list and there hasn't been a proper already added for that particular episode
# then add it to our list of propers
if cur_proper.indexerid != -1 and (cur_proper.indexerid, cur_proper.season, cur_proper.episode) not in map(
operator.attrgetter('indexerid', 'season', 'episode'), verified_propers):
logger.log(u'Found a proper that may be useful: %s' % cur_proper.name)
verified_propers.append(cur_proper)
return finalPropers
return verified_propers
def _downloadPropers(properList):
for curProper in properList:
def _download_propers(proper_list):
historyLimit = datetime.datetime.today() - datetime.timedelta(days=30)
for cur_proper in proper_list:
history_limit = datetime.datetime.today() - datetime.timedelta(days=30)
# make sure the episode has been downloaded before
myDB = db.DBConnection()
historyResults = myDB.select(
'SELECT resource FROM history '
'WHERE showid = ? AND season = ? AND episode = ? AND quality = ? AND date >= ? '
my_db = db.DBConnection()
history_results = my_db.select(
'SELECT resource FROM history ' +
'WHERE showid = ? AND season = ? AND episode = ? AND quality = ? AND date >= ? ' +
'AND action IN (' + ','.join([str(x) for x in Quality.SNATCHED]) + ')',
[curProper.indexerid, curProper.season, curProper.episode, curProper.quality,
historyLimit.strftime(history.dateFormat)])
[cur_proper.indexerid, cur_proper.season, cur_proper.episode, cur_proper.quality,
history_limit.strftime(history.dateFormat)])
# if we didn't download this episode in the first place we don't know what quality to use for the proper so we can't do it
if len(historyResults) == 0:
logger.log(
u'Unable to find an original history entry for proper ' + curProper.name + ' so I\'m not downloading it.')
# if we didn't download this episode in the first place we don't know what quality to use for the proper = skip
if 0 == len(history_results):
logger.log(u'Skipping download because cannot find an original history entry for proper ' + cur_proper.name)
continue
else:
# get the show object
showObj = helpers.findCertainShow(sickbeard.showList, curProper.indexerid)
if showObj == None:
show_obj = helpers.findCertainShow(sickbeard.showList, cur_proper.indexerid)
if None is show_obj:
logger.log(u'Unable to find the show with indexerid ' + str(
curProper.indexerid) + ' so unable to download the proper', logger.ERROR)
cur_proper.indexerid) + ' so unable to download the proper', logger.ERROR)
continue
# make sure that none of the existing history downloads are the same proper we're trying to download
clean_proper_name = _genericName(helpers.remove_non_release_groups(curProper.name, showObj.anime))
isSame = False
for curResult in historyResults:
clean_proper_name = _generic_name(helpers.remove_non_release_groups(cur_proper.name, show_obj.is_anime()))
is_same = False
for result in history_results:
# if the result exists in history already we need to skip it
if _genericName(helpers.remove_non_release_groups(curResult['resource'])) == clean_proper_name:
isSame = True
if clean_proper_name == _generic_name(helpers.remove_non_release_groups(result['resource'])):
is_same = True
break
if isSame:
if is_same:
logger.log(u'This proper is already in history, skipping it', logger.DEBUG)
continue
epObj = showObj.getEpisode(curProper.season, curProper.episode)
ep_obj = show_obj.getEpisode(cur_proper.season, cur_proper.episode)
# make the result object
result = curProper.provider.get_result([epObj], curProper.url)
result = cur_proper.provider.get_result([ep_obj], cur_proper.url)
if None is result:
continue
result.name = curProper.name
result.quality = curProper.quality
result.version = curProper.version
result.name = cur_proper.name
result.quality = cur_proper.quality
result.version = cur_proper.version
# snatch it
search.snatchEpisode(result, SNATCHED_PROPER)
search.snatch_episode(result, SNATCHED_PROPER)
def _genericName(name):
def _recent_history(aired_since_shows, aired_since_anime):
recent_shows, recent_anime = [], []
aired_since_shows = aired_since_shows.toordinal()
aired_since_anime = aired_since_anime.toordinal()
my_db = db.DBConnection()
sql_results = my_db.select(
'SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.airdate FROM tv_episodes AS e' +
' INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id)' +
' WHERE e.airdate >= %s' % min(aired_since_shows, aired_since_anime) +
' AND (e.status IN (%s))' % ','.join([str(x) for x in Quality.DOWNLOADED + Quality.SNATCHED])
)
for sqlshow in sql_results:
show = helpers.findCertainShow(sickbeard.showList, sqlshow['showid'])
if show:
if sqlshow['airdate'] >= aired_since_shows and not show.is_anime:
sqlshow['showid'] not in recent_shows and recent_shows.append(sqlshow['showid'])
else:
sqlshow['showid'] not in recent_anime and show.is_anime and recent_anime.append(sqlshow['showid'])
return recent_shows, recent_anime
def _generic_name(name):
return name.replace('.', ' ').replace('-', ' ').replace('_', ' ').lower()
def _set_lastProperSearch(when):
logger.log(u'Setting the last Proper search in the DB to ' + str(when), logger.DEBUG)
def _set_last_proper_search(when):
myDB = db.DBConnection()
sqlResults = myDB.select('SELECT * FROM info')
logger.log(u'Setting the last Proper search in the DB to %s' % when, logger.DEBUG)
if len(sqlResults) == 0:
myDB.action('INSERT INTO info (last_backlog, last_indexer, last_proper_search) VALUES (?,?,?)',
[0, 0, str(when)])
my_db = db.DBConnection()
sql_results = my_db.select('SELECT * FROM info')
if 0 == len(sql_results):
my_db.action('INSERT INTO info (last_backlog, last_indexer, last_proper_search) VALUES (?,?,?)',
[0, 0, str(when)])
else:
myDB.action('UPDATE info SET last_proper_search=' + str(when))
my_db.action('UPDATE info SET last_proper_search=%s' % when)
def _get_lastProperSearch():
myDB = db.DBConnection()
sqlResults = myDB.select('SELECT * FROM info')
def _get_last_proper_search():
my_db = db.DBConnection()
sql_results = my_db.select('SELECT * FROM info')
try:
last_proper_search = datetime.date.fromordinal(int(sqlResults[0]['last_proper_search']))
last_proper_search = datetime.date.fromordinal(int(sql_results[0]['last_proper_search']))
except:
return datetime.date.fromordinal(1)

78
sickbeard/providers/__init__.py
View file

@ -16,42 +16,56 @@
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['womble',
'btn',
'thepiratebay',
'kat',
'torrentleech',
'scc',
'torrentday',
'hdbits',
'iptorrents',
'omgwtfnzbs',
'speedcd',
'nyaatorrents',
'torrentbytes',
'bitsoup',
'tokyotoshokan',
'animenzb',
'rarbg',
'morethan',
'alpharatio',
'pisexy',
'torrentshack',
'beyondhd',
'gftracker',
'transmithe_net',
'grabtheinfo',
'scenetime',
'pretome',
'torrenting',
'funfile',
]
from os import sys
import sickbeard
import generic
from . import generic
from sickbeard import logger
# usenet
from . import newznab, omgwtfnzbs, womble
# torrent
from . import alpharatio, beyondhd, bitmetv, bitsoup, btn, freshontv, funfile, gftracker, grabtheinfo, \
hdbits, hdspace, iptorrents, kat, morethan, pisexy, pretome, rarbg, scc, scenetime, shazbat, speedcd, strike, \
thepiratebay, torrentbytes, torrentday, torrenting, torrentleech, torrentshack, transmithe_net, tvchaosuk
# anime
from . import nyaatorrents, tokyotoshokan
__all__ = ['omgwtfnzbs',
'womble',
'alpharatio',
'beyondhd',
'bitmetv',
'bitsoup',
'btn',
'freshontv',
'funfile',
'gftracker',
'grabtheinfo',
'hdbits',
'hdspace',
'iptorrents',
'kat',
'morethan',
'pisexy',
'pretome',
'rarbg',
'scc',
'scenetime',
'shazbat',
'speedcd',
'strike',
'thepiratebay',
'torrentbytes',
'torrentday',
'torrenting',
'torrentleech',
'torrentshack',
'transmithe_net',
'tvchaosuk',
'nyaatorrents',
'tokyotoshokan',
]
def sortedProviderList():

75
sickbeard/providers/alpharatio.py
View file

@ -18,12 +18,12 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -36,52 +36,37 @@ class AlphaRatioProvider(generic.TorrentProvider):
self.url_base = 'https://alpharatio.cc/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'login.php',
'search': self.url_base + 'torrents.php?searchstr=%s'
+ '&tags_type=1&order_by=time&order_way=desc'
+ '&filter_cat[1]=1&filter_cat[2]=1&filter_cat[3]=1&filter_cat[4]=1&filter_cat[5]=1'
+ '&action=basic&searchsubmit=1',
'search': self.url_base + 'torrents.php?searchstr=%s%s&' + '&'.join(
['tags_type=1', 'order_by=time', 'order_way=desc'] +
['filter_cat[%s]=1' % c for c in 1, 2, 3, 4, 5] +
['action=basic', 'searchsubmit=1']),
'get': self.url_base + '%s'}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = AlphaRatioCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'session' in self.session.cookies
if logged_in():
return True
return super(AlphaRatioProvider, self)._authorised(logged_in=(lambda x=None: self.has_all_cookies('session')),
post_params={'keeplogged': '1', 'login': 'Login'})
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'keeplogged': '1', 'login': 'Login'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and 'Invalid Username/password' in response:
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'view', 'get': 'download'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['search'] % (search_string, ('', '&freetorrent=1')[self.freeleech])
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_url = self.urls['search'] % search_string
html = self.get_url(search_url)
cnt = len(items[mode])
@ -98,50 +83,44 @@ class AlphaRatioProvider(generic.TorrentProvider):
for tr in torrent_rows[1:]:
try:
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
if mode != 'Cache' and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers, size = [tryInt(n, n) for n in [
tr.find_all('td')[x].get_text().strip() for x in (-2, -1, -4)]]
if self._peers_fail(mode, seeders, leechers):
continue
title = tr.find('a', title=rc['info']).get_text().strip()
link = str(tr.find('a', title=rc['get'])['href']).replace('&amp;', '&').lstrip('/')
download_url = self.urls['get'] % link
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, use_or=False)
class AlphaRatioCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.cache_data()
return self.provider.get_cache_data()
provider = AlphaRatioProvider()

115
sickbeard/providers/animenzb.py
View file

@ -1,115 +0,0 @@
# Author: Nic Wolfe <nic@wolfeden.ca>
# URL: http://code.google.com/p/sickbeard/
#
# This file is part of Sick Beard.
#
# Sick Beard is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Sick Beard is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Sick Beard. If not, see <http://www.gnu.org/licenses/>.
import datetime
import urllib
from . import generic
from sickbeard import classes, show_name_helpers, logger, tvcache
class AnimeNZBProvider(generic.NZBProvider):
def __init__(self):
generic.NZBProvider.__init__(self, 'AnimeNZB', anime_only=True)
self.url = 'http://animenzb.com/'
self.cache = AnimeNZBCache(self)
def _get_season_search_strings(self, ep_obj):
return [x for x in show_name_helpers.makeSceneSeasonSearchString(self.show, ep_obj)]
def _get_episode_search_strings(self, ep_obj, add_string=''):
return [x for x in show_name_helpers.makeSceneSearchString(self.show, ep_obj)]
def _do_search(self, search_string, search_mode='eponly', epcount=0, age=0):
results = []
if self.show and not self.show.is_anime:
return results
params = {'cat': 'anime',
'q': search_string.encode('utf-8'),
'max': '100'}
search_url = self.url + 'rss?' + urllib.urlencode(params)
logger.log(u'Search url: %s' % search_url, logger.DEBUG)
data = self.cache.getRSSFeed(search_url)
if data and 'entries' in data:
items = data.entries
for curItem in items:
(title, url) = self._get_title_and_url(curItem)
if title and url:
results.append(curItem)
else:
logger.log(u'The data returned from %s is incomplete, this result is unusable' % self.name,
logger.DEBUG)
return results
def find_propers(self, date=None):
results = []
for item in self._do_search('v2|v3|v4|v5'):
(title, url) = self._get_title_and_url(item)
if 'published_parsed' in item and item['published_parsed']:
result_date = item.published_parsed
if result_date:
result_date = datetime.datetime(*result_date[0:6])
else:
logger.log(u'Unable to figure out the date for entry %s, skipping it' % title)
continue
if not date or result_date > date:
search_result = classes.Proper(title, url, result_date, self.show)
results.append(search_result)
return results
class AnimeNZBCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
def _getRSSData(self):
params = {'cat': 'anime'.encode('utf-8'),
'max': '100'.encode('utf-8')}
rss_url = self.provider.url + 'rss?' + urllib.urlencode(params)
logger.log(u'%s cache update URL: %s' % (self.provider.name, rss_url), logger.DEBUG)
data = self.getRSSFeed(rss_url)
if data and 'entries' in data:
return data.entries
return []
provider = AnimeNZBProvider()

56
sickbeard/providers/beyondhd.py
View file

@ -16,7 +16,6 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import time
from . import generic
@ -32,13 +31,12 @@ class BeyondHDProvider(generic.TorrentProvider):
self.url_base = 'https://beyondhd.me/'
self.urls = {'config_provider_home_uri': self.url_base,
'cache': self.url_base + 'api_tv.php?passkey=%s&cats=%s',
'search': '&search=%s',
}
'browse': self.url_base + 'api_tv.php?passkey=%s&cats=%s',
'search': '&search=%s'}
self.categories = {'Season': '89',
'Episode': '40,44,48,46,43,45',
'Cache': '40,44,48,89,46,43,45'}
'Episode': '40,44,48,43,45',
'Cache': '40,44,48,89,43,45'}
self.url = self.urls['config_provider_home_uri']
@ -53,54 +51,54 @@ class BeyondHDProvider(generic.TorrentProvider):
logger.log(u'Incorrect authentication credentials for %s : %s' % (self.name, data_json['error']), logger.DEBUG)
raise AuthException('Authentication credentials for %s are incorrect, check your config' % self.name)
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._check_auth():
return results
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
for mode in search_params.keys():
if 'Cache' != mode:
if mode in ['Season', 'Episode']:
show_type = self.show.air_by_date and 'Air By Date' \
or self.show.is_sports and 'Sports' or self.show.is_anime and 'Anime' or None
if show_type:
logger.log(u'Provider does not carry shows of type: [%s], skipping' % show_type, logger.DEBUG)
return results
mode_cats = (mode, 'Cache')['Propers' == mode]
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_url = self.urls['cache'] % (self.passkey, self.categories[mode])
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['browse'] % (self.passkey, self.categories[mode_cats])
if 'Cache' != mode:
search_url += self.urls['search'] % re.sub('[\.\s]+', ' ', search_string)
data_json = self.get_url(search_url, json=True)
cnt = len(results)
cnt = len(items[mode])
if data_json and 'results' in data_json and self._check_auth_from_data(data_json):
for item in data_json['results']:
seeders, leechers = item['seeders'], item['leechers']
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers = item.get('seeders', 0), item.get('leechers', 0)
if self._peers_fail(mode, seeders, leechers):
continue
title, download_url = item['file'], item['get']
title, download_url = item.get('file'), item.get('get')
if title and download_url:
results.append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(item.get('size'))))
self._log_result(mode, len(results) - cnt, search_url)
time.sleep(1.1)
# Sort items by seeders
results.sort(key=lambda tup: tup[2], reverse=True)
self._log_search(mode, len(items[mode]) - cnt, search_url)
self._sort_seeders(mode, items)
results = list(set(results + items[mode]))
return results
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
def _episode_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, scene=False, use_or=False)
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date, ['proper', 'repack'])
return generic.TorrentProvider._episode_strings(self, ep_obj, scene=False, **kwargs)
class BeyondHDCache(tvcache.TVCache):
@ -108,11 +106,9 @@ class BeyondHDCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 10 # cache update frequency
def _cache_data(self):
def _getRSSData(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = BeyondHDProvider()

131
sickbeard/providers/bitmetv.py Normal file
View file

@ -0,0 +1,131 @@
# coding=utf-8
#
# This file is part of SickGear.
#
# SickGear is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# SickGear is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import traceback
from . import generic
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
class BitmetvProvider(generic.TorrentProvider):
def __init__(self):
generic.TorrentProvider.__init__(self, 'BitMeTV')
self.url_base = 'http://www.bitmetv.org/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'links.php',
'search': self.url_base + 'browse.php?%s&search=%s',
'get': self.url_base + '%s'}
self.categories = {'shows': 0, 'anime': 86} # exclusively one cat per key
self.url = self.urls['config_provider_home_uri']
self.digest, self.minseed, self.minleech = 3 * [None]
self.cache = BitmetvCache(self)
def _authorised(self, **kwargs):
return super(BitmetvProvider, self)._authorised(
logged_in=(lambda x=None: (None is x or 'Other Links' in x) and self.has_all_cookies() and
self.session.cookies['uid'] in self.digest and self.session.cookies['pass'] in self.digest),
failed_msg=(lambda x=None: u'Invalid cookie details for %s. Check settings'))
def _search_provider(self, search_params, **kwargs):
results = []
if not self._authorised():
return results
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
category = 'cat=%s' % self.categories[
(mode in ['Season', 'Episode'] and self.show and self.show.is_anime) and 'anime' or 'shows']
search_url = self.urls['search'] % (category, search_string)
html = self.get_url(search_url)
cnt = len(items[mode])
try:
if not html or self._has_no_results(html):
raise generic.HaltParseException
with BS4Parser(html, features=['html5lib', 'permissive'], attr='cellpadding="5"') as soup:
torrent_table = soup.find('table', attrs={'cellpadding': 5})
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
if 2 > len(torrent_rows):
raise generic.HaltParseException
for tr in torrent_rows[1:]:
try:
seeders, leechers, size = [tryInt(n, n) for n in [
(tr.find_all('td')[x].get_text().strip()) for x in (-3, -2, -5)]]
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', href=rc['info'])
title = 'title' in info.attrs and info.attrs['title'] or info.get_text().strip()
download_url = self.urls['get'] % tr.find('a', href=rc['get']).get('href')
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_search(mode, len(items[mode]) - cnt, search_url)
self._sort_seeders(mode, items)
results = list(set(results + items[mode]))
return results
@staticmethod
def ui_string(key):
return 'bitmetv_digest' == key and 'use... \'uid=xx; pass=yy\'' or ''
class BitmetvCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.update_freq = 7 # cache update frequency
def _cache_data(self):
return self.provider.cache_data()
provider = BitmetvProvider()

71
sickbeard/providers/bitsoup.py
View file

@ -16,12 +16,12 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -33,51 +33,30 @@ class BitSoupProvider(generic.TorrentProvider):
self.url_base = 'https://www.bitsoup.me/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'takelogin.php',
'search': self.url_base + 'browse.php?search=%s%s',
'search': self.url_base + 'browse.php?search=%s&%s&incldead=0&blah=0',
'get': self.url_base + '%s'}
self.categories = '&c42=1&c45=1&c49=1&c7=1&incldead=0&blah=0'
self.categories = {'shows': [42, 45, 49, 32, 7], 'anime': [23]}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.cache = BitSoupCache(self)
def _do_login(self):
logged_in = lambda: 'uid' in self.session.cookies and 'pass' in self.session.cookies
if logged_in():
return True
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'ssl': 'yes'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and 'Username or password incorrect' in response:
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['search'] % (search_string, self._categories_string(mode))
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_url = self.urls['search'] % (search_string, self.categories)
html = self.get_url(search_url)
cnt = len(items[mode])
@ -85,7 +64,7 @@ class BitSoupProvider(generic.TorrentProvider):
if not html or self._has_no_results(html):
raise generic.HaltParseException
with BS4Parser(html, 'html.parser') as soup:
with BS4Parser(html, 'html.parser', attr='class="koptekst"') as soup:
torrent_table = soup.find('table', attrs={'class': 'koptekst'})
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
@ -94,40 +73,36 @@ class BitSoupProvider(generic.TorrentProvider):
for tr in torrent_rows[1:]:
try:
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-3, -2)]
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers, size = [tryInt(n, n) for n in [
(tr.find_all('td')[x].get_text().strip()) for x in (-3, -2, -5)]]
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', href=rc['info'])
title = info.get_text().strip()
download_url = self.urls['get'] % str(tr.find('a', href=rc['get'])['href']).lstrip('/')
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# for each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _episode_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, sep_date='|', use_or=False)
return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='|', **kwargs)
class BitSoupCache(tvcache.TVCache):
@ -135,11 +110,11 @@ class BitSoupCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = BitSoupProvider()

274
sickbeard/providers/btn.py
View file

@ -15,16 +15,19 @@
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import datetime
import time
import math
import socket
import re
import time
from . import generic
from sickbeard import classes, scene_exceptions, logger, tvcache
from sickbeard.helpers import sanitizeSceneName
from sickbeard.exceptions import ex, AuthException
from lib import jsonrpclib
from sickbeard import helpers, logger, scene_exceptions, tvcache
from sickbeard.helpers import tryInt
try:
import json
except ImportError:
from lib import simplejson as json
import random
class BTNProvider(generic.TorrentProvider):
@ -33,110 +36,101 @@ class BTNProvider(generic.TorrentProvider):
generic.TorrentProvider.__init__(self, 'BTN')
self.url_base = 'https://broadcasthe.net'
self.url_api = 'http://api.btnapps.net'
self.url_api = 'https://api.btnapps.net'
self.proper_search_terms = ['%.proper.%', '%.repack.%']
self.url = self.url_base
self.api_key = None
self.api_key, self.minseed, self.minleech = 3 * [None]
self.reject_m2ts = False
self.session.headers = {'Content-Type': 'application/json-rpc'}
self.cache = BTNCache(self)
def _check_auth_from_data(self, data_json):
def _authorised(self, **kwargs):
if data_json is None:
return self._check_auth()
return self._check_auth()
if 'api-error' not in data_json:
return True
logger.log(u'Incorrect authentication credentials for %s : %s' % (self.name, data_json['api-error']),
logger.DEBUG)
raise AuthException('Your authentication credentials for %s are incorrect, check your config.' % self.name)
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, age=0, **kwargs):
self._check_auth()
params = {}
if search_params:
params.update(search_params)
if age:
params['age'] = '<=%i' % age # age in seconds
results = []
data_json = self._api_call(params)
if not (data_json and self._check_auth_from_data(data_json)):
self._log_result('rpc search', 0, self.name)
else:
for mode in search_params.keys():
for search_param in search_params[mode]:
found_torrents = {} if 'torrents' not in data_json else data_json['torrents']
params = {}
if 'Propers' == mode:
params.update({'release': search_param})
age = 4 * 24 * 60 * 60
else:
search_param and params.update(search_param)
age and params.update(dict(age='<=%i' % age)) # age in seconds
# We got something, we know the API sends max 1000 results at a time.
# See if there are more than 1000 results for our query, if not we
# keep requesting until we've got everything.
# max 150 requests per hour so limit at that. Scan every 15 minutes. 60 / 15 = 4.
max_pages = 150
results_per_page = 1000
json_rpc = (lambda param_dct, items_per_page=1000, offset=0:
'{"jsonrpc": "2.0", "id": "%s", "method": "getTorrents", "params": ["%s", %s, %s, %s]}' %
(''.join(random.sample('abcdefghijklmnopqrstuvwxyz0123456789', 8)),
self.api_key, json.dumps(param_dct), items_per_page, offset))
if 'results' in data_json and int(data_json['results']) >= results_per_page:
pages_needed = int(math.ceil(int(data_json['results']) / results_per_page))
if pages_needed > max_pages:
pages_needed = max_pages
try:
response = helpers.getURL(self.url_api, post_data=json_rpc(params), session=self.session, json=True)
error_text = response['error']['message']
logger.log(('Call Limit' in error_text and u'Action aborted because the %(prov)s 150 calls/hr limit was reached' or
u'Action prematurely ended. %(prov)s server error response = %(desc)s') % {'prov': self.name, 'desc': error_text}, logger.WARNING)
return results
except:
data_json = response and 'result' in response and response['result'] or {}
# +1 because range(1,4) = 1, 2, 3
for page in range(1, pages_needed + 1):
data_json = self._api_call(params, results_per_page, page * results_per_page)
# Note that this these are individual requests and might time out individually. This would result in 'gaps'
# in the results. There is no way to fix this though.
if 'torrents' in data_json:
found_torrents.update(data_json['torrents'])
if data_json:
cnt = len(results)
for torrentid, torrent_info in found_torrents.iteritems():
title, url = self._get_title_and_url(torrent_info)
if title and url:
results.append(torrent_info)
self._log_result('search', len(results) - cnt, self.name + ' JSON-RPC API')
found_torrents = {} if 'torrents' not in data_json else data_json['torrents']
# We got something, we know the API sends max 1000 results at a time.
# See if there are more than 1000 results for our query, if not we
# keep requesting until we've got everything.
# max 150 requests per hour so limit at that. Scan every 15 minutes. 60 / 15 = 4.
max_pages = 5 # 150 was the old value and impractical
results_per_page = 1000
if 'results' in data_json and int(data_json['results']) >= results_per_page:
pages_needed = int(math.ceil(int(data_json['results']) / results_per_page))
if pages_needed > max_pages:
pages_needed = max_pages
# +1 because range(1,4) = 1, 2, 3
for page in range(1, pages_needed + 1):
try:
response = helpers.getURL(self.url_api, json=True, session=self.session,
post_data=json_rpc(params, results_per_page, page * results_per_page))
error_text = response['error']['message']
logger.log(('Call Limit' in error_text and u'Action prematurely ended because the %(prov)s 150 calls/hr limit was reached' or
u'Action prematurely ended. %(prov)s server error response = %(desc)s') % {'prov': self.name, 'desc': error_text}, logger.WARNING)
return results
except:
data_json = response and 'result' in response and response['result'] or {}
# Note that this these are individual requests and might time out individually. This would result in 'gaps'
# in the results. There is no way to fix this though.
if 'torrents' in data_json:
found_torrents.update(data_json['torrents'])
cnt = len(results)
for torrentid, torrent_info in found_torrents.iteritems():
seeders, leechers = [tryInt(n) for n in torrent_info.get('Seeders'), torrent_info.get('Leechers')]
if self._peers_fail(mode, seeders, leechers) or \
self.reject_m2ts and re.match(r'(?i)m2?ts', torrent_info.get('Container', '')):
continue
title, url = self._title_and_url(torrent_info)
if title and url:
results.append(torrent_info)
self._log_search(mode, len(results) - cnt, self.name)
return results
def _api_call(self, params=None, results_per_page=1000, offset=0):
if None is params:
params = {}
logger.log(u'Searching with parameters: ' + str(params), logger.DEBUG)
parsed_json = {}
server = jsonrpclib.Server(self.url_api)
try:
parsed_json = server.getTorrents(self.api_key, params, int(results_per_page), int(offset))
except jsonrpclib.jsonrpc.ProtocolError as error:
if 'Call Limit' in error.message:
logger.log(u'Request ignored because the %s 150 calls/hr limit was reached' % self.name, logger.WARNING)
else:
logger.log(u'JSON-RPC protocol error while accessing %s: %s' % (self.name, ex(error)), logger.ERROR)
return {'api-error': ex(error)}
except socket.timeout:
logger.log(u'Timeout while accessing ' + self.name, logger.WARNING)
except socket.error as error:
# timeouts are sometimes thrown as socket errors
logger.log(u'Socket error while accessing %s: %s' % (self.name, error[1]), logger.ERROR)
except Exception as error:
errorstring = str(error)
if errorstring.startswith('<') and errorstring.endswith('>'):
errorstring = errorstring[1:-1]
logger.log(u'Error while accessing %s: %s' % (self.name, errorstring), logger.ERROR)
return parsed_json
def _get_title_and_url(self, data_json):
def _title_and_url(self, data_json):
# The BTN API gives a lot of information in response,
# however SickGear is built mostly around Scene or
@ -165,45 +159,44 @@ class BTNProvider(generic.TorrentProvider):
return title, url
def _get_season_search_strings(self, ep_obj, **kwargs):
def _season_strings(self, ep_obj, **kwargs):
search_params = []
current_params = {'category': 'Season'}
base_params = {'category': 'Season'}
# Search for entire seasons: no need to do special things for air by date or sports shows
if ep_obj.show.air_by_date or ep_obj.show.is_sports:
# Search for the year of the air by date show
current_params['name'] = str(ep_obj.airdate).split('-')[0]
base_params['name'] = str(ep_obj.airdate).split('-')[0]
elif ep_obj.show.is_anime:
current_params['name'] = '%s' % ep_obj.scene_absolute_number
base_params['name'] = '%s' % ep_obj.scene_absolute_number
else:
current_params['name'] = 'Season %s' % (ep_obj.season, ep_obj.scene_season)[bool(ep_obj.show.is_scene)]
base_params['name'] = 'Season %s' % (ep_obj.season, ep_obj.scene_season)[bool(ep_obj.show.is_scene)]
# search
if 1 == ep_obj.show.indexer:
current_params['tvdb'] = ep_obj.show.indexerid
search_params.append(current_params)
elif 2 == ep_obj.show.indexer:
current_params['tvrage'] = ep_obj.show.indexerid
search_params.append(current_params)
base_params['tvdb'] = ep_obj.show.indexerid
search_params.append(base_params)
# elif 2 == ep_obj.show.indexer:
# current_params['tvrage'] = ep_obj.show.indexerid
# search_params.append(current_params)
else:
name_exceptions = list(
set([sanitizeSceneName(a) for a in scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
set([helpers.sanitizeSceneName(a) for a in
scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
for name in name_exceptions:
# Search by name if we don't have tvdb or tvrage id
cur_return = current_params.copy()
cur_return['series'] = name
search_params.append(cur_return)
series_param = {'series': name}
series_param.update(base_params)
search_params.append(series_param)
return search_params
return [dict({'Season': search_params})]
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
def _episode_strings(self, ep_obj, **kwargs):
if not ep_obj:
return [{}]
to_return = []
search_params = {'category': 'Episode'}
search_params = []
base_params = {'category': 'Episode'}
# episode
if ep_obj.show.air_by_date or ep_obj.show.is_sports:
@ -211,54 +204,35 @@ class BTNProvider(generic.TorrentProvider):
# BTN uses dots in dates, we just search for the date since that
# combined with the series identifier should result in just one episode
search_params['name'] = date_str.replace('-', '.')
base_params['name'] = date_str.replace('-', '.')
elif ep_obj.show.is_anime:
search_params['name'] = '%s' % ep_obj.scene_absolute_number
base_params['name'] = '%s' % ep_obj.scene_absolute_number
else:
# Do a general name search for the episode, formatted like SXXEYY
season, episode = ((ep_obj.season, ep_obj.episode),
(ep_obj.scene_season, ep_obj.scene_episode))[bool(ep_obj.show.is_scene)]
search_params['name'] = 'S%02dE%02d' % (season, episode)
base_params['name'] = 'S%02dE%02d' % (season, episode)
# search
if 1 == ep_obj.show.indexer:
search_params['tvdb'] = ep_obj.show.indexerid
to_return.append(search_params)
elif 2 == ep_obj.show.indexer:
search_params['tvrage'] = ep_obj.show.indexerid
to_return.append(search_params)
base_params['tvdb'] = ep_obj.show.indexerid
search_params.append(base_params)
# elif 2 == ep_obj.show.indexer:
# search_params['tvrage'] = ep_obj.show.indexerid
# to_return.append(search_params)
else:
# add new query string for every exception
name_exceptions = list(
set([sanitizeSceneName(a) for a in scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
for cur_exception in name_exceptions:
cur_return = search_params.copy()
cur_return['series'] = cur_exception
to_return.append(cur_return)
set([helpers.sanitizeSceneName(a) for a in
scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
for name in name_exceptions:
series_param = {'series': name}
series_param.update(base_params)
search_params.append(series_param)
return to_return
return [dict({'Episode': search_params})]
def find_propers(self, search_date=None):
results = []
search_terms = ['%.proper.%', '%.repack.%']
for term in search_terms:
for item in self._do_search({'release': term}, age=4 * 24 * 60 * 60):
if item['Time']:
try:
result_date = datetime.datetime.fromtimestamp(float(item['Time']))
except TypeError:
continue
if not search_date or result_date > search_date:
title, url = self._get_title_and_url(item)
results.append(classes.Proper(title, url, result_date, self.show))
return results
def get_cache_data(self, **kwargs):
def cache_data(self, **kwargs):
# Get the torrents uploaded since last check.
seconds_since_last_update = int(math.ceil(time.time() - time.mktime(kwargs['age'])))
@ -275,7 +249,7 @@ class BTNProvider(generic.TorrentProvider):
% self.name, logger.WARNING)
seconds_since_last_update = 86400
return self._do_search(search_params=None, age=seconds_since_last_update)
return self._search_provider(dict({'Cache': ['']}), age=seconds_since_last_update)
class BTNCache(tvcache.TVCache):
@ -283,11 +257,11 @@ class BTNCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 15 # cache update frequency
self.update_freq = 15 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data(age=self._getLastUpdate().timetuple(), min_time=self.minTime)
return self.provider.cache_data(age=self._getLastUpdate().timetuple(), min_time=self.update_freq)
provider = BTNProvider()

137
sickbeard/providers/freshontv.py Normal file
View file

@ -0,0 +1,137 @@
# coding=utf-8
#
# This file is part of SickGear.
#
# SickGear is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# SickGear is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import traceback
from . import generic
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
class FreshOnTVProvider(generic.TorrentProvider):
def __init__(self):
generic.TorrentProvider.__init__(self, 'FreshOnTV')
self.url_base = 'https://freshon.tv/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'login.php?action=makelogin',
'search': self.url_base + 'browse.php?incldead=%s&words=0&cat=0&search=%s',
'get': self.url_base + '%s'}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = FreshOnTVCache(self)
def _authorised(self, **kwargs):
return super(FreshOnTVProvider, self)._authorised(
post_params={'login': 'Do it!'},
failed_msg=(lambda x=None: 'DDoS protection by CloudFlare' in x and
u'Unable to login to %s due to CloudFlare DDoS javascript check' or
'Username does not exist' in x and
u'Invalid username or password for %s. Check settings' or
u'Failed to authenticate or parse a response from %s, abort provider'))
def _search_provider(self, search_params, **kwargs):
results = []
if not self._authorised():
return results
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
freeleech = (0, 3)[self.freeleech]
rc = dict((k, re.compile('(?i)' + v))
for (k, v) in {'info': 'detail', 'get': 'download', 'name': '_name'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string, search_url = self._title_and_url((
isinstance(search_string, unicode) and unidecode(search_string) or search_string,
self.urls['search'] % (freeleech, search_string)))
# returns top 15 results by default, expandable in user profile to 100
html = self.get_url(search_url)
cnt = len(items[mode])
try:
if not html or self._has_no_results(html):
raise generic.HaltParseException
with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
torrent_table = soup.find('table', attrs={'class': 'frame'})
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
if 2 > len(torrent_rows):
raise generic.HaltParseException
for tr in torrent_rows[1:]:
try:
if tr.find('img', alt='Nuked'):
continue
seeders, leechers, size = [tryInt(n, n) for n in [
(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1, -4)]]
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', href=rc['info'], attrs={'class': rc['name']})
title = 'title' in info.attrs and info.attrs['title'] or info.get_text().strip()
download_url = self.urls['get'] % str(tr.find('a', href=rc['get'])['href']).lstrip('/')
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_search(mode, len(items[mode]) - cnt, search_url)
self._sort_seeders(mode, items)
results = list(set(results + items[mode]))
return results
def _get_episode_search_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='|', **kwargs)
class FreshOnTVCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.update_freq = 20
def _cache_data(self):
return self.provider.cache_data()
provider = FreshOnTVProvider()

75
sickbeard/providers/funfile.py
View file

@ -16,12 +16,12 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -33,51 +33,39 @@ class FunFileProvider(generic.TorrentProvider):
self.url_base = 'https://www.funfile.org/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'takelogin.php',
'search': self.url_base + 'browse.php?%ssearch=%s',
'search': self.url_base + 'browse.php?%s&search=%s&incldead=0&showspam=1&',
'get': self.url_base + '%s'}
self.categories = 'cat=7&incldead=0&s_title=1&showspam=1&'
self.categories = {'shows': [7], 'anime': [44]}
self.url = self.urls['config_provider_home_uri']
self.url_timeout = 90
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.cache = FunFileCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: None is not self.session.cookies.get('uid', domain='.funfile.org') and None is not self.session.cookies.get('pass', domain='.funfile.org')
if logged_in():
return True
return super(FunFileProvider, self)._authorised(
logged_in=(lambda x=None: None is not self.session.cookies.get('uid', domain='.funfile.org') and
None is not self.session.cookies.get('pass', domain='.funfile.org')),
post_params={'login': 'Login', 'returnto': '/'}, timeout=self.url_timeout)
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'submit': 'Log in'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session, timeout=self.url_timeout)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and 'Username or password incorrect' in response:
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download',
'cats': 'cat=(?:7)'}.items())
'cats': 'cat=(?:%s)' % self._categories_string(template='', delimiter='|')
}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['search'] % (self._categories_string(mode), search_string)
search_url = self.urls['search'] % (self.categories, search_string)
html = self.get_url(search_url, timeout=self.url_timeout)
cnt = len(items[mode])
@ -98,53 +86,44 @@ class FunFileProvider(generic.TorrentProvider):
if not info:
continue
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
if None is tr.find('a', href=rc['cats'])\
or ('Cache' != mode and (seeders < self.minseed or leechers < self.minleech)):
seeders, leechers, size = [tryInt(n, n) for n in [
(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1, -4)]]
if None is tr.find('a', href=rc['cats']) or self._peers_fail(mode, seeders, leechers):
continue
title = 'title' in info.attrs and info.attrs['title'] or info.get_text().strip()
download_url = self.urls['get'] % tr.find('a', href=rc['get']).get('href')
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except (generic.HaltParseException, AttributeError):
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, use_or=False)
class FunFileCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 15 # cache update frequency
self.update_freq = 15 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = FunFileProvider()

428
sickbeard/providers/generic.py
View file

@ -19,24 +19,26 @@
from __future__ import with_statement
import time
import datetime
import itertools
import math
import os
import re
import itertools
import time
from base64 import b16encode, b32decode
import sickbeard
import requests
import requests.cookies
from hachoir_parser import guessParser
from hachoir_core.stream import FileInputStream
from sickbeard import helpers, classes, logger, db, tvcache, encodingKludge as ek
from sickbeard.common import Quality, MULTI_EP_RESULT, SEASON_RESULT, USER_AGENT
from sickbeard.exceptions import SickBeardException, AuthException, ex
from sickbeard.helpers import maybe_plural, _remove_file_failed as remove_file_failed
from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
from sickbeard.show_name_helpers import allPossibleShowNames
from hachoir_parser import guessParser
from hachoir_core.stream import FileInputStream
class HaltParseException(SickBeardException):
@ -53,6 +55,8 @@ class GenericProvider:
self.name = name
self.supportsBacklog = supports_backlog
self.anime_only = anime_only
if anime_only:
self.proper_search_terms = 'v1|v2|v3|v4|v5'
self.url = ''
self.show = None
@ -62,6 +66,7 @@ class GenericProvider:
self.enabled = False
self.enable_recentsearch = False
self.enable_backlog = False
self.categories = None
self.cache = tvcache.TVCache(self)
@ -89,10 +94,10 @@ class GenericProvider:
return '%s.png' % ('newznab', default_name[0])[any(default_name)]
def _check_auth(self):
def _authorised(self):
return True
def _do_login(self):
def _check_auth(self):
return True
def is_active(self):
@ -133,7 +138,7 @@ class GenericProvider:
"""
# check for auth
if not self._do_login():
if not self._authorised():
return
return helpers.getURL(url, post_data=post_data, params=params, headers=self.headers, timeout=timeout,
@ -145,7 +150,7 @@ class GenericProvider:
"""
# check for auth
if not self._do_login():
if not self._authorised():
return False
if GenericProvider.TORRENT == self.providerType:
@ -239,25 +244,25 @@ class GenericProvider:
def get_quality(self, item, anime=False):
"""
Figures out the quality of the given RSS item node
item: An elementtree.ElementTree element representing the <item> tag of the RSS feed
Returns a Quality value obtained from the node's data
Returns a Quality value obtained from the node's data
"""
(title, url) = self._get_title_and_url(item) # @UnusedVariable
(title, url) = self._title_and_url(item) # @UnusedVariable
quality = Quality.sceneQuality(title, anime)
return quality
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, search_mode='eponly', epcount=0, age=0):
return []
def _get_season_search_strings(self, episode):
def _season_strings(self, episode):
return []
def _get_episode_search_strings(self, *args, **kwargs):
def _episode_strings(self, *args, **kwargs):
return []
def _get_title_and_url(self, item):
def _title_and_url(self, item):
"""
Retrieves the title and URL data from the item
@ -267,25 +272,14 @@ class GenericProvider:
"""
title, url = None, None
try:
if isinstance(item, tuple):
title = item[0]
url = item[1]
else:
if 'title' in item:
title = item.title
if 'link' in item:
url = item.link
title, url = isinstance(item, tuple) and (item[0], item[1]) or \
(item.get('title', None), item.get('link', None))
except Exception:
pass
if title:
title = re.sub(r'\s+', '.', u'%s' % title)
if url:
url = str(url).replace('&amp;', '&')
title = title and re.sub(r'\s+', '.', u'%s' % title)
url = url and str(url).replace('&amp;', '&')
return title, url
@ -310,21 +304,21 @@ class GenericProvider:
# found result, search next episode
continue
# skip if season already searched
if 1 < len(episodes) and ep_obj.scene_season == searched_scene_season:
continue
# mark season searched for season pack searches so we can skip later on
searched_scene_season = ep_obj.scene_season
if 'sponly' == search_mode:
# get season search results
for curString in self._get_season_search_strings(ep_obj):
item_list += self._do_search(curString, search_mode, len(episodes))
# skip if season already searched
if 1 < len(episodes) and searched_scene_season == ep_obj.scene_season:
continue
searched_scene_season = ep_obj.scene_season
# get season search params
search_params = self._season_strings(ep_obj)
else:
# get single episode search results
for curString in self._get_episode_search_strings(ep_obj):
item_list += self._do_search(curString, 'eponly', len(episodes))
# get single episode search params
search_params = self._episode_strings(ep_obj)
for cur_param in search_params:
item_list += self._search_provider(cur_param, search_mode=search_mode, epcount=len(episodes))
# if we found what we needed already from cache then return results and exit
if len(results) == len(episodes):
@ -349,18 +343,18 @@ class GenericProvider:
# filter results
cl = []
parser = NameParser(False, convert=True)
for item in item_list:
(title, url) = self._get_title_and_url(item)
(title, url) = self._title_and_url(item)
# parse the file name
try:
parser = NameParser(False, convert=True)
parse_result = parser.parse(title)
except InvalidNameException:
logger.log(u'Unable to parse the filename ' + title + ' into a valid episode', logger.DEBUG)
logger.log(u'Unable to parse the filename %s into a valid episode' % title, logger.DEBUG)
continue
except InvalidShowException:
logger.log(u'No show name or scene exception matched the parsed filename ' + title, logger.DEBUG)
logger.log(u'No match for search criteria in the parsed filename ' + title, logger.DEBUG)
continue
show_obj = parse_result.show
@ -372,28 +366,28 @@ class GenericProvider:
if not (show_obj.air_by_date or show_obj.is_sports):
if 'sponly' == search_mode:
if len(parse_result.episode_numbers):
logger.log(u'This is supposed to be a season pack search but the result ' + title
+ u' is not a valid season pack, skipping it', logger.DEBUG)
logger.log(u'This is supposed to be a season pack search but the result ' + title +
u' is not a valid season pack, skipping it', logger.DEBUG)
add_cache_entry = True
if len(parse_result.episode_numbers)\
and (parse_result.season_number not in set([ep.season for ep in episodes])
or not [ep for ep in episodes if ep.scene_episode in parse_result.episode_numbers]):
logger.log(u'The result ' + title + u' doesn\'t seem to be a valid episode that we are trying'
+ u' to snatch, ignoring', logger.DEBUG)
and (parse_result.season_number not in set([ep.season for ep in episodes]) or not [
ep for ep in episodes if ep.scene_episode in parse_result.episode_numbers]):
logger.log(u'The result ' + title + u' doesn\'t seem to be a valid episode that we are trying' +
u' to snatch, ignoring', logger.DEBUG)
add_cache_entry = True
else:
if not len(parse_result.episode_numbers)\
and parse_result.season_number\
and not [ep for ep in episodes
if ep.season == parse_result.season_number
and ep.episode in parse_result.episode_numbers]:
logger.log(u'The result ' + title + u' doesn\'t seem to be a valid season that we are trying'
+ u' to snatch, ignoring', logger.DEBUG)
if ep.season == parse_result.season_number and
ep.episode in parse_result.episode_numbers]:
logger.log(u'The result ' + title + u' doesn\'t seem to be a valid season that we are trying' +
u' to snatch, ignoring', logger.DEBUG)
add_cache_entry = True
elif len(parse_result.episode_numbers) and not [ep for ep in episodes if
ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers]:
logger.log(u'The result ' + title + ' doesn\'t seem to be a valid episode that we are trying'
+ u' to snatch, ignoring', logger.DEBUG)
logger.log(u'The result ' + title + ' doesn\'t seem to be a valid episode that we are trying' +
u' to snatch, ignoring', logger.DEBUG)
add_cache_entry = True
if not add_cache_entry:
@ -402,8 +396,8 @@ class GenericProvider:
actual_episodes = parse_result.episode_numbers
else:
if not parse_result.is_air_by_date:
logger.log(u'This is supposed to be a date search but the result ' + title
+ u' didn\'t parse as one, skipping it', logger.DEBUG)
logger.log(u'This is supposed to be a date search but the result ' + title +
u' didn\'t parse as one, skipping it', logger.DEBUG)
add_cache_entry = True
else:
airdate = parse_result.air_date.toordinal()
@ -412,8 +406,8 @@ class GenericProvider:
[show_obj.indexerid, airdate])
if 1 != len(sql_results):
logger.log(u'Tried to look up the date for the episode ' + title + ' but the database didn\'t'
+ u' give proper results, skipping it', logger.WARNING)
logger.log(u'Tried to look up the date for the episode ' + title + ' but the database didn\'t' +
u' give proper results, skipping it', logger.WARNING)
add_cache_entry = True
if not add_cache_entry:
@ -462,8 +456,8 @@ class GenericProvider:
logger.log(u'Single episode result.', logger.DEBUG)
elif 1 < len(ep_obj):
ep_num = MULTI_EP_RESULT
logger.log(u'Separating multi-episode result to check for later - result contains episodes: '
+ str(parse_result.episode_numbers), logger.DEBUG)
logger.log(u'Separating multi-episode result to check for later - result contains episodes: ' +
str(parse_result.episode_numbers), logger.DEBUG)
elif 0 == len(ep_obj):
ep_num = SEASON_RESULT
logger.log(u'Separating full season result to check for later', logger.DEBUG)
@ -480,7 +474,7 @@ class GenericProvider:
return results
def find_propers(self, search_date=None):
def find_propers(self, search_date=None, **kwargs):
results = self.cache.listPropers(search_date)
@ -494,16 +488,25 @@ class GenericProvider:
"""
return ''
@staticmethod
def _log_result(mode='cache', count=0, url='url missing'):
def _log_search(self, mode='Cache', count=0, url='url missing'):
"""
Simple function to log the result of a search
Simple function to log the result of a search types except propers
:param count: count of successfully processed items
:param url: source url of item(s)
"""
mode = mode.lower()
logger.log(u'%s in response from %s' % (('No %s items' % mode,
'%s %s item%s' % (count, mode, maybe_plural(count)))[0 < count], url))
if 'Propers' != mode:
self.log_result(mode, count, url)
def log_result(self, mode='Cache', count=0, url='url missing'):
"""
Simple function to log the result of any search
:param count: count of successfully processed items
:param url: source url of item(s)
"""
str1, thing, str3 = (('', '%s item' % mode.lower(), ''), (' usable', 'proper', ' found'))['Propers' == mode]
logger.log(u'%s %s in response from %s' % (('No' + str1, count)[0 < count], (
'%s%s%s%s' % (('', 'freeleech ')[getattr(self, 'freeleech', False)], thing, maybe_plural(count), str3)),
re.sub('(\s)\s+', r'\1', url)))
def check_auth_cookie(self):
@ -529,8 +532,34 @@ class GenericProvider:
return False, 'Cookies not correctly formatted key=value pairs e.g. uid=xx;pass=yy)'
def has_all_cookies(self, cookies=None, pre=''):
class NZBProvider(GenericProvider):
cookies = cookies or ['uid', 'pass']
return False not in ['%s%s' % (pre, item) in self.session.cookies for item in ([cookies], cookies)[isinstance(cookies, list)]]
def _categories_string(self, mode='Cache', template='c%s=1', delimiter='&'):
return delimiter.join([('%s', template)[any(template)] % c for c in sorted(self.categories['shows'] + (
[], [] if 'anime' not in self.categories else self.categories['anime'])[
('Cache' == mode and helpers.has_anime()) or ((mode in ['Season', 'Episode']) and self.show and self.show.is_anime)])])
@staticmethod
def _bytesizer(size_dim=''):
try:
value = float('.'.join(re.findall('(?i)(\d+)(?:[\.,](\d+))?', size_dim)[0]))
except TypeError:
return size_dim
except IndexError:
return None
try:
value *= 1024 ** ['b', 'k', 'm', 'g', 't'].index(re.findall('(t|g|m|k)[i]?b', size_dim.lower())[0])
except IndexError:
pass
return int(math.ceil(value))
class NZBProvider(object, GenericProvider):
def __init__(self, name, supports_backlog=True, anime_only=False):
GenericProvider.__init__(self, name, supports_backlog, anime_only)
@ -543,7 +572,7 @@ class NZBProvider(GenericProvider):
def maybe_apikey(self):
if hasattr(self, 'needs_auth'):
if hasattr(self, 'needs_auth') and self.needs_auth:
if hasattr(self, 'key') and 0 < len(self.key):
return self.key
if hasattr(self, 'api_key') and 0 < len(self.api_key):
@ -562,7 +591,7 @@ class NZBProvider(GenericProvider):
return GenericProvider._check_auth(self)
def _find_propers(self, search_date=None):
def find_propers(self, search_date=None, shows=None, anime=None, **kwargs):
cache_results = self.cache.listPropers(search_date)
results = [classes.Proper(x['name'], x['url'], datetime.datetime.fromtimestamp(x['time']), self.show) for x in
@ -573,11 +602,20 @@ class NZBProvider(GenericProvider):
term_items_found = False
do_search_alt = False
search_terms = ['.proper.', '.repack.']
proper_check = re.compile(r'(?i)\b(proper)|(repack)\b')
search_terms = []
regex = []
if shows:
search_terms += ['.proper.', '.repack.']
regex += ['proper|repack']
proper_check = re.compile(r'(?i)(\b%s\b)' % '|'.join(regex))
if anime:
terms = 'v1|v2|v3|v4|v5'
search_terms += [terms]
regex += [terms]
proper_check = re.compile(r'(?i)(%s)' % '|'.join(regex))
while index < len(search_terms):
search_params = {'q': search_terms[index]}
search_params = {'q': search_terms[index], 'maxage': 4}
if alt_search:
if do_search_alt:
@ -595,9 +633,9 @@ class NZBProvider(GenericProvider):
else:
index += 1
for item in self._do_search(search_params, age=4):
for item in self._search_provider({'Propers': [search_params]}):
(title, url) = self._get_title_and_url(item)
(title, url) = self._title_and_url(item)
if not proper_check.search(title):
continue
@ -620,8 +658,13 @@ class NZBProvider(GenericProvider):
return results
def cache_data(self, *args, **kwargs):
class TorrentProvider(GenericProvider):
search_params = {'Cache': [{}]}
return self._search_provider(search_params)
class TorrentProvider(object, GenericProvider):
def __init__(self, name, supports_backlog=True, anime_only=False):
GenericProvider.__init__(self, name, supports_backlog, anime_only)
@ -639,12 +682,21 @@ class TorrentProvider(GenericProvider):
return self._seed_ratio
@staticmethod
def _sort_seeders(mode, items):
mode in ['Season', 'Episode'] and items[mode].sort(key=lambda tup: tup[2], reverse=True)
def _peers_fail(self, mode, seeders=0, leechers=0):
return 'Cache' != mode and (seeders < getattr(self, 'minseed', 0) or leechers < getattr(self, 'minleech', 0))
def get_quality(self, item, anime=False):
if isinstance(item, tuple):
name = item[0]
elif isinstance(item, dict):
name, url = self._get_title_and_url(item)
name, url = self._title_and_url(item)
else:
name = item.title
return Quality.sceneQuality(name, anime)
@ -664,51 +716,59 @@ class TorrentProvider(GenericProvider):
Quality.FULLHDBLURAY: '1080p Bluray x264'
}.get(quality, '')
def _get_season_search_strings(self, ep_obj, detail_only=False, scene=True):
if ep_obj.show.air_by_date or ep_obj.show.is_sports:
ep_detail = str(ep_obj.airdate).split('-')[0]
elif ep_obj.show.is_anime:
ep_detail = ep_obj.scene_absolute_number
else:
ep_detail = 'S%02d' % int((ep_obj.season, ep_obj.scene_season)[bool(ep_obj.show.is_scene)])
detail = ({}, {'Season_only': [ep_detail]})[detail_only and not self.show.is_sports and not self.show.is_anime]
return [dict({'Season': self._build_search_strings(ep_detail, scene)}.items() + detail.items())]
def _get_episode_search_strings(self, ep_obj, add_string='', detail_only=False, scene=True, sep_date=' ', use_or=True):
def _season_strings(self, ep_obj, detail_only=False, scene=True, prefix='', **kwargs):
if not ep_obj:
return []
if self.show.air_by_date or self.show.is_sports:
ep_detail = str(ep_obj.airdate).replace('-', sep_date)
if self.show.is_sports:
month = ep_obj.airdate.strftime('%b')
ep_detail = ([ep_detail] + [month], '%s|%s' % (ep_detail, month))[use_or]
elif self.show.is_anime:
ep_detail = ep_obj.scene_absolute_number
else:
season, episode = ((ep_obj.season, ep_obj.episode),
(ep_obj.scene_season, ep_obj.scene_episode))[bool(ep_obj.show.is_scene)]
ep_dict = {'seasonnumber': season, 'episodenumber': episode}
ep_detail = sickbeard.config.naming_ep_type[2] % ep_dict
append = (add_string, '')[self.show.is_anime]
detail = ({}, {'Episode_only': [ep_detail]})[detail_only and not self.show.is_sports and not self.show.is_anime]
return [dict({'Episode': self._build_search_strings(ep_detail, scene, append)}.items() + detail.items())]
show = ep_obj.show
ep_dict = self._ep_dict(ep_obj)
sp_detail = (show.air_by_date or show.is_sports) and str(ep_obj.airdate).split('-')[0] or \
(show.is_anime and ep_obj.scene_absolute_number or
'S%(seasonnumber)02d' % ep_dict if 'sp_detail' not in kwargs.keys() else kwargs['sp_detail'](ep_dict))
sp_detail = ([sp_detail], sp_detail)[isinstance(sp_detail, list)]
detail = ({}, {'Season_only': sp_detail})[detail_only and not self.show.is_sports and not self.show.is_anime]
return [dict({'Season': self._build_search_strings(sp_detail, scene, prefix)}.items() + detail.items())]
def _build_search_strings(self, ep_detail, process_name=True, append=''):
def _episode_strings(self, ep_obj, detail_only=False, scene=True, prefix='', sep_date=' ', date_or=False, **kwargs):
if not ep_obj:
return []
show = ep_obj.show
if show.air_by_date or show.is_sports:
ep_detail = [str(ep_obj.airdate).replace('-', sep_date)]\
if 'date_detail' not in kwargs.keys() else kwargs['date_detail'](ep_obj.airdate)
if show.is_sports:
month = ep_obj.airdate.strftime('%b')
ep_detail = (ep_detail + [month], ['%s|%s' % (x, month) for x in ep_detail])[date_or]
elif show.is_anime:
ep_detail = ep_obj.scene_absolute_number \
if 'ep_detail_anime' not in kwargs.keys() else kwargs['ep_detail_anime'](ep_obj.scene_absolute_number)
else:
ep_dict = self._ep_dict(ep_obj)
ep_detail = sickbeard.config.naming_ep_type[2] % ep_dict \
if 'ep_detail' not in kwargs.keys() else kwargs['ep_detail'](ep_dict)
ep_detail = ([ep_detail], ep_detail)[isinstance(ep_detail, list)]
detail = ({}, {'Episode_only': ep_detail})[detail_only and not show.is_sports and not show.is_anime]
return [dict({'Episode': self._build_search_strings(ep_detail, scene, prefix)}.items() + detail.items())]
@staticmethod
def _ep_dict(ep_obj):
season, episode = ((ep_obj.season, ep_obj.episode),
(ep_obj.scene_season, ep_obj.scene_episode))[bool(ep_obj.show.is_scene)]
return {'seasonnumber': season, 'episodenumber': episode}
def _build_search_strings(self, ep_detail, process_name=True, prefix=''):
"""
Build a list of search strings for querying a provider
:param ep_detail: String of episode detail or List of episode details
:param process_name: Bool Whether to call sanitizeSceneName() on show name
:param append: String to append to search strings
:param prefix: String to insert to search strings
:return: List of search string parameters
"""
if not isinstance(ep_detail, list):
ep_detail = [ep_detail]
if not isinstance(append, list):
append = [append]
ep_detail = ([ep_detail], ep_detail)[isinstance(ep_detail, list)]
prefix = ([prefix], prefix)[isinstance(prefix, list)]
search_params = []
crop = re.compile(r'([\.\s])(?:\1)+')
@ -716,9 +776,71 @@ class TorrentProvider(GenericProvider):
if process_name:
name = helpers.sanitizeSceneName(name)
for detail in ep_detail:
search_params += [crop.sub(r'\1', '%s %s' % (name, detail) + ('', ' ' + x)[any(x)]) for x in append]
search_params += [crop.sub(r'\1', '%s %s%s' % (name, x, detail)) for x in prefix]
return search_params
def _authorised(self, logged_in=None, post_params=None, failed_msg=None, url=None, timeout=30):
maxed_out = (lambda x: re.search(r'(?i)[1-3]((<[^>]+>)|\W)*(attempts|tries|remain)[\W\w]{,40}?(remain|left|attempt)', x))
logged_in, failed_msg = [None is not a and a or b for (a, b) in (
(logged_in, (lambda x=None: self.has_all_cookies())),
(failed_msg, (lambda x='': maxed_out(x) and u'Urgent abort, running low on login attempts. Password flushed to prevent service disruption to %s.' or
(re.search(r'(?i)(username|password)((<[^>]+>)|\W)*(or|and|/|\s)((<[^>]+>)|\W)*(password|incorrect)', x) and
u'Invalid username or password for %s. Check settings' or
u'Failed to authenticate or parse a response from %s, abort provider')))
)]
if logged_in():
return True
if hasattr(self, 'digest'):
self.cookies = re.sub(r'(?i)([\s\']+|cookie\s*:)', '', self.digest)
success, msg = self._check_cookie()
if not success:
self.cookies = None
logger.log(u'%s: [%s]' % (msg, self.cookies), logger.WARNING)
return False
elif not self._check_auth():
return False
if isinstance(url, type([])):
for i in range(0, len(url)):
helpers.getURL(url.pop(), session=self.session)
if not url:
if hasattr(self, 'urls'):
url = self.urls.get('login_action')
if url:
response = helpers.getURL(url, session=self.session)
try:
action = re.findall('[<]form[\w\W]+?action="([^"]+)', response)[0]
url = self.urls['config_provider_home_uri'] + action.lstrip('/')
except KeyError:
return super(TorrentProvider, self)._authorised()
else:
url = self.urls.get('login')
if not url:
return super(TorrentProvider, self)._authorised()
if hasattr(self, 'username') and hasattr(self, 'password'):
creds = dict(username=self.username, password=self.password)
if not post_params:
post_params = creds.copy()
elif self.password not in post_params.values() and isinstance(post_params, type({})):
post_params.update(creds)
response = helpers.getURL(url, post_data=post_params, session=self.session, timeout=timeout)
if response:
if logged_in(response):
return True
if maxed_out(response) and hasattr(self, 'password'):
self.password = None
sickbeard.save_config()
logger.log(failed_msg(response) % self.name, logger.ERROR)
return False
def _check_auth(self):
if hasattr(self, 'username') and hasattr(self, 'password'):
@ -742,67 +864,43 @@ class TorrentProvider(GenericProvider):
raise AuthException('%s for %s is empty in config provider options' % (setting, self.name))
def _find_propers(self, search_date=datetime.datetime.today(), search_terms=None):
def find_propers(self, **kwargs):
"""
Search for releases of type PROPER
:param search_date: Filter search on episodes since this date
:param search_terms: String or list of strings that qualify PROPER release types
:return: list of Proper objects
"""
results = []
my_db = db.DBConnection()
sql_results = my_db.select(
'SELECT s.show_name, e.showid, e.season, e.episode, e.status, e.airdate FROM tv_episodes AS e' +
' INNER JOIN tv_shows AS s ON (e.showid = s.indexer_id)' +
' WHERE e.airdate >= ' + str(search_date.toordinal()) +
' AND (e.status IN (%s)' % ','.join([str(x) for x in Quality.DOWNLOADED]) +
' OR (e.status IN (%s)))' % ','.join([str(x) for x in Quality.SNATCHED])
)
if not sql_results:
return results
clean_term = re.compile(r'(?i)[^a-z\|\.]+')
for sqlshow in sql_results:
showid, season, episode = [int(sqlshow[item]) for item in ('showid', 'season', 'episode')]
self.show = helpers.findCertainShow(sickbeard.showList, showid)
if not self.show:
continue
cur_ep = self.show.getEpisode(season, episode)
search_terms = getattr(self, 'proper_search_terms', ['proper', 'repack'])
if not isinstance(search_terms, list):
if None is search_terms:
search_terms = ['proper', 'repack']
elif not isinstance(search_terms, list):
if '' == search_terms:
search_terms = 'proper|repack'
search_terms = [search_terms]
search_terms = 'proper|repack'
search_terms = [search_terms]
for proper_term in search_terms:
proper_check = re.compile(r'(?i)(?:%s)' % clean_term.sub('', proper_term))
items = self._search_provider({'Propers': search_terms})
search_string = self._get_episode_search_strings(cur_ep, add_string=proper_term)
for item in self._do_search(search_string[0]):
title, url = self._get_title_and_url(item)
if not proper_check.search(title):
continue
results.append(classes.Proper(title, url, datetime.datetime.today(), self.show))
clean_term = re.compile(r'(?i)[^a-z1-9\|\.]+')
for proper_term in search_terms:
proper_check = re.compile(r'(?i)(?:%s)' % clean_term.sub('', proper_term))
for item in items:
title, url = self._title_and_url(item)
if proper_check.search(title):
results.append(classes.Proper(title, url, datetime.datetime.today(),
helpers.findCertainShow(sickbeard.showList, None)))
return results
@staticmethod
def _has_no_results(*html):
return re.search(r'(?i)<(?:h\d|strong)[^>]*>(?:'
+ 'your\ssearch\sdid\snot\smatch|'
+ 'nothing\sfound|'
+ 'no\storrents\sfound|'
+ '.*?there\sare\sno\sresults|'
+ '.*?no\shits\.\sTry\sadding'
+ ')', html[0])
return re.search(r'(?i)<(?:b|h\d|strong)[^>]*>(?:' +
'your\ssearch\sdid\snot\smatch|' +
'nothing\sfound|' +
'no\storrents\sfound|' +
'.*?there\sare\sno\sresults|' +
'.*?no\shits\.\sTry\sadding' +
')', html[0])
def get_cache_data(self, *args, **kwargs):
def cache_data(self, *args, **kwargs):
search_params = {'Cache': ['']}
return self._do_search(search_params)
return self._search_provider(search_params)

82
sickbeard/providers/gftracker.py
View file

@ -16,13 +16,13 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import time
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -33,53 +33,39 @@ class GFTrackerProvider(generic.TorrentProvider):
self.url_base = 'https://thegft.org/'
self.urls = {'config_provider_home_uri': self.url_base,
'login_get': self.url_base + 'login.php',
'login_post': self.url_base + 'loginsite.php',
'cache': self.url_base + 'browse.php?view=0&c26=1&c37=1&c19=1&c47=1&c17=1&c4=1&searchtype=1',
'login_init': self.url_base + 'login.php',
'login': self.url_base + 'loginsite.php',
'browse': self.url_base + 'browse.php?view=0&%s&searchtype=1%s',
'search': '&search=%s',
'get': self.url_base + '%s'}
self.categories = {'shows': [4, 17, 19, 26, 37, 47], 'anime': [16]}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.cache = GFTrackerCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'gft_uid' in self.session.cookies and 'gft_pass' in self.session.cookies
if logged_in():
return True
return super(GFTrackerProvider, self)._authorised(logged_in=(lambda x=None: self.has_all_cookies(pre='gft_')),
url=[self.urls['login_init']])
if self._check_auth():
helpers.getURL(self.urls['login_get'], session=self.session)
login_params = {'username': self.username, 'password': self.password}
response = helpers.getURL(self.urls['login_post'], post_data=login_params, session=self.session)
if response and logged_in():
return True
logger.log(u'Failed to authenticate with %s, abort provider.' % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'details', 'get': 'download',
'seeders': r'(^\d+)', 'leechers': r'(\d+)$'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_url = self.urls['cache']
if 'Cache' != mode:
search_url += self.urls['search'] % search_string
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['browse'] % (self._categories_string(mode),
(self.urls['search'] % search_string, '')['Cache' == mode])
html = self.get_url(search_url)
@ -99,44 +85,41 @@ class GFTrackerProvider(generic.TorrentProvider):
for tr in torrent_rows[1:]:
try:
seeders, leechers = 2 * [tr.find_all('td')[-1].get_text().strip()]
seeders = int(rc['seeders'].findall(seeders)[0])
leechers = int(rc['leechers'].findall(leechers)[0])
if mode != 'Cache' and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers = [tryInt(n) for n in [
rc['seeders'].findall(seeders)[0], rc['leechers'].findall(leechers)[0]]]
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', href=rc['info'])
title = ('title' in info.attrs and info['title']) or info.get_text().strip()
size = tr.find_all('td')[-2].get_text().strip()
download_url = self.urls['get'] % str(tr.find('a', href=rc['get'])['href']).lstrip('/')
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
'Cache' != mode and items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _season_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date)
return generic.TorrentProvider._season_strings(self, ep_obj, scene=False)
def _get_season_search_strings(self, ep_obj, **kwargs):
def _episode_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._get_season_search_strings(self, ep_obj, scene=False)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, scene=False, use_or=False)
return generic.TorrentProvider._episode_strings(self, ep_obj, scene=False, **kwargs)
class GFTrackerCache(tvcache.TVCache):
@ -144,10 +127,11 @@ class GFTrackerCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 17 # cache update frequency
self.update_freq = 17 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.cache_data()
return self.provider.get_cache_data()
provider = GFTrackerProvider()

75
sickbeard/providers/grabtheinfo.py
View file

@ -16,12 +16,12 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -33,54 +33,33 @@ class GrabTheInfoProvider(generic.TorrentProvider):
self.url_base = 'http://grabthe.info/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'takelogin.php',
'cache': self.url_base + 'browse.php?%s',
'browse': self.url_base + 'browse.php?%s&incldead=%s&blah=0%s',
'search': '&search=%s',
'get': self.url_base + '%s'}
self.categories = 'c56=1&c8=1&c61=1&c10=1&incldead=0&blah=0'
self.categories = {'shows': [36, 32, 43, 56, 8, 10, 61]}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = GrabTheInfoCache(self)
def _do_login(self):
logged_in = lambda: 'uid' in self.session.cookies and 'pass' in self.session.cookies
if logged_in():
return True
if self._check_auth():
login_params = {'username': self.username, 'password': self.password}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and 'Username or password incorrect' in response:
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['browse'] % (self._categories_string(), ('0', '3')[self.freeleech],
(self.urls['search'] % search_string, '')['Cache' == mode])
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_url = self.urls['cache'] % self.categories
if 'cache' != mode.lower():
search_url += self.urls['search'] % search_string
html = self.get_url(search_url)
cnt = len(items[mode])
@ -113,36 +92,32 @@ class GrabTheInfoProvider(generic.TorrentProvider):
if None is download_url:
continue
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers, size = [tryInt(n, n) for n in [
(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1, -3)]]
if self._peers_fail(mode, seeders, leechers):
continue
except (AttributeError, TypeError, KeyError):
except (AttributeError, TypeError, ValueError, KeyError):
continue
if title:
items[mode].append((title, self.urls['get']
% str(download_url['href'].lstrip('/')), seeders))
items[mode].append((title, self.urls['get'] % str(download_url['href'].lstrip('/')),
seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# for each search mode sort all the items by seeders
'Cache' != mode and items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _episode_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, sep_date='|', use_or=False)
return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='|', **kwargs)
class GrabTheInfoCache(tvcache.TVCache):
@ -150,11 +125,11 @@ class GrabTheInfoCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = GrabTheInfoProvider()

164
sickbeard/providers/hdbits.py
View file

@ -16,12 +16,13 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import urllib
from . import generic
from sickbeard import classes, logger, tvcache
from sickbeard import logger, tvcache
from sickbeard.exceptions import AuthException
from sickbeard.helpers import tryInt
from sickbeard.indexers import indexer_config
try:
import json
@ -40,11 +41,13 @@ class HDBitsProvider(generic.TorrentProvider):
'search': self.url_base + 'api/torrents',
'get': self.url_base + 'download.php?%s'}
self.categories = 2 # TV
self.categories = [3, 5, 2]
self.proper_search_terms = [' proper ', ' repack ']
self.url = self.urls['config_provider_home_uri']
self.username, self.passkey = 2 * [None]
self.username, self.passkey, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = HDBitsCache(self)
def check_auth_from_data(self, parsed_json):
@ -55,112 +58,105 @@ class HDBitsProvider(generic.TorrentProvider):
return True
def _get_season_search_strings(self, ep_obj, **kwargs):
def _season_strings(self, ep_obj, **kwargs):
return [self._build_search_strings(show=ep_obj.show, season=ep_obj)]
params = super(HDBitsProvider, self)._season_strings(ep_obj)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
show = ep_obj.show
if indexer_config.INDEXER_TVDB == show.indexer and show.indexerid:
params[0]['Season'].insert(0, dict(tvdb=dict(
id=show.indexerid,
season=(show.air_by_date or show.is_sports) and str(ep_obj.airdate)[:7] or
(show.is_anime and ('%d' % ep_obj.scene_absolute_number) or
(ep_obj.season, ep_obj.scene_season)[bool(show.is_scene)]))))
return [self._build_search_strings(show=ep_obj.show, episode=ep_obj)]
return params
def _get_title_and_url(self, item):
def _episode_strings(self, ep_obj, **kwargs):
title = item['name']
if title:
title = u'' + title.replace(' ', '.')
params = super(HDBitsProvider, self)._episode_strings(ep_obj, sep_date='|')
url = self.urls['get'] % urllib.urlencode({'id': item['id'], 'passkey': self.passkey})
show = ep_obj.show
if indexer_config.INDEXER_TVDB == show.indexer and show.indexerid:
id_param = dict(
id=show.indexerid,
episode=show.air_by_date and str(ep_obj.airdate).replace('-', ' ') or
(show.is_sports and ep_obj.airdate.strftime('%b') or
(show.is_anime and ('%i' % int(ep_obj.scene_absolute_number)) or
(ep_obj.episode, ep_obj.scene_episode)[bool(show.is_scene)])))
if not(show.air_by_date and show.is_sports and show.is_anime):
id_param['season'] = (ep_obj.season, ep_obj.scene_season)[bool(show.is_scene)]
params[0]['Episode'].insert(0, dict(tvdb=id_param))
return title, url
return params
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
self._check_auth()
logger.log(u'Search url: %s search_params: %s' % (self.urls['search'], search_params), logger.DEBUG)
response_json = self.get_url(self.urls['search'], post_data=search_params, json=True)
if response_json and 'data' in response_json and self.check_auth_from_data(response_json):
return response_json['data']
logger.log(u'Resulting JSON from %s isn\'t correct, not parsing it' % self.name, logger.ERROR)
return []
def find_propers(self, search_date=None):
results = []
api_data = {'username': self.username, 'passkey': self.passkey, 'category': self.categories}
search_terms = [' proper ', ' repack ']
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
for term in search_terms:
for item in self._do_search(self._build_search_strings(search_term=term)):
if item['utadded']:
for mode in search_params.keys():
for search_string in search_params[mode]:
post_data = api_data.copy()
if isinstance(search_string, dict):
post_data.update(search_string)
id_search = True
else:
post_data['search'] = search_string
id_search = False
post_data = json.dumps(post_data)
search_url = self.urls['search']
json_resp = self.get_url(search_url, post_data=post_data, json=True)
if not (json_resp and 'data' in json_resp and self.check_auth_from_data(json_resp)):
logger.log(u'Response from %s does not contain any json data, abort' % self.name, logger.ERROR)
return results
cnt = len(items[mode])
for item in json_resp['data']:
try:
result_date = datetime.datetime.fromtimestamp(int(item['utadded']))
except:
result_date = None
if result_date and (not search_date or result_date > search_date):
title, url = self._get_title_and_url(item)
if not re.search('(?i)(?:%s)' % term.strip(), title):
seeders, leechers, size = [tryInt(n, n) for n in [item.get(x) for x in 'seed', 'leech', 'size']]
if self._peers_fail(mode, seeders, leechers)\
or self.freeleech and re.search('(?i)no', item.get('freeleech', 'no')):
continue
results.append(classes.Proper(title, url, result_date, self.show))
title = item['name']
download_url = self.urls['get'] % urllib.urlencode({'id': item['id'], 'passkey': self.passkey})
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, item.get('seeders', 0), self._bytesizer(size)))
self._log_search(mode, len(items[mode]) - cnt,
('search_string: ' + search_string, self.name)['Cache' == mode])
self._sort_seeders(mode, items)
if id_search and len(items[mode]):
return items[mode]
results = list(set(results + items[mode]))
return results
def _build_search_strings(self, show=None, episode=None, season=None, search_term=None):
request_params = {'username': self.username, 'passkey': self.passkey, 'category': [self.categories]}
if episode or season:
param = {'id': show.indexerid}
if episode:
if show.air_by_date:
param['episode'] = str(episode.airdate).replace('-', '|')
elif show.is_sports:
param['episode'] = episode.airdate.strftime('%b')
elif show.is_anime:
param['episode'] = '%i' % int(episode.scene_absolute_number)
else:
param['season'] = episode.scene_season
param['episode'] = episode.scene_episode
if season:
if show.air_by_date or show.is_sports:
param['season'] = str(season.airdate)[:7]
elif show.is_anime:
param['season'] = '%d' % season.scene_absolute_number
else:
param['season'] = season.scene_season
request_params['tvdb'] = param
if search_term:
request_params['search'] = search_term
return json.dumps(request_params)
def get_cache_data(self):
self._check_auth()
response_json = self.get_url(self.urls['search'], post_data=self._build_search_strings(), json=True)
if response_json and 'data' in response_json and self.check_auth_from_data(response_json):
return response_json['data']
return []
class HDBitsCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 15 # cache update frequency
self.update_freq = 15 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = HDBitsProvider()

139
sickbeard/providers/hdspace.py Normal file
View file

@ -0,0 +1,139 @@
# coding=utf-8
#
# This file is part of SickGear.
#
# SickGear is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# SickGear is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import traceback
from . import generic
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
class HDSpaceProvider(generic.TorrentProvider):
def __init__(self):
generic.TorrentProvider.__init__(self, 'HDSpace')
self.url_base = 'https://hd-space.org/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'index.php?page=login',
'browse': self.url_base + 'index.php?page=torrents&' + '&'.join(['options=0', 'active=1', 'category=']),
'search': '&search=%s',
'get': self.url_base + '%s'}
self.categories = {'shows': [21, 22, 24, 25, 27, 28]}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = HDSpaceCache(self)
def _authorised(self, **kwargs):
return super(HDSpaceProvider, self)._authorised(post_params={'uid': self.username, 'pwd': self.password})
def _search_provider(self, search_params, **kwargs):
results = []
if not self._authorised():
return results
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'torrent-details', 'get': 'download', 'fl': 'free',
'peers': 'page=peers', 'nodots': '[\.\s]+'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['browse'] + self._categories_string(template='', delimiter=';')
if 'Cache' != mode:
search_url += self.urls['search'] % rc['nodots'].sub(' ', search_string)
html = self.get_url(search_url)
cnt = len(items[mode])
try:
if not html or self._has_no_results(html):
raise generic.HaltParseException
with BS4Parser(html, features=['html5lib', 'permissive'], attr='width="100%"\Wclass="lista"') as soup:
torrent_table = soup.find_all('table', attrs={'class': 'lista'})[-1]
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
if 2 > len(torrent_rows):
raise generic.HaltParseException
for tr in torrent_rows[1:]:
if tr.find('td', class_='header'):
continue
downlink = tr.find('a', href=rc['get'])
if None is downlink:
continue
try:
seeders, leechers = [int(x.get_text().strip()) for x in tr.find_all('a', href=rc['peers'])]
if self._peers_fail(mode, seeders, leechers)\
or self.freeleech and None is tr.find('img', title=rc['fl']):
continue
info = tr.find('a', href=rc['info'])
title = ('title' in info.attrs and info['title']) or info.get_text().strip()
size = tr.find_all('td')[-5].get_text().strip()
download_url = self.urls['get'] % str(downlink['href']).lstrip('/')
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_search(mode, len(items[mode]) - cnt, search_url)
self._sort_seeders(mode, items)
results = list(set(results + items[mode]))
return results
def _season_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._season_strings(self, ep_obj, scene=False)
def _episode_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._episode_strings(self, ep_obj, scene=False, **kwargs)
class HDSpaceCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.update_freq = 17 # cache update frequency
def _cache_data(self):
return self.provider.cache_data()
provider = HDSpaceProvider()

62
sickbeard/providers/iptorrents.py
View file

@ -16,11 +16,10 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
@ -33,49 +32,38 @@ class IPTorrentsProvider(generic.TorrentProvider):
self.url_base = 'https://iptorrents.eu/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'torrents/',
'search': self.url_base + 't?l73=1&l78=1&l66=1&l65=1&l79=1&l5=1&l4=1&qf=ti%s&q=%s#torrents',
'search': self.url_base + 't?%s;q=%s;qf=ti%s%s#torrents',
'get': self.url_base + '%s'}
self.categories = {'shows': [4, 5, 22, 23, 24, 25, 26, 55, 65, 66, 73, 78, 79], 'anime': [60]}
self.proper_search_terms = None
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = IPTorrentsCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'uid' in self.session.cookies and 'pass' in self.session.cookies
if logged_in():
return True
return super(IPTorrentsProvider, self)._authorised(post_params={'php': ''})
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'login': 'submit'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
logger.log(u'Failed to authenticate with %s, abort provider' % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
freeleech = self.freeleech and '&free=on' or ''
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
# URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_url = '%s%s' % (self.urls['search'] % (freeleech, search_string),
(';o=seeders', '')['Cache' == mode])
search_url = self.urls['search'] % (self._categories_string(mode, '%s', ';'), search_string,
('', ';free')[self.freeleech], (';o=seeders', '')['Cache' == mode])
html = self.get_url(search_url)
cnt = len(items[mode])
@ -94,45 +82,41 @@ class IPTorrentsProvider(generic.TorrentProvider):
try:
seeders, leechers = [int(tr.find('td', attrs={'class': x}).get_text().strip())
for x in ('t_seeders', 't_leechers')]
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', href=rc['info'])
title = ('title' in info.attrs and info['title']) or info.get_text().strip()
size = tr.find_all('td')[-4].get_text().strip()
download_url = self.urls['get'] % str(tr.find('a', href=rc['get'])['href']).lstrip('/')
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date, '')
class IPTorrentsCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = IPTorrentsProvider()

103
sickbeard/providers/kat.py
View file

@ -17,15 +17,15 @@
from __future__ import with_statement
import re
import os
import datetime
import re
import traceback
import urllib
from . import generic
from sickbeard import config, logger, tvcache, show_name_helpers, helpers
from sickbeard import config, logger, show_name_helpers, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import (has_anime, tryInt)
from sickbeard.common import Quality, mediaExtensions
from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
from lib.unidecode import unidecode
@ -38,10 +38,11 @@ class KATProvider(generic.TorrentProvider):
self.url_base = 'https://kat.ph/'
self.urls = {'config_provider_home_uri': self.url_base,
'search': [self.url_base, 'http://katproxy.com/'],
'cache_params': 'tv/?field=time_add&sorder=desc',
'search_params': 'usearch/%s/?field=seeders&sorder=desc'}
'base': [self.url_base, 'http://katproxy.com/'],
'search': 'usearch/%s/',
'sorted': '?field=time_add&sorder=desc'}
self.proper_search_terms = None
self.url = self.urls['config_provider_home_uri']
self.minseed, self.minleech = 2 * [None]
@ -108,7 +109,7 @@ class KATProvider(generic.TorrentProvider):
except Exception:
logger.log(u'Failed to quality parse ' + self.name + ' Traceback: ' + traceback.format_exc(), logger.ERROR)
def _get_season_search_strings(self, ep_obj, **kwargs):
def _season_strings(self, ep_obj, **kwargs):
if ep_obj.show.air_by_date or ep_obj.show.is_sports:
airdate = str(ep_obj.airdate).split('-')[0]
@ -119,54 +120,44 @@ class KATProvider(generic.TorrentProvider):
season = (ep_obj.season, ep_obj.scene_season)[bool(ep_obj.show.is_scene)]
ep_detail = ['S%(s)02i -S%(s)02iE' % {'s': season}, 'Season %s -Ep*' % season]
return [{'Season': self._build_search_strings(ep_detail, append=(' category:tv', '')[self.show.is_anime])}]
return [{'Season': self._build_search_strings(ep_detail)}]
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
def _episode_strings(self, ep_obj, **kwargs):
if not ep_obj:
return []
return generic.TorrentProvider._episode_strings(self, ep_obj, date_or=True,
ep_detail=lambda x: '%s|%s' % (config.naming_ep_type[2] % x,
config.naming_ep_type[0] % x),
ep_detail_anime=lambda x: '%02i' % x, **kwargs)
if self.show.air_by_date or self.show.is_sports:
ep_detail = str(ep_obj.airdate).replace('-', ' ')
if self.show.is_sports:
ep_detail += '|' + ep_obj.airdate.strftime('%b')
elif self.show.is_anime:
ep_detail = '%02i' % ep_obj.scene_absolute_number
else:
season, episode = ((ep_obj.season, ep_obj.episode),
(ep_obj.scene_season, ep_obj.scene_episode))[bool(ep_obj.show.is_scene)]
ep_dict = {'seasonnumber': season, 'episodenumber': episode}
ep_detail = '%s|%s' % (config.naming_ep_type[2] % ep_dict, config.naming_ep_type[0] % ep_dict)
# include provider specific appends
if not isinstance(add_string, list):
add_string = [add_string]
add_string = [x + ' category:tv' for x in add_string]
return [{'Episode': self._build_search_strings(ep_detail, append=(add_string, '')[self.show.is_anime])}]
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, search_mode='eponly', epcount=0, **kwargs):
results = []
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'link': 'normal'}.items())
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'link': 'normal', 'get': '^magnet', 'verif': 'verif'}.items())
url = 0
for mode in search_params.keys():
for search_string in search_params[mode]:
search_show = mode in ['Season', 'Episode']
if not search_show and has_anime():
search_params[mode] *= (1, 2)['Cache' == mode]
'Propers' == mode and search_params[mode].append('v1|v2|v3|v4|v5')
self.url = self.urls['search'][url]
search_args = ('search_params', 'cache_params')['Cache' == mode]
search_url = self.url + self.urls[search_args]
if 'Cache' != mode:
search_url %= urllib.quote(unidecode(search_string))
for enum, search_string in enumerate(search_params[mode]):
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
html = helpers.getURL(search_url)
self.url = self.urls['base'][url]
search_url = self.url + (self.urls['search'] % urllib.quote('%scategory:%s' % (
('', '%s ' % search_string)['Cache' != mode],
('tv', 'anime')[(search_show and bool(self.show and self.show.is_anime)) or bool(enum)])))
self.session.headers.update({'Referer': search_url})
html = self.get_url(search_url + self.urls['sorted'])
cnt = len(items[mode])
try:
if not html or self._has_no_results(html) or re.search(r'did not match any documents', html):
if not html or 'kastatic' not in html or self._has_no_results(html) or re.search(r'(?is)<(?:h\d)[^>]*>.*?(?:did\snot\smatch)', html):
if html and 'kastatic' not in html:
url += (1, 0)[url == len(self.urls['search'])]
url += (1, 0)[url == len(self.urls['base'])]
raise generic.HaltParseException
with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
@ -178,8 +169,9 @@ class KATProvider(generic.TorrentProvider):
for tr in torrent_rows[1:]:
try:
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers, size = [tryInt(n, n) for n in [
tr.find_all('td')[x].get_text().strip() for x in (-2, -1, -5)]]
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('div', {'class': 'torrentname'})
@ -187,11 +179,11 @@ class KATProvider(generic.TorrentProvider):
.strip()
link = self.url + info.find('a', {'class': rc['link']})['href'].lstrip('/')
download_magnet = tr.find('a', 'imagnet')['href']
except (AttributeError, TypeError):
download_magnet = tr.find('a', href=rc['get'])['href']
except (AttributeError, TypeError, ValueError):
continue
if self.confirmed and not tr.find('a', 'iverify'):
if self.confirmed and not (tr.find('a', title=rc['verif']) or tr.find('i', title=rc['verif'])):
logger.log(u'Skipping untrusted non-verified result: %s' % title, logger.DEBUG)
continue
@ -201,36 +193,31 @@ class KATProvider(generic.TorrentProvider):
title = self._find_season_quality(title, link, ep_number)
if title and download_magnet:
items[mode].append((title, download_magnet, seeders))
items[mode].append((title, download_magnet, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date, '')
class KATCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = KATProvider()

70
sickbeard/providers/morethan.py
View file

@ -18,12 +18,12 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -35,8 +35,8 @@ class MoreThanProvider(generic.TorrentProvider):
self.url_base = 'https://www.morethan.tv/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'login.php',
'search': self.url_base + 'torrents.php?searchstr=%s&tags_type=1&order_by=time&order_way=desc'
+ '&filter_cat[2]=1&action=basic&searchsubmit=1',
'search': self.url_base + 'torrents.php?searchstr=%s&' + '&'.join([
'tags_type=1', 'order_by=time', '&order_way=desc', 'filter_cat[2]=1', 'action=basic', 'searchsubmit=1']),
'get': self.url_base + '%s'}
self.url = self.urls['config_provider_home_uri']
@ -44,41 +44,24 @@ class MoreThanProvider(generic.TorrentProvider):
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.cache = MoreThanCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'session' in self.session.cookies
if logged_in():
return True
return super(MoreThanProvider, self)._authorised(logged_in=(lambda x=None: self.has_all_cookies('session')),
post_params={'keeplogged': '1', 'login': 'Log in'})
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'login': 'submit'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and 'username or password was incorrect' in response:
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v))
for (k, v) in {'info': 'view', 'get': 'download', 'name': 'showname', 'nuked': 'nuked'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['search'] % search_string
# fetches 15 results by default, and up to 100 if allowed in user profile
@ -102,8 +85,9 @@ class MoreThanProvider(generic.TorrentProvider):
continue
try:
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers, size = [tryInt(n, n) for n in [
tr.find_all('td')[x].get_text().strip() for x in (-2, -1, -4)]]
if self._peers_fail(mode, seeders, leechers):
continue
title = tr.find('a', title=rc['info']).get_text().strip()
@ -113,44 +97,36 @@ class MoreThanProvider(generic.TorrentProvider):
link = str(tr.find('a', title=rc['get'])['href']).replace('&amp;', '&').lstrip('/')
download_url = self.urls['get'] + link
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# for each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, use_or=False)
class MoreThanCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.cache_data()
return self.provider.get_cache_data()
provider = MoreThanProvider()

315
sickbeard/providers/newznab.py
View file

@ -16,13 +16,13 @@
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import urllib
import time
import urllib
import sickbeard
import generic
from sickbeard import helpers, scene_exceptions, logger, tvcache
from . import generic
from sickbeard import helpers, logger, scene_exceptions, tvcache
from sickbeard.exceptions import AuthException
@ -71,7 +71,7 @@ class NewznabProvider(generic.NZBProvider):
"""
Uses the newznab provider url and apikey to get the capabilities.
Makes use of the default newznab caps param. e.a. http://yournewznab/api?t=caps&apikey=skdfiw7823sdkdsfjsfk
Returns a tuple with (succes or not, array with dicts [{"id": "5070", "name": "Anime"},
Returns a tuple with (succes or not, array with dicts [{"id": "5070", "name": "Anime"},
{"id": "5080", "name": "Documentary"}, {"id": "5020", "name": "Foreign"}...etc}], error message)
"""
return_categories = []
@ -110,222 +110,233 @@ class NewznabProvider(generic.NZBProvider):
% (self.name or '', self.url or '', self.maybe_apikey() or '', self.cat_ids or '', self.enabled,
self.search_mode or '', self.search_fallback, self.enable_recentsearch, self.enable_backlog)
def _get_season_search_strings(self, ep_obj):
def _season_strings(self, ep_obj):
to_return = []
cur_params = {}
search_params = []
base_params = {}
# season
if ep_obj.show.air_by_date or ep_obj.show.is_sports:
date_str = str(ep_obj.airdate).split('-')[0]
cur_params['season'] = date_str
cur_params['q'] = date_str.replace('-', '.')
base_params['season'] = date_str
base_params['q'] = date_str.replace('-', '.')
elif ep_obj.show.is_anime:
cur_params['season'] = '%d' % ep_obj.scene_absolute_number
base_params['season'] = '%d' % ep_obj.scene_absolute_number
else:
cur_params['season'] = str((ep_obj.season, ep_obj.scene_season)[bool(ep_obj.show.is_scene)])
base_params['season'] = str((ep_obj.season, ep_obj.scene_season)[bool(ep_obj.show.is_scene)])
# search
rid = helpers.mapIndexersToShow(ep_obj.show)[2]
if rid:
cur_return = cur_params.copy()
cur_return['rid'] = rid
to_return.append(cur_return)
ids = helpers.mapIndexersToShow(ep_obj.show)
if ids[1]: # or ids[2]:
params = base_params.copy()
use_id = False
if ids[1] and self.supports_tvdbid():
params['tvdbid'] = ids[1]
use_id = True
if ids[2]:
params['rid'] = ids[2]
use_id = True
use_id and search_params.append(params)
# add new query strings for exceptions
name_exceptions = list(
set([helpers.sanitizeSceneName(a) for a in
scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
for cur_exception in name_exceptions:
cur_return = cur_params.copy()
if 'q' in cur_return:
cur_return['q'] = cur_exception + '.' + cur_return['q']
to_return.append(cur_return)
params = base_params.copy()
if 'q' in params:
params['q'] = '%s.%s' % (cur_exception, params['q'])
search_params.append(params)
return to_return
return [{'Season': search_params}]
def _get_episode_search_strings(self, ep_obj):
to_return = []
params = {}
def _episode_strings(self, ep_obj):
search_params = []
base_params = {}
if not ep_obj:
return [params]
return [base_params]
if ep_obj.show.air_by_date or ep_obj.show.is_sports:
date_str = str(ep_obj.airdate)
params['season'] = date_str.partition('-')[0]
params['ep'] = date_str.partition('-')[2].replace('-', '/')
base_params['season'] = date_str.partition('-')[0]
base_params['ep'] = date_str.partition('-')[2].replace('-', '/')
elif ep_obj.show.is_anime:
params['ep'] = '%i' % int(
base_params['ep'] = '%i' % int(
ep_obj.scene_absolute_number if int(ep_obj.scene_absolute_number) > 0 else ep_obj.scene_episode)
else:
params['season'], params['ep'] = ((ep_obj.season, ep_obj.episode),
(ep_obj.scene_season, ep_obj.scene_episode))[bool(ep_obj.show.is_scene)]
base_params['season'], base_params['ep'] = (
(ep_obj.season, ep_obj.episode), (ep_obj.scene_season, ep_obj.scene_episode))[ep_obj.show.is_scene]
# search
rid = helpers.mapIndexersToShow(ep_obj.show)[2]
if rid:
cur_return = params.copy()
cur_return['rid'] = rid
to_return.append(cur_return)
ids = helpers.mapIndexersToShow(ep_obj.show)
if ids[1]: # or ids[2]:
params = base_params.copy()
use_id = False
if ids[1]:
if self.supports_tvdbid():
params['tvdbid'] = ids[1]
use_id = True
if ids[2]:
params['rid'] = ids[2]
use_id = True
use_id and search_params.append(params)
# add new query strings for exceptions
name_exceptions = list(
set([helpers.sanitizeSceneName(a) for a in
scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
for cur_exception in name_exceptions:
cur_return = params.copy()
cur_return['q'] = cur_exception
to_return.append(cur_return)
params = base_params.copy()
params['q'] = cur_exception
search_params.append(params)
if ep_obj.show.is_anime:
# Experimental, add a searchstring without search explicitly for the episode!
# Remove the ?ep=e46 paramater and use add the episode number to the query paramater.
# Can be usefull for newznab indexers that do not have the episodes 100% parsed.
# Start with only applying the searchstring to anime shows
params['q'] = cur_exception
params_no_ep = params.copy()
# Experimental, add a search string without search explicitly for the episode!
# Remove the ?ep=e46 parameter and use the episode number to the query parameter.
# Can be useful for newznab indexers that do not have the episodes 100% parsed.
# Start with only applying the search string to anime shows
params = base_params.copy()
params['q'] = '%s.%02d' % (cur_exception, int(params['ep']))
if 'ep' in params:
params.pop('ep')
search_params.append(params)
params_no_ep['q'] = '%s.%02d' % (params_no_ep['q'], int(params_no_ep['ep']))
if 'ep' in params_no_ep:
params_no_ep.pop('ep')
to_return.append(params_no_ep)
return [{'Episode': search_params}]
return to_return
def supports_tvdbid(self):
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
return self.get_id() not in ['sick_beard_index']
def _search_provider(self, search_params, **kwargs):
api_key = self._check_auth()
if 'rid' not in search_params and 'q' not in search_params:
logger.log('Error no rid or search term given.')
return []
params = {'t': 'tvsearch',
'maxage': sickbeard.USENET_RETENTION,
'limit': 100,
'attrs': 'rageid',
'offset': 0}
# category ids
cat = []
if self.show:
if self.show.is_sports:
cat = ['5060']
elif self.show.is_anime:
cat = ['5070']
params['cat'] = ','.join([self.cat_ids] + cat)
# if max_age is set, use it, don't allow it to be missing
if not params['maxage'] or age:
params['maxage'] = age
if search_params:
params.update(search_params)
base_params = {'t': 'tvsearch',
'maxage': sickbeard.USENET_RETENTION or 0,
'limit': 100,
'attrs': 'rageid',
'offset': 0}
if isinstance(api_key, basestring):
params['apikey'] = api_key
base_params['apikey'] = api_key
results = []
offset = total = 0
total = 0
# hardcoded to stop after a max of 4 hits (400 items) per query
while (offset <= total) and (offset < 400):
search_url = '%sapi?%s' % (self.url, urllib.urlencode(params))
logger.log(u'Search url: ' + search_url, logger.DEBUG)
for mode in search_params.keys():
for i, params in enumerate(search_params[mode]):
data = self.cache.getRSSFeed(search_url)
time.sleep(1.1)
if not data or not self.check_auth_from_data(data):
break
# category ids
cat = []
cat_anime = ('5070', '6070')['nzbs_org' == self.get_id()]
cat_sport = '5060'
if 'Episode' == mode or 'Season' == mode:
if not ('rid' in params or 'tvdbid' in params or 'q' in params or not self.supports_tvdbid()):
logger.log('Error no rid, tvdbid, or search term available for search.')
continue
for item in data.entries:
title, url = self._get_title_and_url(item)
if title and url:
results.append(item)
if self.show:
if self.show.is_sports:
cat = [cat_sport]
elif self.show.is_anime:
cat = [cat_anime]
else:
logger.log(u'The data returned from %s is incomplete, this result is unusable' % self.name,
logger.DEBUG)
cat = [cat_sport, cat_anime]
# get total and offset attribs
try:
if 0 == total:
total = int(data.feed.newznab_response['total'] or 0)
hits = (total / 100 + int(0 < (total % 100)))
hits += int(0 == hits)
offset = int(data.feed.newznab_response['offset'] or 0)
except AttributeError:
break
if self.cat_ids or len(cat):
base_params['cat'] = ','.join(sorted(set(self.cat_ids.split(',') + cat)))
# No items found, prevent from doing another search
if 0 == total:
break
request_params = base_params.copy()
request_params.update(params)
if offset != params['offset']:
logger.log('Tell your newznab provider to fix their bloody newznab responses')
break
offset = 0
batch_count = not 0
params['offset'] += params['limit']
if total <= params['offset']:
logger.log('%s item%s found that will be used for episode matching' % (total, helpers.maybe_plural(total)),
logger.DEBUG)
break
# hardcoded to stop after a max of 4 hits (400 items) per query
while (offset <= total) and (offset < (200, 400)[self.supports_tvdbid()]) and batch_count:
cnt = len(results)
search_url = '%sapi?%s' % (self.url, urllib.urlencode(request_params))
data = self.cache.getRSSFeed(search_url)
i and time.sleep(1.1)
if not data or not self.check_auth_from_data(data):
break
for item in data.entries:
title, url = self._title_and_url(item)
if title and url:
results.append(item)
else:
logger.log(u'The data returned from %s is incomplete, this result is unusable' % self.name,
logger.DEBUG)
# get total and offset attribs
try:
if 0 == total:
total = int(data.feed.newznab_response['total'] or 0)
hits = (total / 100 + int(0 < (total % 100)))
hits += int(0 == hits)
offset = int(data.feed.newznab_response['offset'] or 0)
except AttributeError:
break
# No items found or cache mode, prevent from doing another search
if 0 == total or 'Cache' == mode:
break
if offset != request_params['offset']:
logger.log('Tell your newznab provider to fix their bloody newznab responses')
break
request_params['offset'] += request_params['limit']
if total <= request_params['offset']:
logger.log('%s item%s found that will be used for episode matching' % (total, helpers.maybe_plural(total)),
logger.DEBUG)
break
# there are more items available than the amount given in one call, grab some more
items = total - request_params['offset']
logger.log('%s more item%s to fetch from a batch of up to %s items.'
% (items, helpers.maybe_plural(items), request_params['limit']), logger.DEBUG)
batch_count = len(results) - cnt
if batch_count:
self._log_search(mode, batch_count, search_url)
if 'tvdbid' in request_params and len(results):
break
# there are more items available than the amount given in one call, grab some more
items = total - params['offset']
logger.log('%s more item%s to fetch from a batch of up to %s items.'
% (items, helpers.maybe_plural(items), params['limit']), logger.DEBUG)
return results
def find_propers(self, search_date=None):
return self._find_propers(search_date)
class NewznabCache(tvcache.TVCache):
def __init__(self, provider):
tvcache.TVCache.__init__(self, provider)
self.minTime = 15 # cache update frequency
def _getRSSData(self):
params = {'t': 'tvsearch',
'cat': self.provider.cat_ids + ',5060,5070',
'attrs': 'rageid'}
has_apikey = self.provider.maybe_apikey()
if has_apikey:
params['apikey'] = has_apikey
rss_url = '%sapi?%s' % (self.provider.url, urllib.urlencode(params))
logger.log(self.provider.name + ' cache update URL: ' + rss_url, logger.DEBUG)
return self.getRSSFeed(rss_url)
self.update_freq = 5 # cache update frequency
def updateCache(self):
if self.shouldUpdate():
result = []
if True or self.shouldUpdate():
try:
self._checkAuth()
except Exception:
return []
return result
data = self._getRSSData()
items = self.provider.cache_data()
if items:
# as long as the http request worked we count this as an update
if not data:
return []
self._clearCache()
self.setLastUpdate()
# clear cache
self._clearCache()
self.setLastUpdate()
if self.provider.check_auth_from_data(data):
items = data.entries
cl = []
for item in items:
ci = self._parseItem(item)
@ -336,11 +347,7 @@ class NewznabCache(tvcache.TVCache):
my_db = self.get_db()
my_db.mass_action(cl)
else:
raise AuthException(
u'Your authentication credentials for ' + self.provider.name + ' are incorrect, check your config')
return []
return result
# overwrite method with that parses the rageid from the newznab feed
def _parseItem(self, *item):

16
sickbeard/providers/nyaatorrents.py
View file

@ -19,7 +19,7 @@
import urllib
from . import generic
from sickbeard import logger, tvcache, show_name_helpers
from sickbeard import logger, show_name_helpers, tvcache
class NyaaProvider(generic.TorrentProvider):
@ -31,7 +31,7 @@ class NyaaProvider(generic.TorrentProvider):
self.cache = NyaaCache(self)
def _do_search(self, search_string, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_string, **kwargs):
results = []
if self.show and not self.show.is_anime:
@ -51,7 +51,7 @@ class NyaaProvider(generic.TorrentProvider):
items = data.entries
for curItem in items:
title, url = self._get_title_and_url(curItem)
title, url = self._title_and_url(curItem)
if title and url:
results.append(curItem)
@ -65,13 +65,13 @@ class NyaaProvider(generic.TorrentProvider):
return generic.TorrentProvider.find_search_results(self, show, episodes, search_mode, manual_search)
def _get_season_search_strings(self, ep_obj, **kwargs):
def _season_strings(self, ep_obj, **kwargs):
return show_name_helpers.makeSceneShowSearchStrings(self.show)
def _get_episode_search_strings(self, ep_obj, **kwargs):
def _episode_strings(self, ep_obj, **kwargs):
return self._get_season_search_strings(ep_obj)
return self._season_strings(ep_obj)
class NyaaCache(tvcache.TVCache):
@ -79,9 +79,9 @@ class NyaaCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 15 # cache update frequency
self.update_freq = 15 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
params = {'page': 'rss', # Use RSS page
'order': '1', # Sort Descending By Date
'cats': '1_37'} # Limit to English-translated Anime (for now)

51
sickbeard/providers/omgwtfnzbs.py
View file

@ -16,18 +16,19 @@
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
from datetime import datetime
import re
import time
import traceback
import generic
import sickbeard
import urllib
from sickbeard import tvcache, classes, logger, show_name_helpers
import sickbeard
from . import generic
from sickbeard import classes, logger, show_name_helpers, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.exceptions import AuthException
from sickbeard.rssfeeds import RSSFeeds
from sickbeard.bs4_parser import BS4Parser
class OmgwtfnzbsProvider(generic.NZBProvider):
@ -80,15 +81,15 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
return True
def _get_season_search_strings(self, ep_obj):
def _season_strings(self, ep_obj):
return [x for x in show_name_helpers.makeSceneSeasonSearchString(self.show, ep_obj)]
def _get_episode_search_strings(self, ep_obj):
def _episode_strings(self, ep_obj):
return [x for x in show_name_helpers.makeSceneSearchString(self.show, ep_obj)]
def _get_title_and_url(self, item):
def _title_and_url(self, item):
return item['release'], item['getnzb']
@ -112,7 +113,7 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
return result
def get_cache_data(self):
def cache_data(self):
api_key = self._init_api()
if False is api_key:
@ -132,11 +133,11 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
return data.entries
return []
def _do_search(self, search, search_mode='eponly', epcount=0, retention=0):
def _search_provider(self, search, search_mode='eponly', epcount=0, retention=0):
api_key = self._init_api()
if False is api_key:
return self.search_html(search)
return self.search_html(search, search_mode)
results = []
if None is not api_key:
params = {'user': self.username,
@ -156,7 +157,7 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
results.append(item)
return results
def search_html(self, search=''):
def search_html(self, search='', search_mode=''):
results = []
if None is self.cookies:
@ -189,7 +190,7 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
title = tr.find('a', href=rc['info'])['title']
download_url = tr.find('a', href=rc['get'])
age = tr.find_all('td')[-1]['data-sort']
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url and age:
@ -202,19 +203,20 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(results) - cnt, search_url)
mode = (mode, search_mode)['Propers' == search_mode]
self._log_search(mode, len(results) - cnt, search_url)
return results
def find_propers(self, search_date=None):
def find_propers(self, **kwargs):
search_terms = ['.PROPER.', '.REPACK.']
results = []
for term in search_terms:
for item in self._do_search(term, retention=4):
for item in self._search_provider(term, search_mode='Propers', retention=4):
if 'usenetage' in item:
title, url = self._get_title_and_url(item)
title, url = self._title_and_url(item)
try:
result_date = datetime.fromtimestamp(int(item['usenetage']))
except:
@ -244,10 +246,8 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
@staticmethod
def ui_string(key):
result = ''
if 'omgwtfnzbs_api_key' == key:
result = 'Or use... \'cookie: cookname=xx; cookpass=yy\''
return result
return 'omgwtfnzbs_api_key' == key and 'Or use... \'cookie: cookname=xx; cookpass=yy\'' or ''
class OmgwtfnzbsCache(tvcache.TVCache):
@ -255,10 +255,11 @@ class OmgwtfnzbsCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20
self.update_freq = 20
def _getRSSData(self):
def _cache_data(self):
return self.provider.cache_data()
return self.provider.get_cache_data()
provider = OmgwtfnzbsProvider()

74
sickbeard/providers/pisexy.py
View file

@ -14,12 +14,12 @@
# along with Sick Beard. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -39,43 +39,26 @@ class PiSexyProvider(generic.TorrentProvider):
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.cache = PiSexyCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'uid' in self.session.cookies and 'pass' in self.session.cookies and\
'pcode' in self.session.cookies and 'pisexy' in self.session.cookies
if logged_in():
return True
return super(PiSexyProvider, self)._authorised(logged_in=lambda x=None: self.has_all_cookies(['uid', 'pass', 'pcode', 'pisexy']))
if self._check_auth():
login_params = {'username': self.username, 'password': self.password}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and 'Username or password incorrect' in response:
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v))
for (k, v) in {'info': 'download', 'get': 'download', 'valid_cat': 'cat=(?:0|50[12])',
'title': r'Download\s([^\s]+).*', 'seeders': r'(^\d+)', 'leechers': r'(\d+)$'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['search'] % search_string
html = self.get_url(search_url)
cnt = len(items[mode])
@ -93,61 +76,46 @@ class PiSexyProvider(generic.TorrentProvider):
for tr in torrent_rows[1:]:
try:
seeders, leechers = 2 * [tr.find_all('td')[-4].get_text().strip()]
seeders = int(rc['seeders'].findall(seeders)[0])
leechers = int(rc['leechers'].findall(leechers)[0])
if 'Cache' != mode:
if not tr.find('a', href=rc['valid_cat']):
continue
if seeders < self.minseed or leechers < self.minleech:
continue
seeders, leechers = [tryInt(n) for n in [
rc['seeders'].findall(seeders)[0], rc['leechers'].findall(leechers)[0]]]
if self._peers_fail(mode, seeders, leechers) or not tr.find('a', href=rc['valid_cat']):
continue
info = tr.find('a', href=rc['info'])
title = 'title' in info.attrs and rc['title'].sub('', info.attrs['title'])\
or info.get_text().strip()
size = tr.find_all('td')[3].get_text().strip()
download_url = self.urls['get'] % str(tr.find('a', href=rc['get'])['href']).lstrip('/')
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, use_or=False)
class PiSexyCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 10 # cache update frequency
def _cache_data(self):
def _getRSSData(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = PiSexyProvider()

39
sickbeard/providers/pretome.py
View file

@ -15,8 +15,6 @@
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import datetime
from . import generic
from sickbeard import tvcache
from sickbeard.rssfeeds import RSSFeeds
@ -31,7 +29,7 @@ class PreToMeProvider(generic.TorrentProvider):
self.url_base = 'https://pretome.info/'
self.urls = {'config_provider_home_uri': self.url_base,
'cache': self.url_base + 'rss.php?cat[]=7&sort=0&type=d&key=%s',
'browse': self.url_base + 'rss.php?cat[]=7&sort=0&type=d&key=%s',
'search': '&st=1&tf=all&search=%s'}
self.url = self.urls['config_provider_home_uri']
@ -39,24 +37,23 @@ class PreToMeProvider(generic.TorrentProvider):
self.passkey = None
self.cache = PreToMeCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
return self._check_auth()
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
self._do_login()
self._authorised()
results = []
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
url = self.urls['cache'] % self.passkey
url = self.urls['browse'] % self.passkey
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = url + (self.urls['search'] % search_string, '')['Cache' == mode]
search_url = (url + self.urls['search'] % search_string, url)['Cache' == mode]
data = RSSFeeds(self).get_feed(search_url)
cnt = len(items[mode])
@ -64,35 +61,27 @@ class PreToMeProvider(generic.TorrentProvider):
for entry in data['entries']:
try:
if entry['title'] and 'download' in entry['link']:
items[mode].append((entry['title'], entry['link']))
items[mode].append((entry['title'], entry['link'], None, None))
except KeyError:
continue
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, use_or=False)
class PreToMeCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 6 # cache update frequency
self.update_freq = 6 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = PreToMeProvider()

103
sickbeard/providers/rarbg.py
View file

@ -21,7 +21,7 @@ import datetime
import time
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import helpers, logger, tvcache
from sickbeard.indexers.indexer_config import INDEXER_TVDB
@ -38,16 +38,14 @@ class RarbgProvider(generic.TorrentProvider):
'api_list': self.url_api + 'mode=list',
'api_search': self.url_api + 'mode=search'}
self.categories = '18;41'
self.params = {'defaults': '&category=%(cat)s&limit=100&sort=last' % {'cat': self.categories}
+ '&ranked=%(ranked)s&token=%(token)s',
self.params = {'defaults': '&format=json_extended&category=18;41&limit=100&sort=last&ranked=%(ranked)s&token=%(token)s',
'param_iid': '&search_imdb=%(sid)s',
'param_tid': '&search_tvdb=%(sid)s',
'param_rid': '&search_tvrage=%(sid)s',
'param_str': '&search_string=%(str)s',
'param_seed': '&min_seeders=%(min_seeds)s',
'param_peer': '&min_leechers=%(min_peers)s'}
self.proper_search_terms = '{{.proper.|.repack.}}'
self.url = self.urls['config_provider_home_uri']
self.minseed, self.minleech, self.token, self.token_expiry = 4 * [None]
@ -55,48 +53,50 @@ class RarbgProvider(generic.TorrentProvider):
self.request_throttle = datetime.datetime.now()
self.cache = RarbgCache(self)
def _do_login(self, reset=False):
def _authorised(self, reset=False, **kwargs):
if not reset and self.token and self.token_expiry and datetime.datetime.now() < self.token_expiry:
return True
response = helpers.getURL(self.urls['api_token'], session=self.session, json=True)
if response and 'token' in response:
self.token = response['token']
self.token_expiry = datetime.datetime.now() + datetime.timedelta(minutes=14)
return True
for r in range(0, 3):
response = helpers.getURL(self.urls['api_token'], session=self.session, json=True)
if response and 'token' in response:
self.token = response['token']
self.token_expiry = datetime.datetime.now() + datetime.timedelta(minutes=14)
return True
time.sleep(1.1)
logger.log(u'No usable API token returned from: %s' % self.urls['api_token'], logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login(reset=True):
if not self._authorised(reset=True):
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
id_search = None
if hasattr(self, 'show') and self.show and self.show.indexer and self.show.indexerid:
sid, search_with = 2 * [None]
if 0 < len(self.show.imdb_info):
sid = self.show.imdb_info['imdb_id']
search_with = 'param_iid'
else:
elif INDEXER_TVDB == self.show.indexer:
sid = self.show.indexerid
if INDEXER_TVDB == self.show.indexer:
search_with = 'param_tid'
else: # INDEXER_TVRAGE == self.show.indexer:
search_with = 'param_rid'
id_search = self.params[search_with] % {'sid': sid}
search_with = 'param_tid'
if sid and search_with:
id_search = self.params[search_with] % {'sid': sid}
dedupe = []
search_types = sorted([x for x in search_params.items()], key=lambda tup: tup[1], reverse=True) # sort type "_only" as first to process
search_types = sorted([x for x in search_params.items()], key=lambda tup: tup[0], reverse=True) # sort type "_only" as first to process
for mode_params in search_types:
mode_search = mode_params[0]
mode_base = mode_search.replace('_only', '')
mode = mode_search.replace('_only', '')
for search_string in mode_params[1]:
search_url = ''
searched_url = search_url = ''
url = 'api_list'
if 'Cache' != mode_search:
url = 'api_search'
@ -115,62 +115,61 @@ class RarbgProvider(generic.TorrentProvider):
if self.minleech:
search_url += self.params['param_peer'] % {'min_peers': self.minleech}
cnt = len(items[mode_base])
cnt = len(items[mode])
for r in range(0, 3):
time_out = 0
while(self.request_throttle > datetime.datetime.now()) and 2 >= time_out:
time_out += 1
time.sleep(1)
data = self.get_url(search_url % {'ranked': int(self.confirmed), 'token': self.token}, json=True)
searched_url = search_url % {'ranked': int(self.confirmed), 'token': self.token}
data = self.get_url(searched_url, json=True)
self.token_expiry = datetime.datetime.now() + datetime.timedelta(minutes=14)
self.request_throttle = datetime.datetime.now() + datetime.timedelta(seconds=3)
if not data:
continue
if 'error' in data:
if 5 == data['error_code']: # Too many requests per second.
continue
elif 2 == data['error_code']: # Invalid token set
if self._do_login(reset=True):
if self._authorised(reset=True):
continue
self._log_result(mode_base, len(items[mode_base]) - cnt, search_url)
return results
self.log_result(mode, len(items[mode]) - cnt, searched_url)
return items[mode]
break
if 'error' not in data:
for item in data['torrent_results']:
try:
title = item['filename']
get = item['download']
if not (title and get) or get in dedupe:
continue
dedupe += [get]
items[mode_base].append((title, get))
except Exception:
pass
title, download_magnet, seeders, size = [
item.get(x) for x in 'title', 'download', 'seeders', 'size']
title = None is title and item.get('filename') or title
if not (title and download_magnet) or download_magnet in dedupe:
continue
dedupe += [download_magnet]
items[mode].append((title, download_magnet, seeders, self._bytesizer(size)))
if 0 < len(items[mode_base]):
results += items[mode_base]
items[mode_base] = []
self._log_search(mode, len(items[mode]) - cnt, searched_url)
self._log_result(mode_base, len(items[mode_base]) - cnt, search_url)
self._sort_seeders(mode, items)
if '_only' in mode_search and 0 < len(results):
results = list(set(results + items[mode]))
if '_only' in mode_search and len(results):
break
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _season_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date, '{{.proper.|.repack.}}')
return generic.TorrentProvider._season_strings(self, ep_obj, detail_only=True)
def _get_season_search_strings(self, ep_obj, **kwargs):
def _episode_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._get_season_search_strings(self, ep_obj, detail_only=True)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
search_params = generic.TorrentProvider._get_episode_search_strings(self, ep_obj, detail_only=True)
search_params = generic.TorrentProvider._episode_strings(self, ep_obj, detail_only=True, date_or=True, **kwargs)
if self.show.air_by_date and self.show.is_sports:
for x, types in enumerate(search_params):
for y, ep_type in enumerate(types):
@ -184,9 +183,9 @@ class RarbgCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = RarbgProvider()

18
sickbeard/providers/rsstorrent.py
View file

@ -1,4 +1,4 @@
# Author: Mr_Orange
# coding=utf-8
#
# This file is part of SickGear.
#
@ -19,8 +19,8 @@ import re
from . import generic
from sickbeard import logger, tvcache
from sickbeard.rssfeeds import RSSFeeds
from sickbeard.exceptions import ex
from sickbeard.rssfeeds import RSSFeeds
from lib.bencode import bdecode
@ -55,7 +55,7 @@ class TorrentRssProvider(generic.TorrentProvider):
self.enable_recentsearch,
self.enable_backlog)
def _get_title_and_url(self, item):
def _title_and_url(self, item):
title, url = None, None
@ -86,10 +86,10 @@ class TorrentRssProvider(generic.TorrentProvider):
return success, err_msg
try:
items = self.get_cache_data()
items = self.cache_data()
for item in items:
title, url = self._get_title_and_url(item)
title, url = self._title_and_url(item)
if not (title and url):
continue
if url.startswith('magnet:'):
@ -111,7 +111,7 @@ class TorrentRssProvider(generic.TorrentProvider):
except Exception as e:
return False, 'Error when trying to load RSS: ' + ex(e)
def get_cache_data(self):
def cache_data(self):
logger.log(u'TorrentRssCache cache update URL: ' + self.url, logger.DEBUG)
@ -125,8 +125,8 @@ class TorrentRssCache(tvcache.TVCache):
def __init__(self, provider):
tvcache.TVCache.__init__(self, provider)
self.minTime = 15
self.update_freq = 15
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()

66
sickbeard/providers/scc.py
View file

@ -16,13 +16,13 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import time
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -44,37 +44,23 @@ class SCCProvider(generic.TorrentProvider):
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.cache = SCCCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'uid' in self.session.cookies and 'pass' in self.session.cookies
if logged_in():
return True
return super(SCCProvider, self)._authorised(post_params={'submit': 'come+on+in'})
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'submit': 'come on in'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
logger.log(u'Failed to authenticate with %s, abort provider.' % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
if not self._do_login():
if not self._authorised():
return results
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string, void = self._get_title_and_url((search_string, None))
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string, void = self._title_and_url((search_string, None))
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
if 'Season' == mode:
searches = [self.urls['archive'] % search_string]
@ -83,6 +69,7 @@ class SCCProvider(generic.TorrentProvider):
self.urls['nonscene'] % search_string]
for search_url in searches:
html = self.get_url(search_url)
cnt = len(items[mode])
@ -99,9 +86,10 @@ class SCCProvider(generic.TorrentProvider):
for tr in torrent_table.find_all('tr')[1:]:
try:
seeders, leechers = [int(tr.find('td', attrs={'class': x}).get_text().strip())
for x in ('ttr_seeders', 'ttr_leechers')]
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers, size = [tryInt(n, n) for n in [
tr.find('td', attrs={'class': x}).get_text().strip()
for x in ('ttr_seeders', 'ttr_leechers', 'ttr_size')]]
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', href=rc['info'])
@ -109,32 +97,28 @@ class SCCProvider(generic.TorrentProvider):
link = str(tr.find('a', href=rc['get'])['href']).lstrip('/')
download_url = self.urls['get'] % link
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
time.sleep(1.1)
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _episode_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, sep_date='.', use_or=False)
return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='.', **kwargs)
class SCCCache(tvcache.TVCache):
@ -142,11 +126,11 @@ class SCCCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = SCCProvider()

90
sickbeard/providers/scenetime.py
View file

@ -15,13 +15,14 @@
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import ast
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -33,52 +34,47 @@ class SceneTimeProvider(generic.TorrentProvider):
self.url_base = 'https://www.scenetime.com/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'takelogin.php',
'search': self.url_base + 'browse.php?%ssearch=%s',
'browse': self.url_base + 'browse_API.php',
'params': {'sec': 'jax', 'cata': 'yes'},
'get': self.url_base + 'download.php/%(id)s/%(title)s.torrent'}
self.categories = 'c2=1&c43=1&c9=1&c63=1&c77=1&c79=1&c101=1&cata=yes&'
self.categories = {'shows': [2, 43, 9, 63, 77, 79, 101]}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = SceneTimeCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'uid' in self.session.cookies and 'pass' in self.session.cookies
if logged_in():
return True
return super(SceneTimeProvider, self)._authorised(post_params={'submit': 'Log in'})
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'submit': 'Log in'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and 'Username or password incorrect' in response:
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': '.*id=(\d+).*',
'cats': 'cat=(?:2|9|43|63|77|79|101)'}.items())
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': '.*id=(\d+).*', 'fl': '\[freeleech\]',
'cats': 'cat=(?:%s)' % self._categories_string(template='', delimiter='|')
}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['search'] % (self.categories, search_string)
html = self.get_url(search_url)
post_data = self.urls['params'].copy()
post_data.update(ast.literal_eval('{%s}' % self._categories_string(template='"c%s": "1"', delimiter=',')))
if 'Cache' != mode:
post_data['search'] = '+'.join(search_string.split())
if self.freeleech:
post_data.update({'freeleech': 'on'})
self.session.headers.update({'Referer': self.url + 'browse.php', 'X-Requested-With': 'XMLHttpRequest'})
html = self.get_url(self.urls['browse'], post_data=post_data)
cnt = len(items[mode])
try:
@ -86,7 +82,7 @@ class SceneTimeProvider(generic.TorrentProvider):
raise generic.HaltParseException
with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
torrent_table = soup.find('div', id='torrenttable').find('table')
torrent_table = soup.find('table', attrs={'cellpadding': 5})
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
if 2 > len(torrent_rows):
@ -94,9 +90,11 @@ class SceneTimeProvider(generic.TorrentProvider):
for tr in torrent_rows[1:]:
try:
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
seeders, leechers, size = [tryInt(n, n) for n in [
tr.find_all('td')[x].get_text().strip() for x in (-2, -1, -3)]]
if None is tr.find('a', href=rc['cats'])\
or ('Cache' != mode and (seeders < self.minseed or leechers < self.minleech)):
or self.freeleech and None is rc['fl'].search(tr.find_all('td')[1].get_text())\
or self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', href=rc['info'])
@ -104,45 +102,37 @@ class SceneTimeProvider(generic.TorrentProvider):
download_url = self.urls['get'] % {'id': re.sub(rc['get'], r'\1', str(info.attrs['href'])),
'title': str(title).replace(' ', '.')}
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt,
('search string: ' + search_string, self.name)['Cache' == mode])
# For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, use_or=False)
class SceneTimeCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 15 # cache update frequency
self.update_freq = 15 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = SceneTimeProvider()

162
sickbeard/providers/shazbat.py Normal file
View file

@ -0,0 +1,162 @@
# coding=utf-8
#
# Author: SickGear
#
# This file is part of SickGear.
#
# SickGear is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# SickGear is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import time
import traceback
from . import generic
from sickbeard import helpers, logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
class ShazbatProvider(generic.TorrentProvider):
def __init__(self):
generic.TorrentProvider.__init__(self, 'Shazbat')
self.url_base = 'https://www.shazbat.tv/'
self.urls = {'config_provider_home_uri': self.url_base,
'login_action': self.url_base + 'login',
'feeds': self.url_base + 'rss_feeds',
'browse': self.url_base + 'torrents?portlet=true',
'search': self.url_base + 'search?portlet=true&search=%s',
'show': self.url_base + 'show?id=%s&show_mode=torrents',
'get': self.url_base + '%s'}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = ShazbatCache(self)
def _authorised(self, **kwargs):
return super(ShazbatProvider, self)._authorised(
logged_in=(lambda x=None: '<input type="password"' not in helpers.getURL(
self.urls['feeds'], session=self.session)),
post_params={'tv_login': self.username, 'tv_password': self.password,
'referer': 'login', 'query': '', 'email': ''})
def _search_provider(self, search_params, **kwargs):
results = []
if not self._authorised():
return results
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'show_id': '"show\?id=(\d+)[^>]+>([^<]+)<\/a>',
'get': 'load_torrent'}.items())
search_types = sorted([x for x in search_params.items()], key=lambda tup: tup[0], reverse=True)
maybe_only = search_types[0][0]
show_detail = '_only' in maybe_only and search_params.pop(maybe_only)[0] or ''
for mode in search_params.keys():
for search_string in search_params[mode]:
if 'Cache' == mode:
search_url = self.urls['browse']
html = self.get_url(search_url)
else:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_string = search_string.replace(show_detail, '').strip()
search_url = self.urls['search'] % search_string
html = self.get_url(search_url)
shows = rc['show_id'].findall(html)
if not any(shows):
continue
html = ''
for show in shows:
sid, title = show
if title not in search_string:
continue
html and time.sleep(1.1)
html += self.get_url(self.urls['show'] % sid)
cnt = len(items[mode])
try:
if not html or self._has_no_results(html):
raise generic.HaltParseException
with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
torrent_rows = soup.tbody.find_all('tr') or soup.table.find_all('tr') or []
if 2 > len(torrent_rows):
raise generic.HaltParseException
for tr in torrent_rows[0:]:
try:
stats = tr.find_all('td')[3].get_text().strip()
seeders, leechers = [(tryInt(x[0], 0), tryInt(x[1], 0)) for x in
re.findall('(?::(\d+))(?:\W*[/]\W*:(\d+))?', stats) if x[0]][0]
if self._peers_fail(mode, seeders, leechers):
continue
sizes = [(tryInt(x[0], x[0]), tryInt(x[1], False)) for x in
re.findall('([\d\.]+\w+)?(?:\s*[\(\[](\d+)[\)\]])?', stats) if x[0]][0]
size = sizes[(0, 1)[1 < len(sizes)]]
for element in [x for x in tr.find_all('td')[2].contents[::-1] if unicode(x).strip()]:
if 'NavigableString' in str(element.__class__):
title = unicode(element).strip()
break
link = str(tr.find('a', href=rc['get'])['href']).replace('&amp;', '&').lstrip('/')
download_url = self.urls['get'] % link
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_search(mode, len(items[mode]) - cnt, search_url)
self._sort_seeders(mode, items)
results = list(set(results + items[mode]))
return results
def _season_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._season_strings(self, ep_obj, detail_only=True, scene=False)
def _episode_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._episode_strings(self, ep_obj, detail_only=True, scene=False, **kwargs)
class ShazbatCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.update_freq = 20 # cache update frequency
def _cache_data(self):
return self.provider.cache_data()
provider = ShazbatProvider()

81
sickbeard/providers/speedcd.py
View file

@ -16,27 +16,27 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import time
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import tvcache
from sickbeard.helpers import tryInt
class SpeedCDProvider(generic.TorrentProvider):
def __init__(self):
generic.TorrentProvider.__init__(self, 'Speedcd')
generic.TorrentProvider.__init__(self, 'SpeedCD')
self.url_base = 'http://speed.cd/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'take_login.php',
'login_action': self.url_base + 'login.php',
'search': self.url_base + 'V3/API/API.php',
'get': self.url_base + 'download.php?torrent=%s'}
self.categories = {'Season': {'c14': 1},
'Episode': {'c2': 1, 'c49': 1},
'Cache': {'c14': 1, 'c2': 1, 'c49': 1}}
self.categories = {'Season': {'c41': 1, 'c53': 1},
'Episode': {'c2': 1, 'c49': 1, 'c50': 1, 'c55': 1},
'Cache': {'c41': 1, 'c2': 1, 'c49': 1, 'c50': 1, 'c53': 1, 'c55': 1}}
self.url = self.urls['config_provider_home_uri']
@ -44,80 +44,65 @@ class SpeedCDProvider(generic.TorrentProvider):
self.freeleech = False
self.cache = SpeedCDCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'inSpeed_speedian' in self.session.cookies
if logged_in():
return True
return super(SpeedCDProvider, self)._authorised(logged_in=(lambda x=None: self.has_all_cookies('inSpeed_speedian')))
if self._check_auth():
login_params = {'username': self.username, 'password': self.password}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and re.search('Incorrect username or Password. Please try again.', response):
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
remove_tag = re.compile(r'<[^>]*>')
for mode in search_params.keys():
search_mode = (mode, 'Episode')['Propers' == mode]
for search_string in search_params[mode]:
search_string = '+'.join(search_string.split())
post_data = dict({'/browse.php?': None, 'cata': 'yes', 'jxt': 4, 'jxw': 'b', 'search': search_string},
**self.categories[mode])
**self.categories[search_mode])
if self.freeleech:
post_data['freeleech'] = 'on'
data_json = self.get_url(self.urls['search'], post_data=post_data, json=True)
cnt = len(items[mode])
try:
if not data_json:
raise generic.HaltParseException
torrents = data_json.get('Fs', [])[0].get('Cn', {}).get('torrents', [])
for torrent in torrents:
for item in torrents:
if self.freeleech and not torrent['free']:
if self.freeleech and not item.get('free'):
continue
seeders, leechers = int(torrent['seed']), int(torrent['leech'])
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers, size = [tryInt(n, n) for n in [item.get(x) for x in 'seed', 'leech', 'size']]
if self._peers_fail(mode, seeders, leechers):
continue
title = remove_tag.sub('', torrent['name'])
url = self.urls['get'] % (torrent['id'])
if title and url:
items[mode].append((title, url, seeders))
title = remove_tag.sub('', item.get('name'))
download_url = self.urls['get'] % item.get('id')
if title and download_url:
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except Exception:
time.sleep(1.1)
self._log_result(mode, len(items[mode]) - cnt,
self._log_search(mode, len(items[mode]) - cnt,
('search string: ' + search_string, self.name)['Cache' == mode])
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _episode_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, sep_date='.', use_or=False)
return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='.', **kwargs)
class SpeedCDCache(tvcache.TVCache):
@ -125,11 +110,11 @@ class SpeedCDCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = SpeedCDProvider()

84
sickbeard/providers/strike.py Normal file
View file

@ -0,0 +1,84 @@
# coding=utf-8
#
# This file is part of SickGear.
#
# SickGear is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# SickGear is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
from . import generic
from sickbeard import helpers
from sickbeard.helpers import tryInt
class StrikeProvider(generic.TorrentProvider):
def __init__(self):
generic.TorrentProvider.__init__(self, 'Strike')
self.url_base = 'https://getstrike.net/'
self.urls = {'config_provider_home_uri': self.url_base,
'search': self.url_base + 'api/v2/torrents/search/?category=%s&phrase=%s'}
self.url = self.urls['config_provider_home_uri']
self.minseed, self.minleech = 2 * [None]
def _search_provider(self, search_params, **kwargs):
results = []
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
for mode in search_params.keys():
search_show = mode in ['Season', 'Episode']
if not search_show and helpers.has_anime():
search_params[mode] *= (1, 2)['Cache' == mode]
for enum, search_string in enumerate(search_params[mode]):
search_url = self.urls['search'] % \
(('tv', 'anime')[(search_show and bool(self.show and self.show.is_anime)) or bool(enum)],
(re.sub('[\.\s]+', ' ', search_string), 'x264')['Cache' == mode])
data_json = self.get_url(search_url, json=True)
cnt = len(items[mode])
try:
for item in data_json['torrents']:
seeders, leechers, title, download_magnet, size = [tryInt(n, n) for n in [item.get(x) for x in [
'seeds', 'leeches', 'torrent_title', 'magnet_uri', 'size']]]
if self._peers_fail(mode, seeders, leechers):
continue
if title and download_magnet:
items[mode].append((title, download_magnet, seeders, self._bytesizer(size)))
except Exception:
pass
self._log_search(mode, len(items[mode]) - cnt, search_url)
self._sort_seeders(mode, items)
results = list(set(results + items[mode]))
return results
def _season_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._season_strings(self, ep_obj, scene=False)
def _episode_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._episode_strings(self, ep_obj, scene=False, **kwargs)
provider = StrikeProvider()

71
sickbeard/providers/thepiratebay.py
View file

@ -19,15 +19,14 @@ from __future__ import with_statement
import os
import re
import datetime
import urllib
import traceback
import urllib
from . import generic
from sickbeard import config, logger, tvcache, show_name_helpers
from sickbeard.bs4_parser import BS4Parser
from sickbeard.common import Quality, mediaExtensions
from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
@ -40,8 +39,9 @@ class ThePirateBayProvider(generic.TorrentProvider):
'https://thepiratebay.mn/', 'https://thepiratebay.vg/',
'https://thepiratebay.la/'],
'search': 'search/%s/0/7/200',
'cache': 'tv/latest/'} # order by seed
'browse': 'tv/latest/'} # order by seed
self.proper_search_terms = None
self.url = self.urls['config_provider_home_uri'][0]
self.minseed, self.minleech = 2 * [None]
@ -115,7 +115,7 @@ class ThePirateBayProvider(generic.TorrentProvider):
return title
def _get_season_search_strings(self, ep_obj, **kwargs):
def _season_strings(self, ep_obj, **kwargs):
if ep_obj.show.air_by_date or ep_obj.show.sports:
airdate = str(ep_obj.airdate).split('-')[0]
@ -128,26 +128,17 @@ class ThePirateBayProvider(generic.TorrentProvider):
return [{'Season': self._build_search_strings(ep_detail)}]
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
def _episode_strings(self, ep_obj, **kwargs):
if self.show.air_by_date or self.show.is_sports:
ep_detail = str(ep_obj.airdate).replace('-', ' ')
if self.show.is_sports:
ep_detail += '|' + ep_obj.airdate.strftime('%b')
elif self.show.is_anime:
ep_detail = '%02i' % ep_obj.scene_absolute_number
else:
season, episode = ((ep_obj.season, ep_obj.episode),
(ep_obj.scene_season, ep_obj.scene_episode))[bool(ep_obj.show.is_scene)]
ep_dict = {'seasonnumber': season, 'episodenumber': episode}
ep_detail = '%s|%s' % (config.naming_ep_type[2] % ep_dict, config.naming_ep_type[0] % ep_dict)
return generic.TorrentProvider._episode_strings(self, ep_obj, date_or=True,
ep_detail=lambda x: '%s|%s' % (config.naming_ep_type[2] % x,
config.naming_ep_type[0] % x),
ep_detail_anime=lambda x: '%02i' % x, **kwargs)
return [{'Episode': self._build_search_strings(ep_detail, append=(add_string, '')[self.show.is_anime])}]
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, search_mode='eponly', epcount=0, **kwargs):
results = []
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v))
for (k, v) in {'info': 'detail', 'get': 'download[^"]+magnet', 'tid': r'.*/(\d{5,}).*',
@ -155,17 +146,17 @@ class ThePirateBayProvider(generic.TorrentProvider):
has_signature = False
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
log_url = '%s %s' % (self.name, search_string) # placebo value
for idx, search_url in enumerate(self.urls['config_provider_home_uri']):
search_url += self.urls['cache'] if 'Cache' == mode\
search_url += self.urls['browse'] if 'Cache' == mode\
else self.urls['search'] % (urllib.quote(search_string))
log_url = u'(%s/%s): %s' % (idx + 1, len(self.urls['config_provider_home_uri']), search_url)
html = self.get_url(search_url)
if html and re.search(r'Pirate\sBay', html[33:7632:]):
has_signature = True
break
@ -177,7 +168,7 @@ class ThePirateBayProvider(generic.TorrentProvider):
if not html or self._has_no_results(html):
raise generic.HaltParseException
with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
with BS4Parser(html, features=['html5lib', 'permissive'], attr='id="searchResult"') as soup:
torrent_table = soup.find('table', attrs={'id': 'searchResult'})
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
@ -187,7 +178,7 @@ class ThePirateBayProvider(generic.TorrentProvider):
for tr in torrent_table.find_all('tr')[1:]:
try:
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', title=rc['info'])
@ -195,7 +186,7 @@ class ThePirateBayProvider(generic.TorrentProvider):
tid = rc['tid'].sub(r'\1', str(info['href']))
download_magnet = tr.find('a', title=rc['get'])['href']
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if self.confirmed and not tr.find('img', title=rc['verify']):
@ -209,39 +200,41 @@ class ThePirateBayProvider(generic.TorrentProvider):
title = self._find_season_quality(title, tid, ep_number)
if title and download_magnet:
items[mode].append((title, download_magnet, seeders))
size = None
try:
size = re.findall('(?i)size[^\d]+(\d+(?:[\.,]\d+)?\W*[bkmgt]\w+)',
tr.find_all(class_='detDesc')[0].get_text())[0]
except Exception:
pass
items[mode].append((title, download_magnet, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, log_url)
self._log_search(mode, len(items[mode]) - cnt, log_url)
# For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
if not has_signature:
logger.log(u'Failed to identify a page from ThePirateBay at %s attempted urls (tpb blocked? general network issue or site dead)' % len(self.urls['config_provider_home_uri']), logger.ERROR)
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date, '')
class ThePirateBayCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = ThePirateBayProvider()

12
sickbeard/providers/tokyotoshokan.py
View file

@ -20,7 +20,7 @@ import traceback
import urllib
from . import generic
from sickbeard import logger, tvcache, show_name_helpers
from sickbeard import logger, show_name_helpers, tvcache
from sickbeard.bs4_parser import BS4Parser
@ -33,7 +33,7 @@ class TokyoToshokanProvider(generic.TorrentProvider):
self.cache = TokyoToshokanCache(self)
def _do_search(self, search_string, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_string, search_mode='eponly', **kwargs):
results = []
if self.show and not self.show.is_anime:
@ -70,11 +70,11 @@ class TokyoToshokanProvider(generic.TorrentProvider):
return generic.TorrentProvider.find_search_results(self, show, episodes, search_mode, manual_search)
def _get_season_search_strings(self, ep_obj, **kwargs):
def _season_strings(self, ep_obj, **kwargs):
return [x.replace('.', ' ') for x in show_name_helpers.makeSceneSeasonSearchString(self.show, ep_obj)]
def _get_episode_search_strings(self, ep_obj, **kwargs):
def _episode_strings(self, ep_obj, **kwargs):
return [x.replace('.', ' ') for x in show_name_helpers.makeSceneSearchString(self.show, ep_obj)]
@ -84,9 +84,9 @@ class TokyoToshokanCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 15 # cache update frequency
self.update_freq = 15 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
params = {'filter': '1'}
url = self.provider.url + 'rss.php?' + urllib.urlencode(params)

83
sickbeard/providers/torrentbytes.py
View file

@ -16,12 +16,12 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -33,58 +33,44 @@ class TorrentBytesProvider(generic.TorrentProvider):
self.url_base = 'https://www.torrentbytes.net/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'takelogin.php',
'search': self.url_base + 'browse.php?search=%s%s',
'search': self.url_base + 'browse.php?search=%s&%s',
'get': self.url_base + '%s'}
self.categories = '&c41=1&c33=1&c38=1&c32=1&c37=1'
self.categories = {'shows': [41, 33, 38, 32, 37]}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = TorrentBytesCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'uid' in self.session.cookies and 'pass' in self.session.cookies
if logged_in():
return True
return super(TorrentBytesProvider, self)._authorised(post_params={'login': 'Log in!'})
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'login': 'Log in!'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and 'Username or password incorrect' in response:
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download'}.items())
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download', 'fl': '\[\W*F\W?L\W*\]'
}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['search'] % (search_string, self._categories_string())
search_url = self.urls['search'] % (search_string, self.categories)
html = self.get_url(search_url)
html = self.get_url(search_url, timeout=90)
cnt = len(items[mode])
try:
if not html or self._has_no_results(html):
raise generic.HaltParseException
with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
with BS4Parser(html, features=['html5lib', 'permissive'], attr='border="1"') as soup:
torrent_table = soup.find('table', attrs={'border': '1'})
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
@ -93,53 +79,46 @@ class TorrentBytesProvider(generic.TorrentProvider):
for tr in torrent_rows[1:]:
try:
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
info = tr.find('a', href=rc['info'])
seeders, leechers, size = [tryInt(n, n) for n in [
tr.find_all('td')[x].get_text().strip() for x in (-2, -1, -4)]]
if self.freeleech and (len(info.contents) < 2 or not rc['fl'].search(info.contents[1].string.strip())) \
or self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', href=rc['info'])
title = 'title' in info.attrs and info.attrs['title'] or info.get_text().strip()
title = 'title' in info.attrs and info.attrs['title'] or info.contents[0]
title = (isinstance(title, list) and title[0] or title).strip()
download_url = self.urls['get'] % str(tr.find('a', href=rc['get'])['href']).lstrip('/')
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, use_or=False)
class TorrentBytesCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = TorrentBytesProvider()

82
sickbeard/providers/torrentday.py
View file

@ -16,11 +16,11 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import time
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import tvcache
from sickbeard.helpers import (has_anime, tryInt)
class TorrentDayProvider(generic.TorrentProvider):
@ -34,90 +34,82 @@ class TorrentDayProvider(generic.TorrentProvider):
'search': self.url_base + 'V3/API/API.php',
'get': self.url_base + 'download.php/%s/%s'}
self.categories = {'Season': {'c14': 1},
'Episode': {'c2': 1, 'c26': 1, 'c7': 1, 'c24': 1},
'Cache': {'c2': 1, 'c26': 1, 'c7': 1, 'c24': 1, 'c14': 1}}
self.categories = {'Season': {'c31': 1, 'c33': 1, 'c14': 1},
'Episode': {'c32': 1, 'c26': 1, 'c7': 1, 'c2': 1},
'Cache': {'c31': 1, 'c33': 1, 'c14': 1, 'c32': 1, 'c26': 1, 'c7': 1, 'c2': 1}}
self.proper_search_terms = None
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = TorrentDayCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'uid' in self.session.cookies and 'pass' in self.session.cookies
if logged_in():
return True
return super(TorrentDayProvider, self)._authorised(
post_params={'submit.x': 0, 'submit.y': 0},
failed_msg=(lambda x=None: re.search(r'(?i)tried((<[^>]+>)|\W)*too((<[^>]+>)|\W)*often', x) and
u'Abort %s, Too many login attempts. Settings must be checked' or (
re.search(r'(?i)username((<[^>]+>)|\W)*or((<[^>]+>)|\W)*password', x) and
u'Invalid username or password for %s. Check settings' or
u'Failed to authenticate with %s, abort provider')))
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'submit.x': 0, 'submit.y': 0}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate'
if response and 'tried too often' in response:
msg = u'Too many login attempts'
logger.log(u'%s, abort provider %s' % (msg, self.name), logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = '+'.join(search_string.split())
post_data = dict({'/browse.php?': None, 'cata': 'yes', 'jxt': 8, 'jxw': 'b', 'search': search_string},
**self.categories[mode])
**self.categories[(mode, 'Episode')['Propers' == mode]])
if ('Cache' == mode and has_anime()) or (
mode in ['Season', 'Episode'] and self.show and self.show.is_anime):
post_data.update({'c29': 1})
if self.freeleech:
post_data.update({'free': 'on'})
data_json = self.get_url(self.urls['search'], post_data=post_data, json=True)
cnt = len(items[mode])
try:
if not data_json:
raise generic.HaltParseException
torrents = data_json.get('Fs', [])[0].get('Cn', {}).get('torrents', [])
torrents = data_json.get('Fs')[0].get('Cn').get('torrents')
for torrent in torrents:
seeders, leechers = int(torrent['seed']), int(torrent['leech'])
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
for item in torrents:
seeders, leechers, size = [tryInt(n, n) for n in [item.get(x) for x in 'seed', 'leech', 'size']]
if self._peers_fail(mode, seeders, leechers):
continue
title = re.sub(r'\[.*=.*\].*\[/.*\]', '', torrent['name'])
title = re.sub(r'\[.*=.*\].*\[/.*\]', '', item['name'])
download_url = self.urls['get'] % (torrent['id'], torrent['fname'])
download_url = self.urls['get'] % (item['id'], item['fname'])
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except Exception:
time.sleep(1.1)
self._log_result(mode, len(items[mode]) - cnt,
self._log_search(mode, len(items[mode]) - cnt,
('search string: ' + search_string, self.name)['Cache' == mode])
# For each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _episode_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date, '')
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, sep_date='.')
return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='.', date_or=True, **kwargs)
class TorrentDayCache(tvcache.TVCache):
@ -125,9 +117,9 @@ class TorrentDayCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
def _getRSSData(self):
def _cache_data(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = TorrentDayProvider()

83
sickbeard/providers/torrenting.py
View file

@ -16,12 +16,12 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -34,53 +34,33 @@ class TorrentingProvider(generic.TorrentProvider):
self.api = 'https://ttonline.us/'
self.urls = {'config_provider_home_uri': self.url_base,
'login_test': self.api + 'rss.php',
'search': self.api + 'browse.php?%ssearch=%s',
'login': self.api + 'secure.php',
'search': self.api + 'browse.php?%s&search=%s',
'get': self.api + '%s'}
self.categories = 'c4=1&c5=1&'
self.categories = {'shows': [4, 5]}
self.url = self.urls['config_provider_home_uri']
self.digest, self.minseed, self.minleech = 3 * [None]
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.cache = TorrentingCache(self)
def _do_login(self):
logged_in = lambda: 'uid' in self.session.cookies and self.session.cookies['uid'] in self.digest and \
'pass' in self.session.cookies and self.session.cookies['pass'] in self.digest
if logged_in():
return True
self.cookies = re.sub(r'(?i)([\s\']+|cookie\s*:)', '', self.digest)
success, msg = self._check_cookie()
if not success:
logger.log(u'%s: [%s]' % (msg, self.cookies), logger.WARNING)
else:
response = helpers.getURL(self.urls['login_test'], session=self.session)
if response and logged_in() and 'Generate RSS' in response[8550:]:
return True
logger.log(u'Invalid cookie details for %s. Check settings' % self.name, logger.ERROR)
self.cookies = None
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download',
'cats': 'cat=(?:4|5)'}.items())
'cats': 'cat=(?:%s)' % self._categories_string(template='', delimiter='|')
}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = self.urls['search'] % (self._categories_string(), search_string)
search_url = self.urls['search'] % (self.categories, search_string)
html = self.get_url(search_url)
cnt = len(items[mode])
@ -97,57 +77,42 @@ class TorrentingProvider(generic.TorrentProvider):
for tr in torrent_rows[1:]:
try:
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
if None is tr.find('a', href=rc['cats'])\
or ('Cache' != mode and (seeders < self.minseed or leechers < self.minleech)):
seeders, leechers, size = [tryInt(n, n) for n in [
tr.find_all('td')[x].get_text().strip() for x in (-2, -1, -3)]]
if None is tr.find('a', href=rc['cats']) or self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', href=rc['info'])
title = 'title' in info.attrs and info.attrs['title'] or info.get_text().strip()
download_url = self.urls['get'] % tr.find('a', href=rc['get']).get('href')
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
results += items[mode]
self._sort_seeders(mode, items)
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, use_or=False)
@staticmethod
def ui_string(key):
result = ''
if 'torrenting_digest' == key:
result = 'use... \'uid=xx; pass=yy\''
return result
class TorrentingCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 7 # cache update frequency
def _cache_data(self):
def _getRSSData(self):
return self.provider.get_cache_data()
return self.provider.cache_data()
provider = TorrentingProvider()

72
sickbeard/providers/torrentleech.py
View file

@ -16,69 +16,51 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from lib.unidecode import unidecode
class TorrentLeechProvider(generic.TorrentProvider):
def __init__(self):
generic.TorrentProvider.__init__(self, 'TorrentLeech')
self.url_base = 'https://torrentleech.org/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'user/account/login/',
'search': self.url_base + 'torrents/browse/index/query/%s/categories/%s',
'cache': self.url_base + 'torrents/browse/index/categories/%s',
'browse': self.url_base + 'torrents/browse/index/categories/%(cats)s',
'search': self.url_base + 'torrents/browse/index/query/%(query)s/categories/%(cats)s',
'get': self.url_base + '%s'}
self.categories = '2,26,27,32'
self.categories = {'shows': [2, 26, 27, 32], 'anime': [7, 34, 35]}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.cache = TorrentLeechCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'tluid' in self.session.cookies and 'tlpass' in self.session.cookies
if logged_in():
return True
return super(TorrentLeechProvider, self)._authorised(logged_in=(lambda x=None: self.has_all_cookies(pre='tl')),
post_params={'remember_me': 'on', 'login': 'submit'})
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'remember_me': 'on', 'login': 'submit'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
logger.log(u'Failed to authenticate with %s, abort provider.' % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'get': 'download'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
if 'Cache' == mode:
search_url = self.urls['cache'] % self.categories
else:
search_url = self.urls['search'] % (search_string, self.categories)
search_url = self.urls[('search', 'browse')['Cache' == mode]] % {
'cats': self._categories_string(mode, '', ','),
'query': isinstance(search_string, unicode) and unidecode(search_string) or search_string}
html = self.get_url(search_url)
@ -98,49 +80,45 @@ class TorrentLeechProvider(generic.TorrentProvider):
try:
seeders, leechers = [int(tr.find('td', attrs={'class': x}).get_text().strip())
for x in ('seeders', 'leechers')]
if mode != 'Cache' and (seeders < self.minseed or leechers < self.minleech):
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('td', {'class': 'name'}).a
title = ('title' in info.attrs and info['title']) or info.get_text().strip()
size = tr.find_all('td')[-5].get_text().strip()
download_url = self.urls['get'] % str(tr.find('a', href=rc['get'])['href']).lstrip('/')
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _episode_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, sep_date='|', use_or=False)
return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='|', **kwargs)
class TorrentLeechCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.cache_data()
return self.provider.get_cache_data()
provider = TorrentLeechProvider()

77
sickbeard/providers/torrentshack.py
View file

@ -18,12 +18,12 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -35,54 +35,38 @@ class TorrentShackProvider(generic.TorrentProvider):
self.url_base = 'https://torrentshack.me/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'login.php?lang=',
'search': self.url_base + 'torrents.php?searchstr=%s'
+ '&release_type=both&searchtags=&tags_type=0&order_by=s3&order_way=desc&torrent_preset=all'
+ '&filter_cat[600]=1&filter_cat[620]=1&filter_cat[700]=1'
+ '&filter_cat[850]=1&filter_cat[980]=1&filter_cat[981]=1',
'search': self.url_base + 'torrents.php?searchstr=%s&%s&' + '&'.join(
['release_type=both', 'searchtags=', 'tags_type=0', 'order_by=s3', 'order_way=desc', 'torrent_preset=all']),
'get': self.url_base + '%s'}
self.categories = {'shows': [600, 620, 700, 981, 980], 'anime': [850]}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.cache = TorrentShackCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'session' in self.session.cookies
if logged_in():
return True
return super(TorrentShackProvider, self)._authorised(logged_in=(lambda x=None: self.has_all_cookies('session')),
post_params={'keeplogged': '1', 'login': 'Login'})
if self._check_auth():
login_params = {'username': self.username, 'password': self.password, 'keeplogged': '1', 'login': 'Login'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
msg = u'Failed to authenticate with %s, abort provider'
if response and 'username or password was incorrect' in response:
msg = u'Invalid username or password for %s. Check settings'
logger.log(msg % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v))
for (k, v) in {'info': 'view', 'get': 'download', 'title': 'view\s+torrent\s+'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
# fetch 15 results by default, and up to 100 if allowed in user profile
search_url = self.urls['search'] % search_string
search_url = self.urls['search'] % (search_string, self._categories_string(mode, 'filter_cat[%s]=1'))
html = self.get_url(search_url)
cnt = len(items[mode])
@ -99,8 +83,9 @@ class TorrentShackProvider(generic.TorrentProvider):
for tr in torrent_rows[1:]:
try:
seeders, leechers = [int(tr.find_all('td')[x].get_text().strip()) for x in (-2, -1)]
if 'Cache' != mode and (seeders < self.minseed or leechers < self.minleech):
seeders, leechers, size = [tryInt(n, n) for n in [
tr.find_all('td')[x].get_text().strip() for x in (-2, -1, -4)]]
if self._peers_fail(mode, seeders, leechers):
continue
info = tr.find('a', title=rc['info'])
@ -109,32 +94,27 @@ class TorrentShackProvider(generic.TorrentProvider):
link = str(tr.find('a', title=rc['get'])['href']).replace('&amp;', '&').lstrip('/')
download_url = self.urls['get'] % link
except (AttributeError, TypeError):
except (AttributeError, TypeError, ValueError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
# for each search mode sort all the items by seeders
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _episode_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, sep_date='.', use_or=False)
return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='.', **kwargs)
class TorrentShackCache(tvcache.TVCache):
@ -142,10 +122,11 @@ class TorrentShackCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 20 # cache update frequency
self.update_freq = 20 # cache update frequency
def _getRSSData(self):
def _cache_data(self):
return self.provider.cache_data()
return self.provider.get_cache_data()
provider = TorrentShackProvider()

133
sickbeard/providers/transmithe_net.py
View file

@ -16,12 +16,11 @@
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import datetime
import traceback
from . import generic
from sickbeard import logger, tvcache, helpers
from sickbeard.bs4_parser import BS4Parser
from sickbeard import helpers, logger, tvcache
from sickbeard.helpers import tryInt
from lib.unidecode import unidecode
@ -32,111 +31,92 @@ class TransmithenetProvider(generic.TorrentProvider):
self.url_base = 'https://transmithe.net/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'index.php?page=login',
'cache': self.url_base + 'index.php?page=torrents&options=0&active=1',
'search': '&search=%s',
'get': self.url_base + '%s'}
'login_action': self.url_base + 'login.php',
'user': self.url_base + 'ajax.php?action=index',
'browse': self.url_base + 'ajax.php?action=browse&auth=%s&passkey=%s',
'search': '&searchstr=%s',
'get': self.url_base + 'torrents.php?action=download&authkey=%s&torrent_pass=%s&id=%s'}
self.url = self.urls['config_provider_home_uri']
self.user_authkey, self.user_passkey = 2 * [None]
self.username, self.password, self.minseed, self.minleech = 4 * [None]
self.freeleech = False
self.cache = TransmithenetCache(self)
def _do_login(self):
def _authorised(self, **kwargs):
logged_in = lambda: 'uid' in self.session.cookies and 'pass' in self.session.cookies
if logged_in():
return True
if not super(TransmithenetProvider, self)._authorised(
logged_in=(lambda x=None: self.has_all_cookies('session')),
post_params={'keeplogged': '1', 'login': 'Login'}):
return False
if not self.user_authkey:
response = helpers.getURL(self.urls['user'], session=self.session, json=True)
if 'response' in response:
self.user_authkey, self.user_passkey = [response['response'].get(v) for v in 'authkey', 'passkey']
return self.user_authkey
if self._check_auth():
login_params = {'uid': self.username, 'pwd': self.password, 'remember_me': 'on', 'login': 'submit'}
response = helpers.getURL(self.urls['login'], post_data=login_params, session=self.session)
if response and logged_in():
return True
logger.log(u'Failed to authenticate with %s, abort provider.' % self.name, logger.ERROR)
return False
def _do_search(self, search_params, search_mode='eponly', epcount=0, age=0):
def _search_provider(self, search_params, **kwargs):
results = []
if not self._do_login():
if not self._authorised():
return results
items = {'Season': [], 'Episode': [], 'Cache': []}
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'torrent-details', 'get': 'download',
'peers': 'page=peers', 'nodots': '[\.\s]+'}.items())
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'nodots': '[\.\s]+'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
if isinstance(search_string, unicode):
search_string = unidecode(search_string)
search_url = self.urls['cache']
search_url = self.urls['browse'] % (self.user_authkey, self.user_passkey)
if 'Cache' != mode:
search_url += self.urls['search'] % rc['nodots'].sub(' ', search_string)
search_url += self.urls['search'] % rc['nodots'].sub('+', search_string)
html = self.get_url(search_url)
data_json = self.get_url(search_url, json=True)
cnt = len(items[mode])
try:
if not html or self._has_no_results(html):
raise generic.HaltParseException
for item in data_json['response']['results']:
if self.freeleech and not item.get('isFreeleech'):
continue
with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
torrent_table = soup.find_all('table', 'lista')[-1]
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
seeders, leechers, group_name, torrent_id, size = [tryInt(n, n) for n in [item.get(x) for x in [
'seeders', 'leechers', 'groupName', 'torrentId', 'size']]]
if self._peers_fail(mode, seeders, leechers):
continue
if 2 > len(torrent_rows):
raise generic.HaltParseException
try:
title_parts = group_name.split('[')
maybe_res = re.findall('((?:72|108)0\w)', title_parts[1])
detail = title_parts[1].split('/')
detail[1] = detail[1].strip().lower().replace('mkv', 'x264')
title = '%s.%s' % (title_parts[0].strip(), '.'.join(
(len(maybe_res) and [maybe_res[0]] or []) + [detail[0].strip(), detail[1]]))
except (IndexError, KeyError):
title = group_name
download_url = self.urls['get'] % (self.user_authkey, self.user_passkey, torrent_id)
for tr in torrent_rows[1:]:
if tr.find('td', class_='header'):
continue
downlink = tr.find('a', href=rc['get'])
if None is downlink:
continue
try:
seeders, leechers = [int(x.get_text().strip()) for x in tr.find_all('a', href=rc['peers'])]
if mode != 'Cache' and (seeders < self.minseed or leechers < self.minleech):
continue
if title and download_url:
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
info = tr.find('a', href=rc['info'])
title = ('data-src' in info.attrs and info['data-src']) or\
('title' in info.attrs and info['title']) or info.get_text().strip()
download_url = self.urls['get'] % str(downlink['href']).lstrip('/')
except (AttributeError, TypeError):
continue
if title and download_url:
items[mode].append((title, download_url, seeders))
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_result(mode, len(items[mode]) - cnt, search_url)
self._log_search(mode, len(items[mode]) - cnt, search_url)
items[mode].sort(key=lambda tup: tup[2], reverse=True)
self._sort_seeders(mode, items)
results += items[mode]
results = list(set(results + items[mode]))
return results
def find_propers(self, search_date=datetime.datetime.today()):
def _season_strings(self, ep_obj, **kwargs):
return self._find_propers(search_date)
return generic.TorrentProvider._season_strings(self, ep_obj, scene=False)
def _get_season_search_strings(self, ep_obj, **kwargs):
def _episode_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._get_season_search_strings(self, ep_obj, scene=False)
def _get_episode_search_strings(self, ep_obj, add_string='', **kwargs):
return generic.TorrentProvider._get_episode_search_strings(self, ep_obj, add_string, scene=False, use_or=False)
return generic.TorrentProvider._episode_strings(self, ep_obj, scene=False, **kwargs)
class TransmithenetCache(tvcache.TVCache):
@ -144,10 +124,11 @@ class TransmithenetCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 17 # cache update frequency
self.update_freq = 17
def _getRSSData(self):
def _cache_data(self):
return self.provider.cache_data()
return self.provider.get_cache_data()
provider = TransmithenetProvider()

189
sickbeard/providers/tvchaosuk.py Normal file
View file

@ -0,0 +1,189 @@
# coding=utf-8
#
# This file is part of SickGear.
#
# SickGear is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# SickGear is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import re
import traceback
from . import generic
from sickbeard import logger, tvcache
from sickbeard.bs4_parser import BS4Parser
from sickbeard.helpers import tryInt
from sickbeard.config import naming_ep_type
from dateutil.parser import parse
from lib.unidecode import unidecode
class TVChaosUKProvider(generic.TorrentProvider):
def __init__(self):
generic.TorrentProvider.__init__(self, 'TVChaosUK')
self.url_base = 'https://tvchaosuk.com/'
self.urls = {'config_provider_home_uri': self.url_base,
'login': self.url_base + 'takelogin.php',
'search': self.url_base + 'browse.php',
'get': self.url_base + '%s'}
self.url = self.urls['config_provider_home_uri']
self.username, self.password, self.freeleech, self.minseed, self.minleech = 5 * [None]
self.search_fallback = True
self.cache = TVChaosUKCache(self)
def _authorised(self, **kwargs):
return super(TVChaosUKProvider, self)._authorised(logged_in=(lambda x=None: self.has_all_cookies(pre='c_secure_')))
def _search_provider(self, search_params, **kwargs):
results = []
if not self._authorised():
return results
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'info': 'detail', 'get': 'download', 'fl': 'free'}.items())
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
if 'Cache' != mode:
kwargs = dict(post_data={'keywords': search_string, 'do': 'quick_sort', 'page': '0',
'category': '0', 'search_type': 't_name', 'sort': 'added',
'order': 'desc', 'daysprune': '-1'})
html = self.get_url(self.urls['search'], **kwargs)
cnt = len(items[mode])
try:
if not html or self._has_no_results(html):
raise generic.HaltParseException
with BS4Parser(html, 'html.parser') as soup:
torrent_table = soup.find('table', id='sortabletable')
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
get_detail = True
if 2 > len(torrent_rows):
raise generic.HaltParseException
for tr in torrent_rows[1:]:
try:
seeders, leechers, size = [tryInt(n, n) for n in [
tr.find_all('td')[x].get_text().strip() for x in (-3, -2, -5)]]
if self._peers_fail(mode, seeders, leechers) \
or self.freeleech and None is tr.find_all('td')[1].find('img', title=rc['fl']):
continue
info = tr.find('a', href=rc['info'])
title = (tr.find('div', attrs={'class': 'tooltip-content'}).get_text() or info.get_text()).strip()
title = re.findall('(?m)(^[^\r\n]+)', title)[0]
download_url = self.urls['get'] % str(tr.find('a', href=rc['get'])['href']).lstrip(
'/').replace(self.urls['config_provider_home_uri'], '')
except Exception:
continue
if get_detail and title.endswith('...'):
try:
with BS4Parser(self.get_url('%s%s' % (self.urls['config_provider_home_uri'], info['href'].lstrip(
'/').replace(self.urls['config_provider_home_uri'], ''))), 'html.parser') as soup_detail:
title = soup_detail.find('td', attrs={'colspan': '3', 'class': 'thead'}).get_text().strip()
title = re.findall('(?m)(^[^\r\n]+)', title)[0]
except IndexError:
continue
except Exception:
get_detail = False
try:
has_series = re.findall('(?i)(.*?series[^\d]*?\d+)(.*)', title)
if has_series:
rc_xtras = re.compile('(?i)([. _-]|^)(special|extra)s?\w*([. _-]|$)')
has_special = rc_xtras.findall(has_series[0][1])
if has_special:
title = has_series[0][0] + rc_xtras.sub(list(
set(list(has_special[0][0]) + list(has_special[0][2])))[0], has_series[0][1])
title = re.sub('(?i)series', r'Season', title)
title_parts = re.findall('(?im)^(.*?)(?:Season[^\d]*?(\d+).*?)?(?:(?:pack|part|pt)\W*?)?(\d+)[^\d]*?of[^\d]*?(?:\d+)(.*?)$', title)
if len(title_parts):
new_parts = [tryInt(part, part.strip()) for part in title_parts[0]]
if not new_parts[1]:
new_parts[1] = 1
new_parts[2] = ('E%02d', ' Pack %d')[mode in 'Season'] % new_parts[2]
title = '%s.S%02d%s.%s' % tuple(new_parts)
dated = re.findall('(?i)([\(\s]*)((?:\d\d\s)?[adfjmnos]\w{2,}\s+(?:19|20)\d\d)([\)\s]*)', title)
if dated:
title = title.replace(''.join(dated[0]), '%s%s%s' % (
('', ' ')[1 < len(dated[0][0])], parse(dated[0][1]).strftime('%Y-%m-%d'),
('', ' ')[1 < len(dated[0][2])]))
add_pad = re.findall('((?:19|20)\d\d\-\d\d\-\d\d)([\w\W])', title)
if len(add_pad) and add_pad[0][1] not in [' ', '.']:
title = title.replace(''.join(add_pad[0]), '%s %s' % (add_pad[0][0], add_pad[0][1]))
if title and download_url:
items[mode].append((title, download_url, seeders, self._bytesizer(size)))
except Exception:
pass
except generic.HaltParseException:
pass
except Exception:
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
self._log_search(mode, len(items[mode]) - cnt,
('search string: ' + search_string.replace('%', ' '), self.name)['Cache' == mode])
if mode in 'Season' and len(items[mode]):
break
self._sort_seeders(mode, items)
results = list(set(results + items[mode]))
return results
def _season_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._season_strings(self, ep_obj, scene=False, prefix='%', sp_detail=(
lambda e: [(('', 'Series %(seasonnumber)d%%')[1 < tryInt(e.get('seasonnumber'))] + '%(episodenumber)dof') % e,
'Series %(seasonnumber)d' % e]))
def _episode_strings(self, ep_obj, **kwargs):
return generic.TorrentProvider._episode_strings(self, ep_obj, scene=False, prefix='%', date_detail=(
lambda d: [d.strftime('%d %b %Y')] + ([d.strftime('%d %B %Y')], [])[d.strftime('%b') == d.strftime('%B')]),
ep_detail=(lambda e: [naming_ep_type[2] % e] + (
[], ['%(episodenumber)dof' % e])[1 == tryInt(e.get('seasonnumber'))]), **kwargs)
@staticmethod
def ui_string(key):
return 'tvchaosuk_tip' == key and 'has missing quality data so you must add quality Custom/Unknown to any wanted show' or ''
class TVChaosUKCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
def _cache_data(self):
return self.provider.cache_data()
provider = TVChaosUKProvider()

7
sickbeard/providers/womble.py
View file

@ -16,8 +16,7 @@
# You should have received a copy of the GNU General Public License
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
import generic
from . import generic
from sickbeard import logger, tvcache
@ -35,7 +34,7 @@ class WombleCache(tvcache.TVCache):
def __init__(self, this_provider):
tvcache.TVCache.__init__(self, this_provider)
self.minTime = 15 # cache update frequency
self.update_freq = 15 # cache update frequency
def updateCache(self):
@ -60,7 +59,7 @@ class WombleCache(tvcache.TVCache):
# By now we know we've got data and no auth errors, all we need to do is put it in the database
for item in data.entries:
title, url = self._get_title_and_url(item)
title, url = self._title_and_url(item)
ci = self._parseItem(title, url)
if None is not ci:
cl.append(ci)

676
sickbeard/search.py
View file

@ -40,11 +40,10 @@ from sickbeard import encodingKludge as ek
from sickbeard import failed_history
from sickbeard.exceptions import ex
from sickbeard.providers.generic import GenericProvider
from sickbeard.blackandwhitelist import BlackAndWhiteList
from sickbeard import common
def _downloadResult(result):
def _download_result(result):
"""
Downloads a result to the appropriate black hole folder.
@ -53,44 +52,44 @@ def _downloadResult(result):
result: SearchResult instance to download.
"""
resProvider = result.provider
if resProvider == None:
logger.log(u"Invalid provider name - this is a coding error, report it please", logger.ERROR)
res_provider = result.provider
if None is res_provider:
logger.log(u'Invalid provider name - this is a coding error, report it please', logger.ERROR)
return False
# nzbs with an URL can just be downloaded from the provider
if result.resultType == "nzb":
newResult = resProvider.download_result(result)
if 'nzb' == result.resultType:
new_result = res_provider.download_result(result)
# if it's an nzb data result
elif result.resultType == "nzbdata":
elif 'nzbdata' == result.resultType:
# get the final file path to the nzb
fileName = ek.ek(os.path.join, sickbeard.NZB_DIR, result.name + ".nzb")
file_name = ek.ek(os.path.join, sickbeard.NZB_DIR, u'%s.nzb' % result.name)
logger.log(u"Saving NZB to " + fileName)
logger.log(u'Saving NZB to %s' % file_name)
newResult = True
new_result = True
# save the data to disk
try:
with ek.ek(open, fileName, 'w') as fileOut:
fileOut.write(result.extraInfo[0])
with ek.ek(open, file_name, 'w') as file_out:
file_out.write(result.extraInfo[0])
helpers.chmodAsParent(fileName)
helpers.chmodAsParent(file_name)
except EnvironmentError as e:
logger.log(u"Error trying to save NZB to black hole: " + ex(e), logger.ERROR)
newResult = False
elif resProvider.providerType == "torrent":
newResult = resProvider.download_result(result)
logger.log(u'Error trying to save NZB to black hole: %s' % ex(e), logger.ERROR)
new_result = False
elif 'torrent' == res_provider.providerType:
new_result = res_provider.download_result(result)
else:
logger.log(u"Invalid provider type - this is a coding error, report it please", logger.ERROR)
newResult = False
logger.log(u'Invalid provider type - this is a coding error, report it please', logger.ERROR)
new_result = False
return newResult
return new_result
def snatchEpisode(result, endStatus=SNATCHED):
def snatch_episode(result, end_status=SNATCHED):
"""
Contains the internal logic necessary to actually "snatch" a result that
has been found.
@ -101,158 +100,140 @@ def snatchEpisode(result, endStatus=SNATCHED):
endStatus: the episode status that should be used for the episode object once it's snatched.
"""
if result is None:
if None is result:
return False
result.priority = 0 # -1 = low, 0 = normal, 1 = high
if sickbeard.ALLOW_HIGH_PRIORITY:
# if it aired recently make it high priority
for curEp in result.episodes:
if datetime.date.today() - curEp.airdate <= datetime.timedelta(days=7):
for cur_ep in result.episodes:
if datetime.date.today() - cur_ep.airdate <= datetime.timedelta(days=7):
result.priority = 1
if re.search('(^|[\. _-])(proper|repack)([\. _-]|$)', result.name, re.I) != None:
endStatus = SNATCHED_PROPER
if None is not re.search('(^|[\. _-])(proper|repack)([\. _-]|$)', result.name, re.I):
end_status = SNATCHED_PROPER
# NZBs can be sent straight to SAB or saved to disk
if result.resultType in ("nzb", "nzbdata"):
if sickbeard.NZB_METHOD == "blackhole":
dlResult = _downloadResult(result)
elif sickbeard.NZB_METHOD == "sabnzbd":
dlResult = sab.sendNZB(result)
elif sickbeard.NZB_METHOD == "nzbget":
is_proper = True if endStatus == SNATCHED_PROPER else False
dlResult = nzbget.sendNZB(result, is_proper)
if result.resultType in ('nzb', 'nzbdata'):
if 'blackhole' == sickbeard.NZB_METHOD:
dl_result = _download_result(result)
elif 'sabnzbd' == sickbeard.NZB_METHOD:
dl_result = sab.sendNZB(result)
elif 'nzbget' == sickbeard.NZB_METHOD:
is_proper = True if SNATCHED_PROPER == end_status else False
dl_result = nzbget.sendNZB(result, is_proper)
else:
logger.log(u"Unknown NZB action specified in config: " + sickbeard.NZB_METHOD, logger.ERROR)
dlResult = False
logger.log(u'Unknown NZB action specified in config: %s' % sickbeard.NZB_METHOD, logger.ERROR)
dl_result = False
# TORRENTs can be sent to clients or saved to disk
elif result.resultType == "torrent":
elif 'torrent' == result.resultType:
# torrents are saved to disk when blackhole mode
if sickbeard.TORRENT_METHOD == "blackhole":
dlResult = _downloadResult(result)
if 'blackhole' == sickbeard.TORRENT_METHOD:
dl_result = _download_result(result)
else:
# make sure we have the torrent file content
if not result.content and not result.url.startswith('magnet'):
result.content = result.provider.get_url(result.url)
if not result.content:
logger.log(u'Torrent content failed to download from ' + result.url, logger.ERROR)
logger.log(u'Torrent content failed to download from %s' % result.url, logger.ERROR)
return False
# Snatches torrent with client
client = clients.getClientIstance(sickbeard.TORRENT_METHOD)()
dlResult = client.sendTORRENT(result)
dl_result = client.sendTORRENT(result)
else:
logger.log(u"Unknown result type, unable to download it", logger.ERROR)
dlResult = False
logger.log(u'Unknown result type, unable to download it', logger.ERROR)
dl_result = False
if not dlResult:
if not dl_result:
return False
if sickbeard.USE_FAILED_DOWNLOADS:
failed_history.logSnatch(result)
ui.notifications.message('Episode snatched', result.name)
ui.notifications.message(u'Episode snatched', result.name)
history.logSnatch(result)
# don't notify when we re-download an episode
sql_l = []
for curEpObj in result.episodes:
with curEpObj.lock:
if isFirstBestMatch(result):
curEpObj.status = Quality.compositeStatus(SNATCHED_BEST, result.quality)
update_imdb_data = True
for cur_ep_obj in result.episodes:
with cur_ep_obj.lock:
if is_first_best_match(result):
cur_ep_obj.status = Quality.compositeStatus(SNATCHED_BEST, result.quality)
else:
curEpObj.status = Quality.compositeStatus(endStatus, result.quality)
cur_ep_obj.status = Quality.compositeStatus(end_status, result.quality)
item = curEpObj.get_sql()
item = cur_ep_obj.get_sql()
if None is not item:
sql_l.append(item)
if curEpObj.status not in Quality.DOWNLOADED:
notifiers.notify_snatch(curEpObj._format_pattern('%SN - %Sx%0E - %EN - %QN'))
if cur_ep_obj.status not in Quality.DOWNLOADED:
notifiers.notify_snatch(cur_ep_obj._format_pattern('%SN - %Sx%0E - %EN - %QN'))
curEpObj.show.load_imdb_info()
update_imdb_data = update_imdb_data and cur_ep_obj.show.load_imdb_info()
if 0 < len(sql_l):
myDB = db.DBConnection()
myDB.mass_action(sql_l)
my_db = db.DBConnection()
my_db.mass_action(sql_l)
return True
def filter_release_name(name, filter_words):
"""
Filters out results based on filter_words
name: name to check
filter_words : Words to filter on, separated by comma
Returns: False if the release name is OK, True if it contains one of the filter_words
"""
if filter_words:
filters = [re.compile('.*%s.*' % filter.strip(), re.I) for filter in filter_words.split(',')]
for regfilter in filters:
if regfilter.search(name):
logger.log(u"" + name + " contains pattern: " + regfilter.pattern, logger.DEBUG)
return True
return False
def pickBestResult(results, show, quality_list=None):
logger.log(u"Picking the best result out of " + str([x.name for x in results]), logger.DEBUG)
def pick_best_result(results, show, quality_list=None):
logger.log(u'Picking the best result out of %s' % [x.name for x in results], logger.DEBUG)
# find the best result for the current episode
bestResult = None
best_result = None
for cur_result in results:
logger.log("Quality of " + cur_result.name + " is " + Quality.qualityStrings[cur_result.quality])
if show.is_anime:
if not show.release_groups.is_valid(cur_result):
continue
logger.log(u'Quality of %s is %s' % (cur_result.name, Quality.qualityStrings[cur_result.quality]))
if show.is_anime and not show.release_groups.is_valid(cur_result):
continue
if quality_list and cur_result.quality not in quality_list:
logger.log(cur_result.name + " is a quality we know we don't want, rejecting it", logger.DEBUG)
logger.log(u'%s is an unwanted quality, rejecting it' % cur_result.name, logger.DEBUG)
continue
if show.rls_ignore_words and filter_release_name(cur_result.name, show.rls_ignore_words):
logger.log(u"Ignoring " + cur_result.name + " based on ignored words filter: " + show.rls_ignore_words,
logger.MESSAGE)
re_extras = dict(re_prefix='.*', re_suffix='.*')
result = show_name_helpers.contains_any(cur_result.name, show.rls_ignore_words, **re_extras)
if None is not result and result:
logger.log(u'Ignored: %s for containing ignore word' % cur_result.name)
continue
if show.rls_require_words and not filter_release_name(cur_result.name, show.rls_require_words):
logger.log(u"Ignoring " + cur_result.name + " based on required words filter: " + show.rls_require_words,
logger.MESSAGE)
result = show_name_helpers.contains_any(cur_result.name, show.rls_require_words, **re_extras)
if None is not result and not result:
logger.log(u'Ignored: %s for not containing any required word match' % cur_result.name)
continue
cur_size = getattr(cur_result, 'size', None)
if sickbeard.USE_FAILED_DOWNLOADS and None is not cur_size and failed_history.hasFailed(
cur_result.name, cur_size, cur_result.provider.name):
logger.log(cur_result.name + u" has previously failed, rejecting it")
logger.log(u'%s has previously failed, rejecting it' % cur_result.name)
continue
if not bestResult or bestResult.quality < cur_result.quality and cur_result.quality != Quality.UNKNOWN:
bestResult = cur_result
if not best_result or best_result.quality < cur_result.quality != Quality.UNKNOWN:
best_result = cur_result
elif bestResult.quality == cur_result.quality:
if "proper" in cur_result.name.lower() or "repack" in cur_result.name.lower():
bestResult = cur_result
elif "internal" in bestResult.name.lower() and "internal" not in cur_result.name.lower():
bestResult = cur_result
elif "xvid" in bestResult.name.lower() and "x264" in cur_result.name.lower():
logger.log(u"Preferring " + cur_result.name + " (x264 over xvid)")
bestResult = cur_result
elif best_result.quality == cur_result.quality:
if re.search('(?i)(proper|repack)', cur_result.name) or \
show.is_anime and re.search('(?i)(v1|v2|v3|v4|v5)', cur_result.name):
best_result = cur_result
elif 'internal' in best_result.name.lower() and 'internal' not in cur_result.name.lower():
best_result = cur_result
elif 'xvid' in best_result.name.lower() and 'x264' in cur_result.name.lower():
logger.log(u'Preferring %s (x264 over xvid)' % cur_result.name)
best_result = cur_result
if bestResult:
logger.log(u"Picked " + bestResult.name + " as the best", logger.DEBUG)
if best_result:
logger.log(u'Picked %s as the best' % best_result.name, logger.DEBUG)
else:
logger.log(u"No result picked.", logger.DEBUG)
logger.log(u'No result picked.', logger.DEBUG)
return bestResult
return best_result
def isFinalResult(result):
def is_final_result(result):
"""
Checks if the given result is good enough quality that we can stop searching for other ones.
@ -261,16 +242,14 @@ def isFinalResult(result):
"""
logger.log(u"Checking if we should keep searching after we've found " + result.name, logger.DEBUG)
logger.log(u'Checking if searching should continue after finding %s' % result.name, logger.DEBUG)
show_obj = result.episodes[0].show
any_qualities, best_qualities = Quality.splitQuality(show_obj.quality)
# if there is a redownload that's higher than this then we definitely need to keep looking
if best_qualities and result.quality < max(best_qualities):
if best_qualities and max(best_qualities) > result.quality:
return False
# if it does not match the shows black and white list its no good
@ -281,10 +260,10 @@ def isFinalResult(result):
elif any_qualities and result.quality in any_qualities:
return True
elif best_qualities and result.quality == max(best_qualities):
elif best_qualities and max(best_qualities) == result.quality:
# if this is the best redownload but we have a higher initial download then keep looking
if any_qualities and result.quality < max(any_qualities):
if any_qualities and max(any_qualities) > result.quality:
return False
# if this is the best redownload and we don't have a higher initial download then we're done
@ -296,13 +275,12 @@ def isFinalResult(result):
return False
def isFirstBestMatch(result):
def is_first_best_match(result):
"""
Checks if the given result is a best quality match and if we want to archive the episode on first match.
"""
logger.log(u"Checking if we should archive our first best quality match for episode " + result.name,
logger.DEBUG)
logger.log(u'Checking if the first best quality match should be archived for episode %s' % result.name, logger.DEBUG)
show_obj = result.episodes[0].show
@ -315,25 +293,25 @@ def isFirstBestMatch(result):
return False
def wantedEpisodes(show, fromDate, make_dict=False):
initialQualities, archiveQualities = common.Quality.splitQuality(show.quality)
allQualities = list(set(initialQualities + archiveQualities))
def wanted_episodes(show, from_date, make_dict=False):
initial_qualities, archive_qualities = common.Quality.splitQuality(show.quality)
all_qualities = list(set(initial_qualities + archive_qualities))
myDB = db.DBConnection()
my_db = db.DBConnection()
if show.air_by_date:
sqlString = 'SELECT ep.status, ep.season, ep.episode, ep.airdate FROM [tv_episodes] AS ep, [tv_shows] AS show WHERE season != 0 AND ep.showid = show.indexer_id AND show.paused = 0 AND ep.showid = ? AND show.air_by_date = 1'
sql_string = 'SELECT ep.status, ep.season, ep.episode, ep.airdate FROM [tv_episodes] AS ep, [tv_shows] AS show WHERE season != 0 AND ep.showid = show.indexer_id AND show.paused = 0 AND ep.showid = ? AND show.air_by_date = 1'
else:
sqlString = 'SELECT status, season, episode, airdate FROM [tv_episodes] WHERE showid = ? AND season > 0'
sql_string = 'SELECT status, season, episode, airdate FROM [tv_episodes] WHERE showid = ? AND season > 0'
if sickbeard.SEARCH_UNAIRED:
statusList = [common.WANTED, common.FAILED, common.UNAIRED]
sqlString += ' AND ( airdate > ? OR airdate = 1 )'
status_list = [common.WANTED, common.FAILED, common.UNAIRED]
sql_string += ' AND ( airdate > ? OR airdate = 1 )'
else:
statusList = [common.WANTED, common.FAILED]
sqlString += ' AND airdate > ?'
status_list = [common.WANTED, common.FAILED]
sql_string += ' AND airdate > ?'
sqlResults = myDB.select(sqlString, [show.indexerid, fromDate.toordinal()])
sql_results = my_db.select(sql_string, [show.indexerid, from_date.toordinal()])
# check through the list of statuses to see if we want any
if make_dict:
@ -341,52 +319,59 @@ def wantedEpisodes(show, fromDate, make_dict=False):
else:
wanted = []
total_wanted = total_replacing = total_unaired = 0
downloadedStatusList = (common.DOWNLOADED, common.SNATCHED, common.SNATCHED_PROPER, common.SNATCHED_BEST)
for result in sqlResults:
downloaded_status_list = (common.DOWNLOADED, common.SNATCHED, common.SNATCHED_PROPER, common.SNATCHED_BEST)
for result in sql_results:
not_downloaded = True
curCompositeStatus = int(result["status"])
curStatus, curQuality = common.Quality.splitCompositeStatus(curCompositeStatus)
cur_composite_status = int(result['status'])
cur_status, cur_quality = common.Quality.splitCompositeStatus(cur_composite_status)
if show.archive_firstmatch and curStatus in downloadedStatusList and curQuality in archiveQualities:
if show.archive_firstmatch and cur_status in downloaded_status_list and cur_quality in archive_qualities:
continue
# special case: already downloaded quality is not in any of the wanted Qualities
other_quality_downloaded = False
if curStatus in downloadedStatusList and curQuality not in allQualities:
if cur_status in downloaded_status_list and cur_quality not in all_qualities:
other_quality_downloaded = True
wantedQualities = allQualities
wanted_qualities = all_qualities
else:
wantedQualities = archiveQualities
wanted_qualities = archive_qualities
if archiveQualities:
highestWantedQuality = max(wantedQualities)
if archive_qualities:
highest_wanted_quality = max(wanted_qualities)
else:
if other_quality_downloaded:
highestWantedQuality = max(initialQualities)
highest_wanted_quality = max(initial_qualities)
else:
highestWantedQuality = 0
highest_wanted_quality = 0
# if we need a better one then say yes
if (curStatus in downloadedStatusList and curQuality < highestWantedQuality) or curStatus in statusList or (sickbeard.SEARCH_UNAIRED and result['airdate'] == 1 and curStatus in (common.SKIPPED, common.IGNORED, common.UNAIRED, common.UNKNOWN, common.FAILED)):
if (cur_status in downloaded_status_list and cur_quality < highest_wanted_quality) or \
cur_status in status_list or \
(sickbeard.SEARCH_UNAIRED and 1 == result['airdate'] and cur_status in (common.SKIPPED, common.IGNORED,
common.UNAIRED, common.UNKNOWN,
common.FAILED)):
if curStatus in (common.WANTED, common.FAILED):
if cur_status in (common.WANTED, common.FAILED):
total_wanted += 1
elif curStatus in (common.UNAIRED, common.SKIPPED, common.IGNORED, common.UNKNOWN):
elif cur_status in (common.UNAIRED, common.SKIPPED, common.IGNORED, common.UNKNOWN):
total_unaired += 1
else:
total_replacing += 1
not_downloaded = False
epObj = show.getEpisode(int(result["season"]), int(result["episode"]))
ep_obj = show.getEpisode(int(result['season']), int(result['episode']))
if make_dict:
wanted.setdefault(epObj.season, []).append(epObj)
wanted.setdefault(ep_obj.season, []).append(ep_obj)
else:
epObj.wantedQuality = [i for i in (initialQualities if not_downloaded else wantedQualities) if (i > curQuality and i != common.Quality.UNKNOWN)]
wanted.append(epObj)
ep_obj.wantedQuality = [i for i in (initial_qualities if not_downloaded else
wanted_qualities) if (i > cur_quality and i != common.Quality.UNKNOWN)]
wanted.append(ep_obj)
if 0 < total_wanted + total_replacing + total_unaired:
actions = []
for msg, total in ['%d episode%s', total_wanted], ['to upgrade %d episode%s', total_replacing], ['%d unaired episode%s', total_unaired]:
for msg, total in ['%d episode%s', total_wanted], \
['to upgrade %d episode%s', total_replacing], \
['%d unaired episode%s', total_unaired]:
if 0 < total:
actions.append(msg % (total, helpers.maybe_plural(total)))
logger.log(u'We want %s for %s' % (' and '.join(actions), show.name))
@ -394,344 +379,331 @@ def wantedEpisodes(show, fromDate, make_dict=False):
return wanted
def searchForNeededEpisodes(episodes):
foundResults = {}
def search_for_needed_episodes(episodes):
found_results = {}
didSearch = False
search_done = False
origThreadName = threading.currentThread().name
orig_thread_name = threading.currentThread().name
providers = [x for x in sickbeard.providers.sortedProviderList() if x.is_active() and x.enable_recentsearch]
for curProvider in providers:
threading.currentThread().name = origThreadName + " :: [" + curProvider.name + "]"
for cur_provider in providers:
threading.currentThread().name = '%s :: [%s]' % (orig_thread_name, cur_provider.name)
curFoundResults = curProvider.search_rss(episodes)
cur_found_results = cur_provider.search_rss(episodes)
didSearch = True
search_done = True
# pick a single result for each episode, respecting existing results
for curEp in curFoundResults:
for cur_ep in cur_found_results:
if curEp.show.paused:
logger.log(
u"Show " + curEp.show.name + " is paused, ignoring all RSS items for " + curEp.prettyName(),
logger.DEBUG)
if cur_ep.show.paused:
logger.log(u'Show %s is paused, ignoring all RSS items for %s' % (cur_ep.show.name, cur_ep.prettyName()),
logger.DEBUG)
continue
# find the best result for the current episode
bestResult = pickBestResult(curFoundResults[curEp], curEp.show)
best_result = pick_best_result(cur_found_results[cur_ep], cur_ep.show)
# if all results were rejected move on to the next episode
if not bestResult:
logger.log(u"All found results for " + curEp.prettyName() + " were rejected.", logger.DEBUG)
if not best_result:
logger.log(u'All found results for %s were rejected.' % cur_ep.prettyName(), logger.DEBUG)
continue
# if it's already in the list (from another provider) and the newly found quality is no better then skip it
if curEp in foundResults and bestResult.quality <= foundResults[curEp].quality:
if cur_ep in found_results and best_result.quality <= found_results[cur_ep].quality:
continue
# filter out possible bad torrents from providers
if bestResult.resultType == "torrent" and sickbeard.TORRENT_METHOD != "blackhole":
bestResult.content = None
if not bestResult.url.startswith('magnet'):
bestResult.content = bestResult.provider.get_url(bestResult.url)
if not bestResult.content:
if 'torrent' == best_result.resultType and 'blackhole' != sickbeard.TORRENT_METHOD:
best_result.content = None
if not best_result.url.startswith('magnet'):
best_result.content = best_result.provider.get_url(best_result.url)
if not best_result.content:
continue
foundResults[curEp] = bestResult
threading.currentThread().name = origThreadName
found_results[cur_ep] = best_result
if not didSearch:
logger.log(
u"No NZB/Torrent providers found or enabled in the SickGear config for recent searches. Please check your settings.",
logger.ERROR)
threading.currentThread().name = orig_thread_name
return foundResults.values()
if not search_done:
logger.log(u'No NZB/Torrent provider enabled to do recent searches. Please check provider options.', logger.ERROR)
return found_results.values()
def searchProviders(show, episodes, manual_search=False):
foundResults = {}
finalResults = []
def search_providers(show, episodes, manual_search=False):
found_results = {}
final_results = []
didSearch = False
search_done = False
origThreadName = threading.currentThread().name
orig_thread_name = threading.currentThread().name
providers = [x for x in sickbeard.providers.sortedProviderList() if x.is_active() and x.enable_backlog]
for providerNum, curProvider in enumerate(providers):
if curProvider.anime_only and not show.is_anime:
logger.log(u"" + str(show.name) + " is not an anime, skipping", logger.DEBUG)
provider_list = [x for x in sickbeard.providers.sortedProviderList() if x.is_active() and x.enable_backlog]
for cur_provider in provider_list:
if cur_provider.anime_only and not show.is_anime:
logger.log(u'%s is not an anime, skipping' % show.name, logger.DEBUG)
continue
threading.currentThread().name = origThreadName + " :: [" + curProvider.name + "]"
threading.currentThread().name = '%s :: [%s]' % (orig_thread_name, cur_provider.name)
provider_id = cur_provider.get_id()
foundResults[curProvider.name] = {}
found_results[provider_id] = {}
searchCount = 0
search_mode = curProvider.search_mode
search_count = 0
search_mode = cur_provider.search_mode
while(True):
searchCount += 1
while True:
search_count += 1
if search_mode == 'eponly':
logger.log(u"Performing episode search for " + show.name)
if 'eponly' == search_mode:
logger.log(u'Performing episode search for %s' % show.name)
else:
logger.log(u"Performing season pack search for " + show.name)
logger.log(u'Performing season pack search for %s' % show.name)
try:
curProvider.cache._clearCache()
searchResults = curProvider.find_search_results(show, episodes, search_mode, manual_search)
cur_provider.cache._clearCache()
search_results = cur_provider.find_search_results(show, episodes, search_mode, manual_search)
except exceptions.AuthException as e:
logger.log(u"Authentication error: " + ex(e), logger.ERROR)
logger.log(u'Authentication error: %s' % ex(e), logger.ERROR)
break
except Exception as e:
logger.log(u"Error while searching " + curProvider.name + ", skipping: " + ex(e), logger.ERROR)
logger.log(u'Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR)
logger.log(traceback.format_exc(), logger.DEBUG)
break
finally:
threading.currentThread().name = origThreadName
threading.currentThread().name = orig_thread_name
didSearch = True
search_done = True
if len(searchResults):
if len(search_results):
# make a list of all the results for this provider
for curEp in searchResults:
for cur_ep in search_results:
# skip non-tv crap
searchResults[curEp] = filter(
lambda x: show_name_helpers.filterBadReleases(x.name, parse=False) and x.show == show, searchResults[curEp])
search_results[cur_ep] = filter(
lambda item: show_name_helpers.pass_wordlist_checks(item.name, parse=False) and
item.show == show, search_results[cur_ep])
if curEp in foundResults:
foundResults[curProvider.name][curEp] += searchResults[curEp]
if cur_ep in found_results:
found_results[provider_id][cur_ep] += search_results[cur_ep]
else:
foundResults[curProvider.name][curEp] = searchResults[curEp]
found_results[provider_id][cur_ep] = search_results[cur_ep]
break
elif not curProvider.search_fallback or searchCount == 2:
elif not cur_provider.search_fallback or search_count == 2:
break
if search_mode == 'sponly':
logger.log(u"FALLBACK EPISODE SEARCH INITIATED ...")
search_mode = 'eponly'
else:
logger.log(u"FALLBACK SEASON PACK SEARCH INITIATED ...")
search_mode = 'sponly'
search_mode = '%sonly' % ('ep', 'sp')['ep' in search_mode]
logger.log(u'Falling back to %s search ...' % ('season pack', 'episode')['ep' in search_mode])
# skip to next provider if we have no results to process
if not len(foundResults[curProvider.name]):
if not len(found_results[provider_id]):
continue
anyQualities, bestQualities = Quality.splitQuality(show.quality)
any_qualities, best_qualities = Quality.splitQuality(show.quality)
# pick the best season NZB
bestSeasonResult = None
if SEASON_RESULT in foundResults[curProvider.name]:
bestSeasonResult = pickBestResult(foundResults[curProvider.name][SEASON_RESULT], show,
anyQualities + bestQualities)
best_season_result = None
if SEASON_RESULT in found_results[provider_id]:
best_season_result = pick_best_result(found_results[provider_id][SEASON_RESULT], show,
any_qualities + best_qualities)
highest_quality_overall = 0
for cur_episode in foundResults[curProvider.name]:
for cur_result in foundResults[curProvider.name][cur_episode]:
if cur_result.quality != Quality.UNKNOWN and cur_result.quality > highest_quality_overall:
for cur_episode in found_results[provider_id]:
for cur_result in found_results[provider_id][cur_episode]:
if Quality.UNKNOWN != cur_result.quality and highest_quality_overall < cur_result.quality:
highest_quality_overall = cur_result.quality
logger.log(u"The highest quality of any match is " + Quality.qualityStrings[highest_quality_overall],
logger.log(u'The highest quality of any match is %s' % Quality.qualityStrings[highest_quality_overall],
logger.DEBUG)
# see if every episode is wanted
if bestSeasonResult:
searchedSeasons = []
searchedSeasons = [str(x.season) for x in episodes]
if best_season_result:
# get the quality of the season nzb
seasonQual = bestSeasonResult.quality
logger.log(
u"The quality of the season " + bestSeasonResult.provider.providerType + " is " + Quality.qualityStrings[
seasonQual], logger.DEBUG)
season_qual = best_season_result.quality
logger.log(u'The quality of the season %s is %s' % (best_season_result.provider.providerType,
Quality.qualityStrings[season_qual]), logger.DEBUG)
myDB = db.DBConnection()
allEps = [int(x["episode"])
for x in myDB.select("SELECT episode FROM tv_episodes WHERE showid = ? AND ( season IN ( " + ','.join(searchedSeasons) + " ) )",
[show.indexerid])]
logger.log(u"Executed query: [SELECT episode FROM tv_episodes WHERE showid = %s AND season in %s]" % (show.indexerid, ','.join(searchedSeasons)))
logger.log(u"Episode list: " + str(allEps), logger.DEBUG)
my_db = db.DBConnection()
sql = 'SELECT episode FROM tv_episodes WHERE showid = %s AND (season IN (%s))' %\
(show.indexerid, ','.join([str(x.season) for x in episodes]))
ep_nums = [int(x['episode']) for x in my_db.select(sql)]
allWanted = True
anyWanted = False
for curEpNum in allEps:
logger.log(u'Executed query: [%s]' % sql)
logger.log(u'Episode list: %s' % ep_nums, logger.DEBUG)
all_wanted = True
any_wanted = False
for ep_num in ep_nums:
for season in set([x.season for x in episodes]):
if not show.wantEpisode(season, curEpNum, seasonQual):
allWanted = False
if not show.wantEpisode(season, ep_num, season_qual):
all_wanted = False
else:
anyWanted = True
any_wanted = True
# if we need every ep in the season and there's nothing better then just download this and be done with it (unless single episodes are preferred)
if allWanted and bestSeasonResult.quality == highest_quality_overall:
logger.log(
u"Every episode in this season is needed, downloading the whole " + bestSeasonResult.provider.providerType + " " + bestSeasonResult.name)
epObjs = []
for curEpNum in allEps:
epObjs.append(show.getEpisode(season, curEpNum))
bestSeasonResult.episodes = epObjs
if all_wanted and highest_quality_overall == best_season_result.quality:
logger.log(u'Every episode in this season is needed, downloading the whole %s %s' %
(best_season_result.provider.providerType, best_season_result.name))
ep_objs = []
for ep_num in ep_nums:
for season in set([x.season for x in episodes]):
ep_objs.append(show.getEpisode(season, ep_num))
best_season_result.episodes = ep_objs
return [bestSeasonResult]
elif not anyWanted:
logger.log(
u"No episodes from this season are wanted at this quality, ignoring the result of " + bestSeasonResult.name,
logger.DEBUG)
return [best_season_result]
elif not any_wanted:
logger.log(u'No episodes from this season are wanted at this quality, ignoring the result of ' +
best_season_result.name, logger.DEBUG)
else:
if bestSeasonResult.provider.providerType == GenericProvider.NZB:
logger.log(u"Breaking apart the NZB and adding the individual ones to our results", logger.DEBUG)
if GenericProvider.NZB == best_season_result.provider.providerType:
logger.log(u'Breaking apart the NZB and adding the individual ones to our results', logger.DEBUG)
# if not, break it apart and add them as the lowest priority results
individualResults = nzbSplitter.splitResult(bestSeasonResult)
individual_results = nzbSplitter.splitResult(best_season_result)
individualResults = filter(
lambda x: show_name_helpers.filterBadReleases(x.name, parse=False) and x.show == show, individualResults)
individual_results = filter(
lambda r: show_name_helpers.pass_wordlist_checks(r.name, parse=False) and r.show == show, individual_results)
for curResult in individualResults:
if len(curResult.episodes) == 1:
epNum = curResult.episodes[0].episode
elif len(curResult.episodes) > 1:
epNum = MULTI_EP_RESULT
for cur_result in individual_results:
if 1 == len(cur_result.episodes):
ep_num = cur_result.episodes[0].episode
elif 1 < len(cur_result.episodes):
ep_num = MULTI_EP_RESULT
if epNum in foundResults[curProvider.name]:
foundResults[curProvider.name][epNum].append(curResult)
if ep_num in found_results[provider_id]:
found_results[provider_id][ep_num].append(cur_result)
else:
foundResults[curProvider.name][epNum] = [curResult]
found_results[provider_id][ep_num] = [cur_result]
# If this is a torrent all we can do is leech the entire torrent, user will have to select which eps not do download in his torrent client
else:
# Season result from Torrent Provider must be a full-season torrent, creating multi-ep result for it.
logger.log(
u"Adding multi episode result for full season torrent. Set the episodes you don't want to 'don't download' in your torrent client if desired!")
epObjs = []
for curEpNum in allEps:
epObjs.append(show.getEpisode(season, curEpNum))
bestSeasonResult.episodes = epObjs
logger.log(u'Adding multi episode result for full season torrent. In your torrent client, set ' +
u'the episodes that you do not want to "don\'t download"')
ep_objs = []
for ep_num in ep_nums:
for season in set([x.season for x in episodes]):
ep_objs.append(show.getEpisode(season, ep_num))
best_season_result.episodes = ep_objs
epNum = MULTI_EP_RESULT
if epNum in foundResults[curProvider.name]:
foundResults[curProvider.name][epNum].append(bestSeasonResult)
ep_num = MULTI_EP_RESULT
if ep_num in found_results[provider_id]:
found_results[provider_id][ep_num].append(best_season_result)
else:
foundResults[curProvider.name][epNum] = [bestSeasonResult]
found_results[provider_id][ep_num] = [best_season_result]
# go through multi-ep results and see if we really want them or not, get rid of the rest
multiResults = {}
if MULTI_EP_RESULT in foundResults[curProvider.name]:
for multiResult in foundResults[curProvider.name][MULTI_EP_RESULT]:
multi_results = {}
if MULTI_EP_RESULT in found_results[provider_id]:
for multi_result in found_results[provider_id][MULTI_EP_RESULT]:
logger.log(u"Seeing if we want to bother with multi episode result " + multiResult.name, logger.DEBUG)
logger.log(u'Checking usefulness of multi episode result %s' % multi_result.name, logger.DEBUG)
if sickbeard.USE_FAILED_DOWNLOADS and failed_history.hasFailed(multiResult.name, multiResult.size,
multiResult.provider.name):
logger.log(multiResult.name + u" has previously failed, rejecting this multi episode result")
if sickbeard.USE_FAILED_DOWNLOADS and failed_history.hasFailed(multi_result.name, multi_result.size,
multi_result.provider.name):
logger.log(u'%s has previously failed, rejecting this multi episode result' % multi_result.name)
continue
# see how many of the eps that this result covers aren't covered by single results
neededEps = []
notNeededEps = []
for epObj in multiResult.episodes:
epNum = epObj.episode
needed_eps = []
not_needed_eps = []
for ep_obj in multi_result.episodes:
ep_num = ep_obj.episode
# if we have results for the episode
if epNum in foundResults[curProvider.name] and len(foundResults[curProvider.name][epNum]) > 0:
neededEps.append(epNum)
if ep_num in found_results[provider_id] and 0 < len(found_results[provider_id][ep_num]):
needed_eps.append(ep_num)
else:
notNeededEps.append(epNum)
not_needed_eps.append(ep_num)
logger.log(
u"Single episode check result is needed episodes: " + str(neededEps) + ", not needed episodes: " + str(notNeededEps),
logger.DEBUG)
logger.log(u'Single episode check result is... needed episodes: %s, not needed episodes: %s' %
(needed_eps, not_needed_eps), logger.DEBUG)
if not notNeededEps:
logger.log(u"All of these episodes were covered by single episode results, ignoring this multi episode result", logger.DEBUG)
if not not_needed_eps:
logger.log(u'All of these episodes were covered by single episode results, ignoring this multi episode result', logger.DEBUG)
continue
# check if these eps are already covered by another multi-result
multiNeededEps = []
multiNotNeededEps = []
for epObj in multiResult.episodes:
epNum = epObj.episode
if epNum in multiResults:
multiNotNeededEps.append(epNum)
multi_needed_eps = []
multi_not_needed_eps = []
for ep_obj in multi_result.episodes:
ep_num = ep_obj.episode
if ep_num in multi_results:
multi_not_needed_eps.append(ep_num)
else:
multiNeededEps.append(epNum)
multi_needed_eps.append(ep_num)
logger.log(
u"Multi episode check result is multi needed episodes: " + str(multiNeededEps) + ", multi not needed episodes: " + str(
multiNotNeededEps), logger.DEBUG)
logger.log(u'Multi episode check result is... multi needed episodes: %s, multi not needed episodes: %s' %
(multi_needed_eps, multi_not_needed_eps), logger.DEBUG)
if not multiNeededEps:
logger.log(
u"All of these episodes were covered by another multi episode nzb, ignoring this multi episode result",
logger.DEBUG)
if not multi_needed_eps:
logger.log(u'All of these episodes were covered by another multi episode nzb, ignoring this multi episode result',
logger.DEBUG)
continue
# if we're keeping this multi-result then remember it
for epObj in multiResult.episodes:
multiResults[epObj.episode] = multiResult
for ep_obj in multi_result.episodes:
multi_results[ep_obj.episode] = multi_result
# don't bother with the single result if we're going to get it with a multi result
for epObj in multiResult.episodes:
epNum = epObj.episode
if epNum in foundResults[curProvider.name]:
logger.log(
u"A needed multi episode result overlaps with a single episode result for episode #" + str(
epNum) + ", removing the single episode results from the list", logger.DEBUG)
del foundResults[curProvider.name][epNum]
for ep_obj in multi_result.episodes:
ep_num = ep_obj.episode
if ep_num in found_results[provider_id]:
logger.log(u'A needed multi episode result overlaps with a single episode result for episode #%s, removing the single episode results from the list' %
ep_num, logger.DEBUG)
del found_results[provider_id][ep_num]
# of all the single ep results narrow it down to the best one for each episode
finalResults += set(multiResults.values())
for curEp in foundResults[curProvider.name]:
if curEp in (MULTI_EP_RESULT, SEASON_RESULT):
final_results += set(multi_results.values())
for cur_ep in found_results[provider_id]:
if cur_ep in (MULTI_EP_RESULT, SEASON_RESULT):
continue
if len(foundResults[curProvider.name][curEp]) == 0:
if 0 == len(found_results[provider_id][cur_ep]):
continue
bestResult = pickBestResult(foundResults[curProvider.name][curEp], show)
best_result = pick_best_result(found_results[provider_id][cur_ep], show)
# if all results were rejected move on to the next episode
if not bestResult:
if not best_result:
continue
# filter out possible bad torrents from providers
if bestResult.resultType == "torrent" and sickbeard.TORRENT_METHOD != "blackhole":
bestResult.content = None
if not bestResult.url.startswith('magnet'):
bestResult.content = bestResult.provider.get_url(bestResult.url)
if not bestResult.content:
if 'torrent' == best_result.resultType and 'blackhole' != sickbeard.TORRENT_METHOD:
best_result.content = None
if not best_result.url.startswith('magnet'):
best_result.content = best_result.provider.get_url(best_result.url)
if not best_result.content:
continue
# add result if its not a duplicate and
found = False
for i, result in enumerate(finalResults):
for bestResultEp in bestResult.episodes:
if bestResultEp in result.episodes:
if result.quality < bestResult.quality:
finalResults.pop(i)
for i, result in enumerate(final_results):
for best_result_ep in best_result.episodes:
if best_result_ep in result.episodes:
if best_result.quality > result.quality:
final_results.pop(i)
else:
found = True
if not found:
finalResults += [bestResult]
final_results += [best_result]
# check that we got all the episodes we wanted first before doing a match and snatch
wantedEpCount = 0
for wantedEp in episodes:
for result in finalResults:
if wantedEp in result.episodes and isFinalResult(result):
wantedEpCount += 1
wanted_ep_count = 0
for wanted_ep in episodes:
for result in final_results:
if wanted_ep in result.episodes and is_final_result(result):
wanted_ep_count += 1
# make sure we search every provider for results unless we found everything we wanted
if wantedEpCount == len(episodes):
if len(episodes) == wanted_ep_count:
break
if not didSearch:
logger.log(u"No NZB/Torrent providers found or enabled in the SickGear config for backlog searches. Please check your settings.",
if not search_done:
logger.log(u'No NZB/Torrent providers found or enabled in the SickGear config for backlog searches. Please check your settings.',
logger.ERROR)
return finalResults
return final_results

8
sickbeard/searchBacklog.py → sickbeard/search_backlog.py
View file

@ -28,7 +28,7 @@ from sickbeard import search_queue
from sickbeard import logger
from sickbeard import ui
from sickbeard import common
from sickbeard.search import wantedEpisodes
from sickbeard.search import wanted_episodes
NORMAL_BACKLOG = 0
LIMITED_BACKLOG = 10
@ -75,7 +75,7 @@ class BacklogSearcher:
logger.log(u'amWaiting: ' + str(self.amWaiting) + ', amActive: ' + str(self.amActive), logger.DEBUG)
return (not self.amWaiting) and self.amActive
def searchBacklog(self, which_shows=None, force_type=NORMAL_BACKLOG):
def search_backlog(self, which_shows=None, force_type=NORMAL_BACKLOG):
if self.amActive:
logger.log(u'Backlog is still running, not starting it again', logger.DEBUG)
@ -112,7 +112,7 @@ class BacklogSearcher:
if curShow.paused:
continue
segments = wantedEpisodes(curShow, fromDate, make_dict=True)
segments = wanted_episodes(curShow, fromDate, make_dict=True)
for season, segment in segments.items():
self.currentSearchInfo = {'title': curShow.name + ' Season ' + str(season)}
@ -166,7 +166,7 @@ class BacklogSearcher:
try:
force_type = self.forcetype
self.forcetype = NORMAL_BACKLOG
self.searchBacklog(force_type=force_type)
self.search_backlog(force_type=force_type)
except:
self.amActive = False
raise

5
sickbeard/searchProper.py → sickbeard/search_propers.py
View file

@ -19,11 +19,10 @@
from __future__ import with_statement
import threading
import sickbeard
class ProperSearcher():
class ProperSearcher:
def __init__(self):
self.lock = threading.Lock()
self.amActive = False
@ -35,4 +34,4 @@ class ProperSearcher():
propersearch_queue_item = sickbeard.search_queue.ProperSearchQueueItem()
sickbeard.searchQueueScheduler.action.add_item(propersearch_queue_item)
self.amActive = False
self.amActive = False

143
sickbeard/search_queue.py
View file

@ -26,7 +26,7 @@ import datetime
import sickbeard
from sickbeard import db, logger, common, exceptions, helpers, network_timezones, generic_queue, search, \
failed_history, history, ui, properFinder
from sickbeard.search import wantedEpisodes
from sickbeard.search import wanted_episodes
search_queue_lock = threading.Lock()
@ -40,6 +40,7 @@ PROPER_SEARCH = 50
MANUAL_SEARCH_HISTORY = []
MANUAL_SEARCH_HISTORY_SIZE = 100
class SearchQueue(generic_queue.GenericQueue):
def __init__(self):
generic_queue.GenericQueue.__init__(self)
@ -58,14 +59,14 @@ class SearchQueue(generic_queue.GenericQueue):
if isinstance(cur_item, (ManualSearchQueueItem, FailedQueueItem)) and cur_item.segment == segment:
return True
return False
def is_show_in_queue(self, show):
with self.lock:
for cur_item in self.queue:
if isinstance(cur_item, (ManualSearchQueueItem, FailedQueueItem)) and cur_item.show.indexerid == show:
return True
return False
def get_all_ep_from_queue(self, show):
with self.lock:
ep_obj_list = []
@ -76,7 +77,7 @@ class SearchQueue(generic_queue.GenericQueue):
if ep_obj_list:
return ep_obj_list
return False
def pause_backlog(self):
with self.lock:
self.min_priority = generic_queue.QueuePriorities.HIGH
@ -90,10 +91,10 @@ class SearchQueue(generic_queue.GenericQueue):
with self.lock:
return self.min_priority >= generic_queue.QueuePriorities.NORMAL
def _is_in_progress(self, itemType):
def _is_in_progress(self, item_type):
with self.lock:
for cur_item in self.queue + [self.currentItem]:
if isinstance(cur_item, itemType):
if isinstance(cur_item, item_type):
return True
return False
@ -108,7 +109,7 @@ class SearchQueue(generic_queue.GenericQueue):
return self._is_in_progress(RecentSearchQueueItem)
def is_propersearch_in_progress(self):
return self._is_in_progress(ProperSearchQueueItem)
return self._is_in_progress(ProperSearchQueueItem)
def is_standard_backlog_in_progress(self):
with self.lock:
@ -155,7 +156,6 @@ class SearchQueue(generic_queue.GenericQueue):
length['failed'].append([cur_item.show.indexerid, cur_item.show.name, cur_item.segment])
return length
def add_item(self, item):
if isinstance(item, (RecentSearchQueueItem, ProperSearchQueueItem)):
# recent and proper searches
@ -185,12 +185,12 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
self.update_providers()
show_list = sickbeard.showList
fromDate = datetime.date.fromordinal(1)
from_date = datetime.date.fromordinal(1)
for curShow in show_list:
if curShow.paused:
continue
self.episodes.extend(wantedEpisodes(curShow, fromDate))
self.episodes.extend(wanted_episodes(curShow, from_date))
if not self.episodes:
logger.log(u'No search of cache for episodes required')
@ -203,7 +203,7 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
try:
logger.log(u'Beginning recent search for episodes')
found_results = search.searchForNeededEpisodes(self.episodes)
found_results = search.search_for_needed_episodes(self.episodes)
if not len(found_results):
logger.log(u'No needed episodes found')
@ -211,7 +211,7 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
for result in found_results:
# just use the first result for now
logger.log(u'Downloading %s from %s' % (result.name, result.provider.name))
self.success = search.snatchEpisode(result)
self.success = search.snatch_episode(result)
# give the CPU a break
time.sleep(common.cpu_presets[sickbeard.CPU_PRESET])
@ -219,7 +219,7 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
except Exception:
logger.log(traceback.format_exc(), logger.DEBUG)
if self.success is None:
if None is self.success:
self.success = False
finally:
@ -231,23 +231,23 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
network_timezones.update_network_dict()
if network_timezones.network_dict:
curDate = (datetime.date.today() + datetime.timedelta(days=1)).toordinal()
cur_date = (datetime.date.today() + datetime.timedelta(days=1)).toordinal()
else:
curDate = (datetime.date.today() - datetime.timedelta(days=2)).toordinal()
cur_date = (datetime.date.today() - datetime.timedelta(days=2)).toordinal()
curTime = datetime.datetime.now(network_timezones.sb_timezone)
cur_time = datetime.datetime.now(network_timezones.sb_timezone)
myDB = db.DBConnection()
sqlResults = myDB.select('SELECT * FROM tv_episodes WHERE status = ? AND season > 0 AND airdate <= ?',
[common.UNAIRED, curDate])
my_db = db.DBConnection()
sql_results = my_db.select('SELECT * FROM tv_episodes WHERE status = ? AND season > 0 AND airdate <= ?',
[common.UNAIRED, cur_date])
sql_l = []
show = None
wanted = False
for sqlEp in sqlResults:
for sqlEp in sql_results:
try:
if not show or int(sqlEp['showid']) != show.indexerid:
if not show or show.indexerid != int(sqlEp['showid']):
show = helpers.findCertainShow(sickbeard.showList, int(sqlEp['showid']))
# for when there is orphaned series in the database but not loaded into our showlist
@ -255,13 +255,13 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
continue
except exceptions.MultipleShowObjectsException:
logger.log(u'ERROR: expected to find a single show matching ' + str(sqlEp['showid']))
logger.log(u'ERROR: expected to find a single show matching %s' % sqlEp['showid'])
continue
try:
end_time = network_timezones.parse_date_time(sqlEp['airdate'], show.airs, show.network) + datetime.timedelta(minutes=helpers.tryInt(show.runtime, 60))
# filter out any episodes that haven't aired yet
if end_time > curTime:
if end_time > cur_time:
continue
except:
# if an error occurred assume the episode hasn't aired yet
@ -279,23 +279,23 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
logger.log(u'No unaired episodes marked wanted')
if 0 < len(sql_l):
myDB = db.DBConnection()
myDB.mass_action(sql_l)
my_db = db.DBConnection()
my_db.mass_action(sql_l)
if wanted:
logger.log(u'Found new episodes marked wanted')
@staticmethod
def update_providers():
origThreadName = threading.currentThread().name
orig_thread_name = threading.currentThread().name
threads = []
logger.log('Updating provider caches with recent upload data')
providers = [x for x in sickbeard.providers.sortedProviderList() if x.is_active() and x.enable_recentsearch]
for curProvider in providers:
for cur_provider in providers:
# spawn separate threads for each provider so we don't need to wait for providers with slow network operation
threads.append(threading.Thread(target=curProvider.cache.updateCache, name=origThreadName +
' :: [' + curProvider.name + ']'))
threads.append(threading.Thread(target=cur_provider.cache.updateCache,
name='%s :: [%s]' % (orig_thread_name, cur_provider.name)))
# start the thread we just created
threads[-1].start()
@ -316,7 +316,7 @@ class ProperSearchQueueItem(generic_queue.QueueItem):
generic_queue.QueueItem.run(self)
try:
properFinder.searchPropers()
properFinder.search_propers()
finally:
self.finish()
@ -325,7 +325,7 @@ class ManualSearchQueueItem(generic_queue.QueueItem):
def __init__(self, show, segment):
generic_queue.QueueItem.__init__(self, 'Manual Search', MANUAL_SEARCH)
self.priority = generic_queue.QueuePriorities.HIGH
self.name = 'MANUAL-' + str(show.indexerid)
self.name = 'MANUAL-%s' % show.indexerid
self.success = None
self.show = show
self.segment = segment
@ -335,30 +335,30 @@ class ManualSearchQueueItem(generic_queue.QueueItem):
generic_queue.QueueItem.run(self)
try:
logger.log('Beginning manual search for: [' + self.segment.prettyName() + ']')
logger.log(u'Beginning manual search for: [%s]' % self.segment.prettyName())
self.started = True
searchResult = search.searchProviders(self.show, [self.segment], True)
if searchResult:
search_result = search.search_providers(self.show, [self.segment], True)
if search_result:
# just use the first result for now
logger.log(u'Downloading ' + searchResult[0].name + ' from ' + searchResult[0].provider.name)
self.success = search.snatchEpisode(searchResult[0])
logger.log(u'Downloading %s from %s' % (search_result[0].name, search_result[0].provider.name))
self.success = search.snatch_episode(search_result[0])
# give the CPU a break
time.sleep(common.cpu_presets[sickbeard.CPU_PRESET])
else:
ui.notifications.message('No downloads were found',
'Couldn\'t find a download for <i>%s</i>' % self.segment.prettyName())
ui.notifications.message('No downloads found',
u'Could not find a download for <i>%s</i>' % self.segment.prettyName())
logger.log(u'Unable to find a download for: [' + self.segment.prettyName() + ']')
logger.log(u'Unable to find a download for: [%s]' % self.segment.prettyName())
except Exception:
logger.log(traceback.format_exc(), logger.DEBUG)
finally:
### Keep a list with the 100 last executed searches
# Keep a list with the 100 last executed searches
fifo(MANUAL_SEARCH_HISTORY, self, MANUAL_SEARCH_HISTORY_SIZE)
if self.success is None:
@ -371,7 +371,7 @@ class BacklogQueueItem(generic_queue.QueueItem):
def __init__(self, show, segment, standard_backlog=False, limited_backlog=False, forced=False):
generic_queue.QueueItem.__init__(self, 'Backlog', BACKLOG_SEARCH)
self.priority = generic_queue.QueuePriorities.LOW
self.name = 'BACKLOG-' + str(show.indexerid)
self.name = 'BACKLOG-%s' % show.indexerid
self.success = None
self.show = show
self.segment = segment
@ -383,19 +383,19 @@ class BacklogQueueItem(generic_queue.QueueItem):
generic_queue.QueueItem.run(self)
try:
logger.log('Beginning backlog search for: [' + self.show.name + ']')
searchResult = search.searchProviders(self.show, self.segment, False)
logger.log(u'Beginning backlog search for: [%s]' % self.show.name)
search_result = search.search_providers(self.show, self.segment, False)
if searchResult:
for result in searchResult:
if search_result:
for result in search_result:
# just use the first result for now
logger.log(u'Downloading ' + result.name + ' from ' + result.provider.name)
search.snatchEpisode(result)
logger.log(u'Downloading %s from %s' % (result.name, result.provider.name))
search.snatch_episode(result)
# give the CPU a break
time.sleep(common.cpu_presets[sickbeard.CPU_PRESET])
else:
logger.log(u'No needed episodes found during backlog search for: [' + self.show.name + ']')
logger.log(u'No needed episodes found during backlog search for: [%s]' % self.show.name)
except Exception:
logger.log(traceback.format_exc(), logger.DEBUG)
@ -407,7 +407,7 @@ class FailedQueueItem(generic_queue.QueueItem):
def __init__(self, show, segment):
generic_queue.QueueItem.__init__(self, 'Retry', FAILED_SEARCH)
self.priority = generic_queue.QueuePriorities.HIGH
self.name = 'RETRY-' + str(show.indexerid)
self.name = 'RETRY-%s' % show.indexerid
self.show = show
self.segment = segment
self.success = None
@ -416,48 +416,49 @@ class FailedQueueItem(generic_queue.QueueItem):
def run(self):
generic_queue.QueueItem.run(self)
self.started = True
try:
for epObj in self.segment:
logger.log(u'Marking episode as bad: [' + epObj.prettyName() + ']')
logger.log(u'Marking episode as bad: [%s]' % epObj.prettyName())
failed_history.markFailed(epObj)
(release, provider) = failed_history.findRelease(epObj)
if release:
failed_history.logFailed(release)
history.logFailed(epObj, release, provider)
failed_history.revertEpisode(epObj)
logger.log('Beginning failed download search for: [' + epObj.prettyName() + ']')
logger.log(u'Beginning failed download search for: []' % epObj.prettyName())
searchResult = search.searchProviders(self.show, self.segment, True)
search_result = search.search_providers(self.show, self.segment, True)
if searchResult:
for result in searchResult:
if search_result:
for result in search_result:
# just use the first result for now
logger.log(u'Downloading ' + result.name + ' from ' + result.provider.name)
search.snatchEpisode(result)
logger.log(u'Downloading %s from %s' % (result.name, result.provider.name))
search.snatch_episode(result)
# give the CPU a break
time.sleep(common.cpu_presets[sickbeard.CPU_PRESET])
else:
pass
#logger.log(u"No valid episode found to retry for: [" + self.segment.prettyName() + "]")
# logger.log(u'No valid episode found to retry for: [%s]' % self.segment.prettyName())
except Exception:
logger.log(traceback.format_exc(), logger.DEBUG)
finally:
### Keep a list with the 100 last executed searches
# Keep a list with the 100 last executed searches
fifo(MANUAL_SEARCH_HISTORY, self, MANUAL_SEARCH_HISTORY_SIZE)
if self.success is None:
self.success = False
self.finish()
def fifo(myList, item, maxSize = 100):
if len(myList) >= maxSize:
myList.pop(0)
myList.append(item)
def fifo(my_list, item, max_size=100):
if len(my_list) >= max_size:
my_list.pop(0)
my_list.append(item)

3
sickbeard/searchRecent.py → sickbeard/search_recent.py
View file

@ -19,7 +19,6 @@
from __future__ import with_statement
import threading
import sickbeard
@ -35,4 +34,4 @@ class RecentSearcher():
recentsearch_queue_item = sickbeard.search_queue.RecentSearchQueueItem()
sickbeard.searchQueueScheduler.action.add_item(recentsearch_queue_item)
self.amActive = False
self.amActive = False

119
sickbeard/show_name_helpers.py
View file

@ -29,56 +29,101 @@ from sickbeard import logger
from sickbeard import db
from sickbeard import encodingKludge as ek
from name_parser.parser import NameParser, InvalidNameException, InvalidShowException
from lib.unidecode import unidecode
from sickbeard.blackandwhitelist import BlackAndWhiteList
def filterBadReleases(name, parse=True):
def pass_wordlist_checks(name, parse=True):
"""
Filters out non-english and just all-around stupid releases by comparing them
to the resultFilters contents.
Filters out non-english and just all-around stupid releases by comparing
the word list contents at boundaries or the end of name.
name: the release name to check
Returns: True if the release name is OK, False if it's bad.
"""
try:
if parse:
if parse:
err_msg = u'Unable to parse the filename %s into a valid ' % name
try:
NameParser().parse(name)
except InvalidNameException:
logger.log(u"Unable to parse the filename " + name + " into a valid episode", logger.DEBUG)
return False
except InvalidShowException:
logger.log(u"Unable to parse the filename " + name + " into a valid show", logger.DEBUG)
return False
except InvalidNameException:
logger.log(err_msg + 'episode', logger.DEBUG)
return False
except InvalidShowException:
logger.log(err_msg + 'show', logger.DEBUG)
return False
resultFilters = ['sub(bed|ed|pack|s)', '(dk|fin|heb|kor|nor|nordic|pl|swe)sub(bed|ed|s)?',
'(dir|sample|sub|nfo)fix', 'sample', '(dvd)?extras',
'dub(bed)?']
word_list = ['sub(bed|ed|pack|s)', '(dk|fin|heb|kor|nor|nordic|pl|swe)sub(bed|ed|s)?',
'(dir|sample|sub|nfo)fix', 'sample', '(dvd)?extras',
'dub(bed)?']
# if any of the bad strings are in the name then say no
if sickbeard.IGNORE_WORDS:
resultFilters.extend(sickbeard.IGNORE_WORDS.split(','))
filters = [re.compile('(^|[\W_])%s($|[\W_])' % re.escape(filter.strip()), re.I) for filter in resultFilters]
for regfilter in filters:
if regfilter.search(name):
logger.log(u"Invalid scene release: " + name + " contained: " + regfilter.pattern + ", ignoring it",
logger.DEBUG)
return False
word_list = ','.join([sickbeard.IGNORE_WORDS] + word_list)
result = contains_any(name, word_list)
if None is not result and result:
logger.log(u'Ignored: %s for containing ignore word' % name, logger.DEBUG)
return False
# if any of the good strings aren't in the name then say no
if sickbeard.REQUIRE_WORDS:
require_words = sickbeard.REQUIRE_WORDS.split(',')
filters = [re.compile('(^|[\W_])%s($|[\W_])' % re.escape(filter.strip()), re.I) for filter in require_words]
for regfilter in filters:
if not regfilter.search(name):
logger.log(u"Invalid scene release: " + name + " didn't contain: " + regfilter.pattern + ", ignoring it",
logger.DEBUG)
return False
result = not_contains_any(name, sickbeard.REQUIRE_WORDS)
if None is not result and result:
logger.log(u'Ignored: %s for not containing required word match' % name, logger.DEBUG)
return False
return True
def not_contains_any(subject, lookup_words, **kwargs):
return contains_any(subject, lookup_words, invert=True, **kwargs)
def contains_any(subject, lookup_words, invert=False, **kwargs):
"""
Check if subject does or does not contain a match from a list or string of regular expression lookup words
word: word to test existence of
lookup_words: List or comma separated string of words to search
re_prefix: insert string to all lookup words
re_suffix: append string to all lookup words
invert: invert function logic "contains any" into "does not contain any"
Returns: None if no checking was done. True for first match found, or if invert is False,
then True for first pattern that does not match, or False
"""
compiled_words = compile_word_list(lookup_words, **kwargs)
if subject and compiled_words:
for rc_filter in compiled_words:
match = rc_filter.search(subject)
if (match and not invert) or (not match and invert):
msg = match and not invert and 'Found match' or ''
msg = not match and invert and 'No match found' or msg
logger.log(u'%s from pattern: %s in text: %s ' % (msg, rc_filter.pattern, subject), logger.DEBUG)
return True
return False
return None
def compile_word_list(lookup_words, re_prefix='(^|[\W_])', re_suffix='($|[\W_])'):
result = []
if lookup_words:
search_raw = isinstance(lookup_words, list)
if not search_raw:
search_raw = not lookup_words.startswith('regex:')
lookup_words = lookup_words[(6, 0)[search_raw]:].split(',')
lookup_words = [x.strip() for x in lookup_words]
for word in [x for x in lookup_words if x]:
try:
# !0 == regex and subject = s / 'what\'s the "time"' / what\'s\ the\ \"time\"
subject = search_raw and re.escape(word) or re.sub(r'([\" \'])', r'\\\1', word)
result.append(re.compile('(?i)%s%s%s' % (re_prefix, subject, re_suffix)))
except Exception as e:
logger.log(u'Failure to compile filter expression: %s ... Reason: %s' % (word, e.message), logger.DEBUG)
diff = len(lookup_words) - len(result)
if diff:
logger.log(u'From %s expressions, %s was discarded during compilation' % (len(lookup_words), diff), logger.DEBUG)
return result
def makeSceneShowSearchStrings(show, season=-1):
showNames = allPossibleShowNames(show, season=season)
@ -195,9 +240,9 @@ def allPossibleShowNames(show, season=-1):
"""
Figures out every possible variation of the name for a particular show. Includes TVDB name, TVRage name,
country codes on the end, eg. "Show Name (AU)", and any scene exception names.
show: a TVShow object that we should get the names of
Returns: a list of all the possible show names
"""
@ -256,13 +301,13 @@ def determineReleaseName(dir_name=None, nzb_name=None):
if len(results) == 1:
found_file = ek.ek(os.path.basename, results[0])
found_file = found_file.rpartition('.')[0]
if filterBadReleases(found_file):
if pass_wordlist_checks(found_file):
logger.log(u"Release name (" + found_file + ") found from file (" + results[0] + ")")
return found_file.rpartition('.')[0]
# If that fails, we try the folder
folder = ek.ek(os.path.basename, dir_name)
if filterBadReleases(folder):
if pass_wordlist_checks(folder):
# NOTE: Multiple failed downloads will change the folder name.
# (e.g., appending #s)
# Should we handle that?

6
sickbeard/show_queue.py
View file

@ -139,7 +139,7 @@ class ShowQueue(generic_queue.GenericQueue):
if ((not after_update and self.isBeingUpdated(show)) or self.isInUpdateQueue(show)) and not force:
logger.log(
u'A refresh was attempted but there is already an update queued or in progress. Since updates do a refresh at the end anyway I\'m skipping this request.',
u'Skipping this refresh as there is already an update queued or in progress and a refresh is done at the end of an update anyway.',
logger.DEBUG)
return
@ -449,7 +449,7 @@ class QueueItemAdd(ShowQueueItem):
# if started with WANTED eps then run the backlog
if WANTED == self.default_status or items_wanted:
logger.log(u'Launching backlog for this show since episodes are WANTED')
sickbeard.backlogSearchScheduler.action.searchBacklog([self.show]) #@UndefinedVariable
sickbeard.backlogSearchScheduler.action.search_backlog([self.show]) #@UndefinedVariable
ui.notifications.message('Show added/search', 'Adding and searching for episodes of' + msg)
else:
ui.notifications.message('Show added', 'Adding' + msg)
@ -655,4 +655,4 @@ class QueueItemForceUpdateWeb(QueueItemUpdate):
def __init__(self, show=None, scheduled_update=False):
ShowQueueItem.__init__(self, ShowQueueActions.FORCEUPDATE, show, scheduled_update)
self.force = True
self.force_web = True
self.force_web = True

0
sickbeard/showUpdater.py → sickbeard/show_updater.py
View file

8
sickbeard/tv.py
View file

@ -2156,14 +2156,14 @@ class TVEpisode(object):
def us(name):
return re.sub('[ -]', '_', name)
def release_name(show, name):
def release_name(name, is_anime=False):
if name:
name = helpers.remove_non_release_groups(helpers.remove_extension(name), show.anime)
name = helpers.remove_non_release_groups(helpers.remove_extension(name), is_anime)
return name
def release_group(show, name):
if name:
name = helpers.remove_non_release_groups(helpers.remove_extension(name), show.anime)
name = helpers.remove_non_release_groups(helpers.remove_extension(name), show.is_anime)
else:
return ""
@ -2205,7 +2205,7 @@ class TVEpisode(object):
'%0XE': '%02d' % self.scene_episode,
'%AB': '%(#)03d' % {'#': self.absolute_number},
'%XAB': '%(#)03d' % {'#': self.scene_absolute_number},
'%RN': release_name(self.show, self.release_name),
'%RN': release_name(self.release_name, self.show.is_anime),
'%RG': release_group(self.show, self.release_name),
'%AD': str(self.airdate).replace('-', ' '),
'%A.D': str(self.airdate).replace('-', '.'),

22
sickbeard/tvcache.py
View file

@ -50,7 +50,7 @@ class TVCache:
self.provider = provider
self.providerID = self.provider.get_id()
self.providerDB = None
self.minTime = 10
self.update_freq = 10
def get_db(self):
return CacheDBConnection(self.providerID)
@ -60,11 +60,11 @@ class TVCache:
myDB = self.get_db()
myDB.action('DELETE FROM provider_cache WHERE provider = ?', [self.providerID])
def _get_title_and_url(self, item):
def _title_and_url(self, item):
# override this in the provider if recent search has a different data layout to backlog searches
return self.provider._get_title_and_url(item)
return self.provider._title_and_url(item)
def _getRSSData(self):
def _cache_data(self):
data = None
return data
@ -83,7 +83,7 @@ class TVCache:
if self.shouldUpdate():
# as long as the http request worked we count this as an update
data = self._getRSSData()
data = self._cache_data()
if not data:
return []
@ -96,7 +96,7 @@ class TVCache:
# parse data
cl = []
for item in data:
title, url = self._get_title_and_url(item)
title, url = self._title_and_url(item)
ci = self._parseItem(title, url)
if ci is not None:
cl.append(ci)
@ -182,9 +182,9 @@ class TVCache:
def shouldUpdate(self):
# if we've updated recently then skip the update
if datetime.datetime.today() - self.lastUpdate < datetime.timedelta(minutes=self.minTime):
if datetime.datetime.today() - self.lastUpdate < datetime.timedelta(minutes=self.update_freq):
logger.log(u'Last update was too soon, using old cache: today()-' + str(self.lastUpdate) + '<' + str(
datetime.timedelta(minutes=self.minTime)), logger.DEBUG)
datetime.timedelta(minutes=self.update_freq)), logger.DEBUG)
return False
return True
@ -255,11 +255,11 @@ class TVCache:
else:
return []
def listPropers(self, date=None, delimiter='.'):
def listPropers(self, date=None):
myDB = self.get_db()
sql = "SELECT * FROM provider_cache WHERE name LIKE '%.PROPER.%' OR name LIKE '%.REPACK.%' AND provider = ?"
if date != None:
if date:
sql += ' AND time >= ' + str(int(time.mktime(date.timetuple())))
return filter(lambda x: x['indexerid'] != 0, myDB.select(sql, [self.providerID]))
@ -291,7 +291,7 @@ class TVCache:
for curResult in sqlResults:
# skip non-tv crap
if not show_name_helpers.filterBadReleases(curResult['name'], parse=False):
if not show_name_helpers.pass_wordlist_checks(curResult['name'], parse=False):
continue
# get the show object, or if it's not one of our shows then ignore it

10
sickbeard/versionChecker.py → sickbeard/version_checker.py
View file

@ -276,9 +276,9 @@ class GitUpdateManager(UpdateManager):
branch = branch_info.strip().replace('refs/heads/', '', 1)
if branch:
return branch
return ""
def _check_github_for_update(self):
"""
Uses git commands to check if there is a newer version that the provided
@ -459,7 +459,7 @@ class SourceUpdateManager(UpdateManager):
return "master"
else:
return sickbeard.CUR_COMMIT_BRANCH
def need_update(self):
# need this to run first to set self._newest_commit_hash
try:
@ -622,7 +622,7 @@ class SourceUpdateManager(UpdateManager):
sickbeard.CUR_COMMIT_HASH = self._newest_commit_hash
sickbeard.CUR_COMMIT_BRANCH = self.branch
except Exception as e:
logger.log(u"Error while trying to update: " + ex(e), logger.ERROR)
logger.log(u"Traceback: " + traceback.format_exc(), logger.DEBUG)
@ -639,4 +639,4 @@ class SourceUpdateManager(UpdateManager):
def list_remote_pulls(self):
# we don't care about testers that don't use git
return []
return []

17
sickbeard/webserve.py
View file

@ -49,7 +49,7 @@ from sickbeard.scene_numbering import get_scene_numbering, set_scene_numbering,
from sickbeard.name_cache import buildNameCache
from sickbeard.browser import foldersAtPath
from sickbeard.blackandwhitelist import BlackAndWhiteList, short_group_names
from sickbeard.searchBacklog import FULL_BACKLOG, LIMITED_BACKLOG
from sickbeard.search_backlog import FULL_BACKLOG, LIMITED_BACKLOG
from tornado import gen
from tornado.web import RequestHandler, StaticFileHandler, authenticated
from lib import adba
@ -3061,7 +3061,7 @@ class Manage(MainHandler):
show_obj = helpers.findCertainShow(sickbeard.showList, int(indexer_id))
if show_obj:
sickbeard.backlogSearchScheduler.action.searchBacklog([show_obj]) # @UndefinedVariable
sickbeard.backlogSearchScheduler.action.search_backlog([show_obj]) # @UndefinedVariable
self.redirect('/manage/backlogOverview/')
@ -3909,6 +3909,12 @@ class ConfigSearch(Config):
t = PageTemplate(headers=self.request.headers, file='config_search.tmpl')
t.submenu = self.ConfigMenu
t.using_rls_ignore_words = [(show.indexerid, show.name)
for show in sickbeard.showList if show.rls_ignore_words.strip()]
t.using_rls_ignore_words.sort(lambda x, y: cmp(x[1], y[1]), reverse=False)
t.using_rls_require_words = [(show.indexerid, show.name)
for show in sickbeard.showList if show.rls_require_words.strip()]
t.using_rls_require_words.sort(lambda x, y: cmp(x[1], y[1]), reverse=False)
return t.respond()
def saveSearch(self, use_nzbs=None, use_torrents=None, nzb_dir=None, sab_username=None, sab_password=None,
@ -4556,6 +4562,13 @@ class ConfigProviders(Config):
except:
curTorrentProvider.freeleech = 0
if hasattr(curTorrentProvider, 'reject_m2ts'):
try:
curTorrentProvider.reject_m2ts = config.checkbox_to_value(
kwargs[curTorrentProvider.get_id() + '_reject_m2ts'])
except:
curTorrentProvider.reject_m2ts = 0
if hasattr(curTorrentProvider, 'search_mode'):
try:
curTorrentProvider.search_mode = str(kwargs[curTorrentProvider.get_id() + '_search_mode']).strip()

2
tests/all_tests.py
View file

@ -24,7 +24,7 @@ if __name__ == '__main__':
import unittest
import sys
test_file_strings = [ x for x in glob.glob('*_tests.py') if not x in __file__]
test_file_strings = [x for x in glob.glob('*_tests.py') if x not in __file__]
module_strings = [file_string[0:len(file_string) - 3] for file_string in test_file_strings]
suites = [unittest.defaultTestLoader.loadTestsFromName(file_string) for file_string in module_strings]
testSuite = unittest.TestSuite(suites)

111
tests/ignore_and_require_words_tests.py Normal file
View file

@ -0,0 +1,111 @@
import os.path
import sys
import unittest
import sickbeard
from sickbeard import show_name_helpers
sys.path.insert(1, os.path.abspath('..'))
class TestCase(unittest.TestCase):
cases_pass_wordlist_checks = [
('[GroupName].Show.Name.-.%02d.[null]', '', '', True),
('[GroupName].Show.Name.-.%02d.[ignore]', '', 'required', False),
('[GroupName].Show.Name.-.%02d.[required]', '', 'required', True),
('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'GroupName', True),
('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', '[GroupName]', True),
('[GroupName].Show.Name.-.%02d.[blahblah]', 'not_ignored', 'Show.Name', True),
('[GroupName].Show.Name.-.%02d.[required]', 'not_ignored', 'required', True),
('[GroupName].Show.Name.-.%02d.[required]', '[not_ignored]', '[required]', True),
('[GroupName].Show.Name.-.%02d.[ignore]', '[ignore]', '', False),
('[GroupName].Show.Name.-.%02d.[required]', '[GroupName]', 'required', False),
('[GroupName].Show.Name.-.%02d.[required]', 'GroupName', 'required', False),
('[GroupName].Show.Name.-.%02d.[ignore]', 'ignore', 'GroupName', False),
('[GroupName].Show.Name.-.%02d.[required]', 'Show.Name', 'required', False),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: no_ignore', '', True),
('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', 'regex: \d?\d80p', True),
('[GroupName].Show.Name.-.%02d.[480p]', 'ignore', 'regex: \[\d?\d80p\]', True),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', '', False),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: \[ignore\]', '', False),
('[GroupName].Show.Name.-.%02d.[ignore]', 'regex: ignore', 'required', False),
# The following test is True because a boundary is added to each regex not overridden with the prefix param
('[GroupName].Show.ONEONE.-.%02d.[required]', 'regex: (one(two)?)', '', True),
('[GroupName].Show.ONETWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False),
('[GroupName].Show.TWO.-.%02d.[required]', 'regex: ((one)?two)', 'required', False),
]
cases_contains = [
('[GroupName].Show.Name.-.%02d.[illegal_regex]', 'regex:??illegal_regex', False),
('[GroupName].Show.Name.-.%02d.[480p]', 'regex:(480|1080)p', True),
('[GroupName].Show.Name.-.%02d.[contains]', 'regex:\[contains\]', True),
('[GroupName].Show.Name.-.%02d.[contains]', '[contains]', True),
('[GroupName].Show.Name.-.%02d.[contains]', 'contains', True),
('[GroupName].Show.Name.-.%02d.[contains]', '[not_contains]', False),
('[GroupName].Show.Name.-.%02d.[null]', '', False)
]
cases_not_contains = [
('[GroupName].Show.Name.-.%02d.[480p]', 'regex:(480|1080)p', False),
('[GroupName].Show.Name.-.%02d.[contains]', 'regex:\[contains\]', False),
('[GroupName].Show.Name.-.%02d.[contains]', '[contains]', False),
('[GroupName].Show.Name.-.%02d.[contains]', 'contains', False),
('[GroupName].Show.Name.-.%02d.[not_contains]', '[blah_blah]', True),
('[GroupName].Show.Name.-.%02d.[null]', '', False)
]
def test_pass_wordlist_checks(self):
# default:[] or copy in a test case tuple to debug in isolation
isolated = []
test_cases = (self.cases_pass_wordlist_checks, isolated)[len(isolated)]
for case_num, (name, ignore_list, require_list, expected_result) in enumerate(test_cases):
name = (name, name % case_num)['%02d' in name]
sickbeard.IGNORE_WORDS = ignore_list
sickbeard.REQUIRE_WORDS = require_list
self.assertEqual(expected_result, show_name_helpers.pass_wordlist_checks(name, False),
'Expected %s with test: "%s" with ignore: "%s", require: "%s"' %
(expected_result, name, ignore_list, require_list))
def test_contains_any(self):
# default:[] or copy in a test case tuple to debug in isolation
isolated = []
test_cases = (self.cases_contains, isolated)[len(isolated)]
for case_num, (name, csv_words, expected_result) in enumerate(test_cases):
name = (name, name % case_num)['%02d' in name]
self.assertEqual(expected_result, self.call_contains_any(name, csv_words),
'Expected %s test: "%s" with csv_words: "%s"' %
(expected_result, name, csv_words))
@staticmethod
def call_contains_any(name, csv_words):
re_extras = dict(re_prefix='.*', re_suffix='.*')
match = show_name_helpers.contains_any(name, csv_words, **re_extras)
return None is not match and match
def test_not_contains_any(self):
# default:[] or copy in a test case tuple to debug in isolation
isolated = []
test_cases = (self.cases_not_contains, isolated)[len(isolated)]
for case_num, (name, csv_words, expected_result) in enumerate(test_cases):
name = (name, name % case_num)['%02d' in name]
self.assertEqual(expected_result, self.call_not_contains_any(name, csv_words),
'Expected %s test: "%s" with csv_words:"%s"' %
(expected_result, name, csv_words))
@staticmethod
def call_not_contains_any(name, csv_words):
re_extras = dict(re_prefix='.*', re_suffix='.*')
match = show_name_helpers.not_contains_any(name, csv_words, **re_extras)
return None is not match and match
if __name__ == '__main__':
unittest.main()

16
tests/scene_helpers_tests.py
View file

@ -23,8 +23,8 @@ class SceneTests(test.SickbeardTestDBCase):
result = show_name_helpers.allPossibleShowNames(s)
self.assertTrue(len(set(expected).intersection(set(result))) == len(expected))
def _test_filterBadReleases(self, name, expected):
result = show_name_helpers.filterBadReleases(name)
def _test_pass_wordlist_checks(self, name, expected):
result = show_name_helpers.pass_wordlist_checks(name)
self.assertEqual(result, expected)
def test_allPossibleShowNames(self):
@ -40,12 +40,12 @@ class SceneTests(test.SickbeardTestDBCase):
self._test_allPossibleShowNames('Show Name Full Country Name', expected=['Show Name Full Country Name', 'Show Name (FCN)'])
self._test_allPossibleShowNames('Show Name (Full Country Name)', expected=['Show Name (Full Country Name)', 'Show Name (FCN)'])
def test_filterBadReleases(self):
self._test_filterBadReleases('Show.S02.German.Stuff-Grp', False)
self._test_filterBadReleases('Show.S02.Some.Stuff-Core2HD', False)
self._test_filterBadReleases('Show.S02.Some.German.Stuff-Grp', False)
# self._test_filterBadReleases('German.Show.S02.Some.Stuff-Grp', True)
self._test_filterBadReleases('Show.S02.This.Is.German', False)
def test_pass_wordlist_checks(self):
self._test_pass_wordlist_checks('Show.S02.German.Stuff-Grp', False)
self._test_pass_wordlist_checks('Show.S02.Some.Stuff-Core2HD', False)
self._test_pass_wordlist_checks('Show.S02.Some.German.Stuff-Grp', False)
# self._test_pass_wordlist_checks('German.Show.S02.Some.Stuff-Grp', True)
self._test_pass_wordlist_checks('Show.S02.This.Is.German', False)
class SceneExceptionTestCase(test.SickbeardTestDBCase):

2
tests/snatch_tests.py
View file

@ -82,7 +82,7 @@ def test_generator(tvdbdid, show_name, curData, forceSearch):
episode.status = c.WANTED
episode.saveToDB()
bestResult = search.searchProviders(show, episode.season, episode.episode, forceSearch)
bestResult = search.search_providers(show, episode.season, episode.episode, forceSearch)
if not bestResult:
self.assertEqual(curData['b'], bestResult)
self.assertEqual(curData['b'], bestResult.name) #first is expected, second is choosen one