mirror of
https://github.com/SickGear/SickGear.git
synced 2025-01-22 09:33:37 +00:00
Merge pull request #1044 from JackDandy/feature/ChangeProviderErrorHandling
Feature/change provider error handling
This commit is contained in:
JackDandy
GitHub
commit
d8cc94dcfc
66 changed files with 1278 additions and 313 deletions
|
|
@ -4,6 +4,9 @@
|
||||||
* Change improve media process to parse anime format 'Show Name 123 - 001 - Ep 1 name'
|
* Change improve media process to parse anime format 'Show Name 123 - 001 - Ep 1 name'
|
||||||
* Add free space stat (if obtainable) of parent folder(s) to footer
|
* Add free space stat (if obtainable) of parent folder(s) to footer
|
||||||
* Add option "Display disk free" to general config/interface page (default enabled)
|
* Add option "Display disk free" to general config/interface page (default enabled)
|
||||||
|
* Add a provider error table to page Manage/Media Search
|
||||||
|
* Add failure handling, skip provider for x hour(s) depending on count of failures
|
||||||
|
* Add detection of Too Many Requests (Supporting providers UC and BTN)
|
||||||
|
|
||||||
|
|
||||||
[develop changelog]
|
[develop changelog]
|
||||||
|
|
|
||||||
|
|
@ -762,6 +762,60 @@ a.whitelink{
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* TABLE BACKGROUND color */
|
||||||
|
.provider-failures.hover-highlight td:before,
|
||||||
|
.provider-failures.focus-highlight td:before{
|
||||||
|
background:#222
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ODD ZEBRA STRIPE color (needs zebra widget) */
|
||||||
|
.provider-failures.hover-highlight .odd td:before,
|
||||||
|
.provider-failures.hover-highlight .odd th:before,
|
||||||
|
.provider-failures.focus-highlight .odd td:before,
|
||||||
|
.provider-failures.focus-highlight .odd th:before{
|
||||||
|
background:#333
|
||||||
|
}
|
||||||
|
/* EVEN ZEBRA STRIPE color (needs zebra widget) */
|
||||||
|
.provider-failures.hover-highlight .even td:before,
|
||||||
|
.provider-failures.hover-highlight .even th:before,
|
||||||
|
.provider-failures.focus-highlight .even td:before,
|
||||||
|
.provider-failures.focus-highlight .even th:before{
|
||||||
|
background-color:#2e2e2e
|
||||||
|
}
|
||||||
|
|
||||||
|
/* HOVER ROW highlight colors */
|
||||||
|
.provider-failures.hover-highlight tbody > tr:hover > td, /* override tablesorter theme row hover */
|
||||||
|
.provider-failures.hover-highlight tbody > tr.odd:hover > td,
|
||||||
|
.provider-failures.hover-highlight tbody > tr.even:hover > td{
|
||||||
|
background-color:#282828
|
||||||
|
}
|
||||||
|
/* HOVER COLUMN highlight colors */
|
||||||
|
.provider-failures.hover-highlight tbody tr th:hover::after,
|
||||||
|
.provider-failures.hover-highlight tbody tr td:hover::after{
|
||||||
|
background-color:#282828
|
||||||
|
}
|
||||||
|
|
||||||
|
/* FOCUS ROW highlight color (touch devices) */
|
||||||
|
.provider-failures.focus-highlight td:focus::before,
|
||||||
|
.provider-failures.focus-highlight th:focus::before{
|
||||||
|
background-color:#181818
|
||||||
|
}
|
||||||
|
/* FOCUS COLUMN highlight color (touch devices) */
|
||||||
|
.provider-failures.focus-highlight td:focus::after,
|
||||||
|
.provider-failures.focus-highlight th:focus::after{
|
||||||
|
background-color:#181818
|
||||||
|
}
|
||||||
|
/* FOCUS CELL highlight color */
|
||||||
|
.provider-failures.focus-highlight th:focus,
|
||||||
|
.provider-failures.focus-highlight td:focus,
|
||||||
|
.provider-failures.focus-highlight .odd th:focus,
|
||||||
|
.provider-failures.focus-highlight .odd td:focus,
|
||||||
|
.provider-failures.focus-highlight .even th:focus,
|
||||||
|
.provider-failures.focus-highlight .even td:focus{
|
||||||
|
background-color:#181818;
|
||||||
|
color:#ddd
|
||||||
|
}
|
||||||
|
|
||||||
/* =======================================================================
|
/* =======================================================================
|
||||||
404.tmpl
|
404.tmpl
|
||||||
========================================================================== */
|
========================================================================== */
|
||||||
|
|
|
||||||
|
|
@ -742,6 +742,60 @@ a.whitelink{
|
||||||
color:#000
|
color:#000
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* TABLE BACKGROUND color */
|
||||||
|
.provider-failures.hover-highlight td:before,
|
||||||
|
.provider-failures.focus-highlight td:before{
|
||||||
|
background:#fff
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ODD ZEBRA STRIPE color (needs zebra widget) */
|
||||||
|
.provider-failures.hover-highlight .odd th:before,
|
||||||
|
.provider-failures.hover-highlight .odd td:before,
|
||||||
|
.provider-failures.focus-highlight .odd th:before,
|
||||||
|
.provider-failures.focus-highlight .odd td:before{
|
||||||
|
background:#f5f1e4
|
||||||
|
}
|
||||||
|
/* EVEN ZEBRA STRIPE color (needs zebra widget) */
|
||||||
|
.provider-failures.hover-highlight .even th:before,
|
||||||
|
.provider-failures.hover-highlight .even td:before,
|
||||||
|
.provider-failures.focus-highlight .even th:before,
|
||||||
|
.provider-failures.focus-highlight .even td:before{
|
||||||
|
background-color:#dfdacf;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* HOVER ROW highlight colors */
|
||||||
|
.provider-failures.hover-highlight tbody > tr:hover > td, /* override tablesorter theme row hover */
|
||||||
|
.provider-failures.hover-highlight tbody > tr.odd:hover > td,
|
||||||
|
.provider-failures.hover-highlight tbody > tr.even:hover > td{
|
||||||
|
background-color:#f4f3c2
|
||||||
|
}
|
||||||
|
/* HOVER COLUMN highlight colors */
|
||||||
|
.provider-failures.hover-highlight tbody tr th:hover::after,
|
||||||
|
.provider-failures.hover-highlight tbody tr td:hover::after{
|
||||||
|
background-color:#f4f3c2
|
||||||
|
}
|
||||||
|
|
||||||
|
/* FOCUS ROW highlight color (touch devices) */
|
||||||
|
.provider-failures.focus-highlight th:focus::before,
|
||||||
|
.provider-failures.focus-highlight td:focus::before{
|
||||||
|
background-color:#dfdead
|
||||||
|
}
|
||||||
|
/* FOCUS COLUMN highlight color (touch devices) */
|
||||||
|
.provider-failures.focus-highlight th:focus::after,
|
||||||
|
.provider-failures.focus-highlight td:focus::after{
|
||||||
|
background-color:#dfdead
|
||||||
|
}
|
||||||
|
/* FOCUS CELL highlight color */
|
||||||
|
.provider-failures.focus-highlight th:focus,
|
||||||
|
.provider-failures.focus-highlight td:focus,
|
||||||
|
.provider-failures.focus-highlight .odd th:focus,
|
||||||
|
.provider-failures.focus-highlight .odd td:focus,
|
||||||
|
.provider-failures.focus-highlight .even th:focus,
|
||||||
|
.provider-failures.focus-highlight .even td:focus{
|
||||||
|
background-color:#dfdead;
|
||||||
|
color:#222
|
||||||
|
}
|
||||||
|
|
||||||
/* =======================================================================
|
/* =======================================================================
|
||||||
404.tmpl
|
404.tmpl
|
||||||
========================================================================== */
|
========================================================================== */
|
||||||
|
|
@ -1381,8 +1435,8 @@ tablesorter.css
|
||||||
}
|
}
|
||||||
|
|
||||||
thead.tablesorter-stickyHeader{
|
thead.tablesorter-stickyHeader{
|
||||||
border-top:2px solid #fff;
|
border-top:2px solid #ddd;
|
||||||
border-bottom:2px solid #fff
|
border-bottom:2px solid #ddd
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Zebra Widget - row alternating colors */
|
/* Zebra Widget - row alternating colors */
|
||||||
|
|
@ -1404,7 +1458,7 @@ thead.tablesorter-stickyHeader{
|
||||||
}
|
}
|
||||||
|
|
||||||
.tablesorter tfoot tr{
|
.tablesorter tfoot tr{
|
||||||
color:#fff;
|
color:#ddd;
|
||||||
text-align:center;
|
text-align:center;
|
||||||
text-shadow:-1px -1px 0 rgba(0, 0, 0, 0.3);
|
text-shadow:-1px -1px 0 rgba(0, 0, 0, 0.3);
|
||||||
background-color:#333;
|
background-color:#333;
|
||||||
|
|
|
||||||
|
|
@ -3191,6 +3191,85 @@ input.get_less_eps{
|
||||||
display:none
|
display:none
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#media-search .section{
|
||||||
|
padding-bottom:10px
|
||||||
|
}
|
||||||
|
#media-search .btn{
|
||||||
|
margin:0 6px 0 0;
|
||||||
|
min-width:70px
|
||||||
|
}
|
||||||
|
#media-search .btn.shows-more,
|
||||||
|
#media-search .btn.shows-less{
|
||||||
|
margin:6px 6px 6px 0;
|
||||||
|
}
|
||||||
|
#media-search .btn.provider-retry{
|
||||||
|
margin:6px 0 6px 4px;
|
||||||
|
}
|
||||||
|
.tablesorter.provider-failures{width:auto;clear:both;margin-bottom:10px}
|
||||||
|
.tablesorter.provider-failures > tbody > tr.tablesorter-childRow td{display:none}
|
||||||
|
.tablesorter.provider-failures.tablesorter > tbody > tr{background-color:transparent}
|
||||||
|
|
||||||
|
.provider-failures.hover-highlight th:hover::after,
|
||||||
|
.provider-failures.hover-highlight td:hover::after,
|
||||||
|
.provider-failures.focus-highlight th:focus::after,
|
||||||
|
.provider-failures.focus-highlight td:focus::after{
|
||||||
|
content:'';
|
||||||
|
position:absolute;
|
||||||
|
width:100%;
|
||||||
|
height:999em;
|
||||||
|
left:0;
|
||||||
|
top:-555em;
|
||||||
|
z-index:-1
|
||||||
|
}
|
||||||
|
.provider-failures.focus-highlight th:focus::before,
|
||||||
|
.provider-failures.focus-highlight td:focus::before{
|
||||||
|
content:'';
|
||||||
|
position:absolute;
|
||||||
|
width:999em;
|
||||||
|
height:100%;
|
||||||
|
left:-555em;
|
||||||
|
top:0;
|
||||||
|
z-index:-2
|
||||||
|
}
|
||||||
|
/* required styles */
|
||||||
|
.provider-failures.hover-highlight,
|
||||||
|
.provider-failures.focus-highlight{
|
||||||
|
overflow:hidden
|
||||||
|
}
|
||||||
|
.provider-failures.hover-highlight th,
|
||||||
|
.provider-failures.hover-highlight td,
|
||||||
|
.provider-failures.focus-highlight th,
|
||||||
|
.provider-failures.focus-highlight td{
|
||||||
|
position:relative;
|
||||||
|
outline:0
|
||||||
|
}
|
||||||
|
/* override the tablesorter theme styling */
|
||||||
|
.provider-failures.hover-highlight,
|
||||||
|
.provider-failures.hover-highlight tbody > tr > td,
|
||||||
|
.provider-failures.focus-highlight,
|
||||||
|
.provider-failures.focus-highlight tbody > tr > td,
|
||||||
|
/* override zebra styling */
|
||||||
|
.provider-failures.hover-highlight tbody tr.even > th,
|
||||||
|
.provider-failures.hover-highlight tbody tr.even > td,
|
||||||
|
.provider-failures.hover-highlight tbody tr.odd > th,
|
||||||
|
.provider-failures.hover-highlight tbody tr.odd > td,
|
||||||
|
.provider-failures.focus-highlight tbody tr.even > th,
|
||||||
|
.provider-failures.focus-highlight tbody tr.even > td,
|
||||||
|
.provider-failures.focus-highlight tbody tr.odd > th,
|
||||||
|
.provider-failures.focus-highlight tbody tr.odd > td{
|
||||||
|
background:transparent
|
||||||
|
}
|
||||||
|
/* table background positioned under the highlight */
|
||||||
|
.provider-failures.hover-highlight td:before,
|
||||||
|
.provider-failures.focus-highlight td:before{
|
||||||
|
content:'';
|
||||||
|
position:absolute;
|
||||||
|
width:100%;
|
||||||
|
height:100%;
|
||||||
|
left:0;
|
||||||
|
top:0;
|
||||||
|
z-index:-3
|
||||||
|
}
|
||||||
/* =======================================================================
|
/* =======================================================================
|
||||||
404.tmpl
|
404.tmpl
|
||||||
========================================================================== */
|
========================================================================== */
|
||||||
|
|
@ -4265,11 +4344,9 @@ tablesorter.css
|
||||||
#display-show .tablesorter{
|
#display-show .tablesorter{
|
||||||
width:100%;
|
width:100%;
|
||||||
margin-right:auto;
|
margin-right:auto;
|
||||||
margin-left:auto;
|
margin-left:auto
|
||||||
color:#000;
|
|
||||||
/* text-align:left;*/
|
/* text-align:left;*/
|
||||||
background-color:#ddd/*;
|
/* border-spacing:0*/
|
||||||
border-spacing:0*/
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#display-show .tablesorter{
|
#display-show .tablesorter{
|
||||||
|
|
@ -4317,20 +4394,6 @@ tablesorter.css
|
||||||
cursor:default
|
cursor:default
|
||||||
}
|
}
|
||||||
|
|
||||||
thead.tablesorter-stickyHeader{
|
|
||||||
border-top:2px solid #ddd;
|
|
||||||
border-bottom:2px solid #ddd
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Zebra Widget - row alternating colors */
|
|
||||||
.tablesorter tr.odd, .sickbeardTable tr.odd{
|
|
||||||
background-color:#f5f1e4
|
|
||||||
}
|
|
||||||
|
|
||||||
.tablesorter tr.even, .sickbeardTable tr.even{
|
|
||||||
background-color:#dfdacf
|
|
||||||
}
|
|
||||||
|
|
||||||
/* filter widget */
|
/* filter widget */
|
||||||
.tablesorter .filtered{
|
.tablesorter .filtered{
|
||||||
display:none
|
display:none
|
||||||
|
|
@ -4346,9 +4409,7 @@ thead.tablesorter-stickyHeader{
|
||||||
|
|
||||||
.tablesorter tr.tablesorter-filter-row,
|
.tablesorter tr.tablesorter-filter-row,
|
||||||
.tablesorter tr.tablesorter-filter-row td{
|
.tablesorter tr.tablesorter-filter-row td{
|
||||||
text-align:center;
|
text-align:center
|
||||||
background:#eee;
|
|
||||||
border-bottom:1px solid #ddd
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* optional disabled input styling */
|
/* optional disabled input styling */
|
||||||
|
|
@ -4362,10 +4423,7 @@ thead.tablesorter-stickyHeader{
|
||||||
}*/
|
}*/
|
||||||
|
|
||||||
.tablesorter tfoot tr{
|
.tablesorter tfoot tr{
|
||||||
color:#ddd;
|
|
||||||
text-align:center;
|
text-align:center;
|
||||||
text-shadow:-1px -1px 0 rgba(0, 0, 0, 0.3);
|
|
||||||
background-color:#333;
|
|
||||||
border-collapse:collapse
|
border-collapse:collapse
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -65,11 +65,7 @@ except AttributeError:
|
||||||
diskfree, min_output = df()
|
diskfree, min_output = df()
|
||||||
if min_output:
|
if min_output:
|
||||||
avail = ', '.join(['%s <span class="footerhighlight">%s</span>' % (drive, free) for (drive, free) in diskfree])
|
avail = ', '.join(['%s <span class="footerhighlight">%s</span>' % (drive, free) for (drive, free) in diskfree])
|
||||||
%>
|
%>#slurp#
|
||||||
<style>
|
|
||||||
.stat-table{margin:0 auto}
|
|
||||||
.stat-table > tbody > tr > td{padding:0 5px}
|
|
||||||
</style>
|
|
||||||
##
|
##
|
||||||
<span class="footerhighlight">$shows_total</span> shows (<span class="footerhighlight">$shows_active</span> active)
|
<span class="footerhighlight">$shows_total</span> shows (<span class="footerhighlight">$shows_active</span> active)
|
||||||
| <span class="footerhighlight">$ep_downloaded</span><%=
|
| <span class="footerhighlight">$ep_downloaded</span><%=
|
||||||
|
|
@ -87,6 +83,10 @@ if min_output:
|
||||||
<br>free space $avail
|
<br>free space $avail
|
||||||
#else
|
#else
|
||||||
<div class="table-responsive">
|
<div class="table-responsive">
|
||||||
|
<style>
|
||||||
|
.stat-table{margin:0 auto}
|
||||||
|
.stat-table > tbody > tr > td{padding:0 5px}
|
||||||
|
</style>
|
||||||
<table class="stat-table" cellspacing="5" cellpadding="5">
|
<table class="stat-table" cellspacing="5" cellpadding="5">
|
||||||
<caption style="display:none">Free space stats for volume/path</caption>
|
<caption style="display:none">Free space stats for volume/path</caption>
|
||||||
<tbody>
|
<tbody>
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
#import sickbeard
|
#import sickbeard
|
||||||
|
#from sickbeard import sbdatetime
|
||||||
##
|
##
|
||||||
#set global $title = 'Media Search'
|
#set global $title = 'Media Search'
|
||||||
#set global $header = 'Media Search'
|
#set global $header = 'Media Search'
|
||||||
|
|
@ -8,131 +9,229 @@
|
||||||
#import os.path
|
#import os.path
|
||||||
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_top.tmpl')
|
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_top.tmpl')
|
||||||
|
|
||||||
|
<input type="hidden" id="sbRoot" value="$sbRoot">
|
||||||
<script type="text/javascript" src="$sbRoot/js/plotTooltip.js?v=$sbPID"></script>
|
<script type="text/javascript" src="$sbRoot/js/plotTooltip.js?v=$sbPID"></script>
|
||||||
<script type="text/javascript" src="$sbRoot/js/manageSearches.js?v=$sbPID"></script>
|
<script type="text/javascript" src="$sbRoot/js/manageSearches.js?v=$sbPID"></script>
|
||||||
<div id="content800">
|
|
||||||
|
<div id="media-search" class="align-left">
|
||||||
|
|
||||||
#if $varExists('header')
|
#if $varExists('header')
|
||||||
<h1 class="header">$header</h1>
|
<h1 class="header">$header</h1>
|
||||||
#else
|
#else
|
||||||
<h1 class="title">$title</h1>
|
<h1 class="title">$title</h1>
|
||||||
#end if
|
#end if
|
||||||
|
|
||||||
<div id="summary2" class="align-left">
|
|
||||||
|
|
||||||
|
<div id="backlog-search" class="section">
|
||||||
<h3>Backlog Search:</h3>
|
<h3>Backlog Search:</h3>
|
||||||
<a id="forcebacklog" class="btn#if $standard_backlog_running or $backlog_is_active# disabled#end if#" href="$sbRoot/manage/manageSearches/forceBacklog"><i class="sgicon-play"></i> Force</a>
|
<a id="forcebacklog" class="btn#if $standard_backlog_running or $backlog_is_active# disabled#end if#" href="$sbRoot/manage/manageSearches/forceBacklog"><i class="sgicon-play"></i> Force</a>
|
||||||
<a id="pausebacklog" class="btn" href="$sbRoot/manage/manageSearches/pauseBacklog?paused=#if $backlog_paused then "0" else "1"#"><i class="#if $backlog_paused then "sgicon-play" else "sgicon-pause"#"></i> #if $backlog_paused then "Unpause" else "Pause"#</a>
|
<a id="pausebacklog" class="btn" href="$sbRoot/manage/manageSearches/pauseBacklog?paused=#if $backlog_paused then "0" else "1"#"><i class="#if $backlog_paused then "sgicon-play" else "sgicon-pause"#"></i> #if $backlog_paused then "Unpause" else "Pause"#</a>
|
||||||
#if $backlog_paused then 'Paused: ' else ''#
|
#if $backlog_paused
|
||||||
|
Paused:
|
||||||
|
#end if##slurp#
|
||||||
#if not $backlog_running and not $backlog_is_active:
|
#if not $backlog_running and not $backlog_is_active:
|
||||||
Not in progress<br />
|
Not in progress
|
||||||
#else
|
#else
|
||||||
Currently running#if $backlog_running_type != "None"# ($backlog_running_type)#end if#<br />
|
Currently running#if $backlog_running_type != "None"# ($backlog_running_type)#end if#
|
||||||
#end if
|
#end if
|
||||||
<br />
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div id="recent-search" class="section">
|
||||||
<h3>Recent Search:</h3>
|
<h3>Recent Search:</h3>
|
||||||
<a id="recentsearch" class="btn#if $recent_search_status# disabled#end if#" href="$sbRoot/manage/manageSearches/forceSearch"><i class="sgicon-play"></i> Force</a>
|
<a id="recentsearch" class="btn#if $recent_search_status# disabled#end if#" href="$sbRoot/manage/manageSearches/forceSearch"><i class="sgicon-play"></i> Force</a>
|
||||||
#if not $recent_search_status
|
#if not $recent_search_status
|
||||||
Not in progress<br />
|
Not in progress
|
||||||
#else
|
#else
|
||||||
In Progress<br />
|
In Progress
|
||||||
#end if
|
#end if
|
||||||
<br />
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div id="propers-search" class="section">
|
||||||
<h3>Find Propers Search:</h3>
|
<h3>Find Propers Search:</h3>
|
||||||
<a id="propersearch" class="btn#if $find_propers_status# disabled#end if#" href="$sbRoot/manage/manageSearches/forceFindPropers"><i class="sgicon-play"></i> Force</a>
|
<a id="propersearch" class="btn#if $find_propers_status# disabled#end if#" href="$sbRoot/manage/manageSearches/forceFindPropers"><i class="sgicon-play"></i> Force</a>
|
||||||
#if not $find_propers_status
|
#if not $find_propers_status
|
||||||
Not in progress<br />
|
Not in progress
|
||||||
#else
|
#else
|
||||||
In Progress<br />
|
In Progress
|
||||||
#end if
|
#end if
|
||||||
<br />
|
</div>
|
||||||
|
|
||||||
<br /><br />
|
|
||||||
|
|
||||||
<h3>Search Queue:</h3>
|
|
||||||
|
<div id="provider-failures" class="section">
|
||||||
|
<h3>Provider Failures:</h3>
|
||||||
|
#if not $provider_fails
|
||||||
|
<p>No current failures. Failure stats display here when appropriate.</p>
|
||||||
|
#else
|
||||||
|
<p>Some providers can be often down over periods, SickGear will back off then retry connecting at a later time</p>
|
||||||
|
|
||||||
|
#for $prov in $provider_fail_stats
|
||||||
|
#if $len($prov['fails'])
|
||||||
|
<input type="button" class="shows-more btn" id="$prov['name']-btn-more" value="Expand" style="display:none"><input type="button" class="shows-less btn" id="$prov['name']-btn-less" value="Collapse"><img src="$sbRoot/images/providers/$prov['prov_img']" width="16" height="16" style="margin:0 6px" />$prov['name']
|
||||||
|
#if $prov['active']
|
||||||
|
#if $prov['next_try']
|
||||||
|
#set nt = $str($prov['next_try']).split('.', 2)
|
||||||
|
... is blocked until $sbdatetime.sbdatetime.sbftime($sbdatetime.sbdatetime.now() + $prov['next_try'], markup=True) (in $nt[0]) <input type="button" class="provider-retry btn" id="$prov['prov_id']-btn-retry" value="Ignore block on next search">
|
||||||
|
#end if
|
||||||
|
#else
|
||||||
|
... is not enabled
|
||||||
|
#end if
|
||||||
|
<table class="manageTable provider-failures tablesorter hover-highlight focus-highlight text-center" cellspacing="0" border="0" cellpadding="0">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th class="text-center" style="width:13em;padding-right:20px">period of 1hr</th>
|
||||||
|
<th class="text-center" style="padding-right:20px">server/timeout</th>
|
||||||
|
<th class="text-center" style="padding-right:20px">network</th>
|
||||||
|
<th class="text-center" style="padding-right:20px">no data</th>
|
||||||
|
<th class="text-center" style="padding-right:20px">other</th>
|
||||||
|
#if $prov['has_limit']
|
||||||
|
<th class="text-center" style="padding-right:20px">hit limit</th>
|
||||||
|
#end if
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
#set $day = []
|
||||||
|
#for $fail in $prov['fails']
|
||||||
|
#set $child = True
|
||||||
|
#if $fail['date'] not in $day
|
||||||
|
#set $day += [$fail['date']]
|
||||||
|
#set $child = False
|
||||||
|
#end if
|
||||||
|
#slurp#
|
||||||
|
<tr#if $fail['multirow'] and $child# class="tablesorter-childRow"#end if#>
|
||||||
|
#if $fail['multirow']
|
||||||
|
#if not $child
|
||||||
|
<td><a href="#" class="provider-fail-parent-toggle" title="Totals (expand for detail)">$sbdatetime.sbdatetime.sbfdate($fail['date_time'])</a></td>
|
||||||
|
#else
|
||||||
|
<td>$sbdatetime.sbdatetime.sbftime($fail['date_time'], markup=True)</td>
|
||||||
|
#end if
|
||||||
|
#else
|
||||||
|
<td>$sbdatetime.sbdatetime.sbfdatetime($fail['date_time'], markup=True)</td>
|
||||||
|
#end if
|
||||||
|
#set $blank = '-'
|
||||||
|
#set $title=None
|
||||||
|
#if $fail['http']['count']
|
||||||
|
#set $title=$fail['http']['code']
|
||||||
|
#end if
|
||||||
|
<td>#if $fail['http']['count']#<span title="#if $child#$title#else#Expand for fail codes#end if#">$fail['http']['count']</span>#else#$blank#end if# / #echo $fail['timeout'].get('count', 0) or $blank#</td>
|
||||||
|
<td>#echo ($fail['connection'].get('count', 0) + $fail['connection_timeout'].get('count', 0)) or $blank#</td>
|
||||||
|
<td>#echo $fail['nodata'].get('count', 0) or $blank#</td>
|
||||||
|
<td>#echo $fail['other'].get('count', 0) or $blank#</td>
|
||||||
|
#if $prov['has_limit']
|
||||||
|
<td>#echo $fail.get('limit', {}).get('count', 0) or $blank#</td>
|
||||||
|
#end if
|
||||||
|
</tr>
|
||||||
|
#end for
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
#end if
|
||||||
|
#end for
|
||||||
|
#end if
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div id="search-queues" class="section">
|
||||||
|
<h3>Search Queues:</h3>
|
||||||
|
|
||||||
#if $queue_length['backlog'] or $queue_length['manual'] or $queue_length['failed']
|
#if $queue_length['backlog'] or $queue_length['manual'] or $queue_length['failed']
|
||||||
<input type="button" class="show-all-more btn" id="all-btn-more" value="Expand All"><input type="button" class="show-all-less btn" id="all-btn-less" value="Collapse All"><br>
|
<input type="button" class="show-all-more btn" id="all-btn-more" value="Expand All"><input type="button" class="show-all-less btn" id="all-btn-less" value="Collapse All"><br>
|
||||||
#end if
|
#end if
|
||||||
<br>
|
<div id="queue-recent" class="section">
|
||||||
Recent: <i>$queue_length['recent'] item$sickbeard.helpers.maybe_plural($queue_length['recent'])</i><br><br>
|
Recent: <i>$queue_length['recent'] item$sickbeard.helpers.maybe_plural($queue_length['recent'])</i>
|
||||||
Proper: <i>$queue_length['proper'] item$sickbeard.helpers.maybe_plural($queue_length['proper'])</i><br><br>
|
</div>
|
||||||
Backlog: <i>$len($queue_length['backlog']) item$sickbeard.helpers.maybe_plural($len($queue_length['backlog']))</i>
|
|
||||||
|
|
||||||
|
<div id="queue-proper" class="section">
|
||||||
|
Proper: <i>$queue_length['proper'] item$sickbeard.helpers.maybe_plural($queue_length['proper'])</i>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<div id="queue-backlog" class="section">
|
||||||
|
Backlog: <i>$len($queue_length['backlog']) item$sickbeard.helpers.maybe_plural($len($queue_length['backlog']))</i>
|
||||||
#if $queue_length['backlog']
|
#if $queue_length['backlog']
|
||||||
<input type="button" class="shows-more btn" id="backlog-btn-more" value="Expand" #if not $queue_length['backlog']# style="display:none" #end if#><input type="button" class="shows-less btn" id="backlog-btn-less" value="Collapse" style="display:none"><br>
|
<input type="button" class="shows-more btn" id="backlog-btn-more" value="Expand" #if not $queue_length['backlog']# style="display:none" #end if#><input type="button" class="shows-less btn" id="backlog-btn-less" value="Collapse" style="display:none"><br>
|
||||||
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
|
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
|
||||||
<thead></thead>
|
<thead></thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
#set $row = 0
|
#set $row = 0
|
||||||
#for $cur_item in $queue_length['backlog']:
|
#for $cur_item in $queue_length['backlog']:
|
||||||
#set $search_type = 'On Demand'
|
#set $search_type = 'On Demand'
|
||||||
#if $cur_item['standard_backlog']:
|
#if $cur_item['standard_backlog']:
|
||||||
#if $cur_item['forced']:
|
#if $cur_item['forced']:
|
||||||
#set $search_type = 'Forced'
|
#set $search_type = 'Forced'
|
||||||
#else
|
#else
|
||||||
#set $search_type = 'Scheduled'
|
#set $search_type = 'Scheduled'
|
||||||
#end if
|
#end if
|
||||||
#if $cur_item['torrent_only']:
|
#if $cur_item['torrent_only']:
|
||||||
#set $search_type += ', Torrent Only'
|
#set $search_type += ', Torrent Only'
|
||||||
#end if
|
#end if
|
||||||
#if $cur_item['limited_backlog']:
|
#if $cur_item['limited_backlog']:
|
||||||
#set $search_type += ' (Limited)'
|
#set $search_type += ' (Limited)'
|
||||||
#else
|
#else
|
||||||
#set $search_type += ' (Full)'
|
#set $search_type += ' (Full)'
|
||||||
#end if
|
#end if
|
||||||
#end if
|
#end if
|
||||||
<tr class="#echo ('odd', 'even')[$row % 2]##set $row+=1#">
|
<tr class="#echo ('odd', 'even')[$row % 2]##set $row+=1#">
|
||||||
<td style="width:80%;text-align:left;color:white">
|
<td style="width:80%;text-align:left;color:white">
|
||||||
<a class="whitelink" href="$sbRoot/home/displayShow?show=$cur_item['indexerid']">$cur_item['name']</a> - $sickbeard.helpers.make_search_segment_html_string($cur_item['segment'])
|
<a class="whitelink" href="$sbRoot/home/displayShow?show=$cur_item['indexerid']">$cur_item['name']</a> - $sickbeard.helpers.make_search_segment_html_string($cur_item['segment'])
|
||||||
</td>
|
</td>
|
||||||
<td style="width:20%;text-align:center;color:white">$search_type</td>
|
<td style="width:20%;text-align:center;color:white">$search_type</td>
|
||||||
</tr>
|
</tr>
|
||||||
#end for
|
#end for
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
#else
|
|
||||||
<br>
|
|
||||||
#end if
|
#end if
|
||||||
<br>
|
</div>
|
||||||
Manual: <i>$len($queue_length['manual']) item$sickbeard.helpers.maybe_plural($len($queue_length['manual']))</i>
|
|
||||||
|
|
||||||
|
<div id="queue-manual" class="section">
|
||||||
|
Manual: <i>$len($queue_length['manual']) item$sickbeard.helpers.maybe_plural($len($queue_length['manual']))</i>
|
||||||
#if $queue_length['manual']
|
#if $queue_length['manual']
|
||||||
<input type="button" class="shows-more btn" id="manual-btn-more" value="Expand" #if not $queue_length['manual']# style="display:none" #end if#><input type="button" class="shows-less btn" id="manual-btn-less" value="Collapse" style="display:none"><br>
|
<input type="button" class="shows-more btn" id="manual-btn-more" value="Expand" #if not $queue_length['manual']# style="display:none" #end if#><input type="button" class="shows-less btn" id="manual-btn-less" value="Collapse" style="display:none"><br>
|
||||||
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
|
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
|
||||||
<thead></thead>
|
<thead></thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
#set $row = 0
|
#set $row = 0
|
||||||
#for $cur_item in $queue_length['manual']:
|
#for $cur_item in $queue_length['manual']:
|
||||||
<tr class="#echo ('odd', 'even')[$row % 2]##set $row+=1#">
|
<tr class="#echo ('odd', 'even')[$row % 2]##set $row+=1#">
|
||||||
<td style="width:100%;text-align:left;color:white">
|
<td style="width:100%;text-align:left;color:white">
|
||||||
<a class="whitelink" href="$sbRoot/home/displayShow?show=$cur_item['indexerid']">$cur_item['name']</a> - $sickbeard.helpers.make_search_segment_html_string($cur_item['segment'])
|
<a class="whitelink" href="$sbRoot/home/displayShow?show=$cur_item['indexerid']">$cur_item['name']</a> - $sickbeard.helpers.make_search_segment_html_string($cur_item['segment'])
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
#end for
|
#end for
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
#else
|
|
||||||
<br>
|
|
||||||
#end if
|
#end if
|
||||||
<br>
|
</div>
|
||||||
Failed: <i>$len($queue_length['failed']) item$sickbeard.helpers.maybe_plural($len($queue_length['failed']))</i>
|
|
||||||
|
|
||||||
|
<div id="queue-failed" class="section">
|
||||||
|
Failed: <i>$len($queue_length['failed']) item$sickbeard.helpers.maybe_plural($len($queue_length['failed']))</i>
|
||||||
#if $queue_length['failed']
|
#if $queue_length['failed']
|
||||||
<input type="button" class="shows-more btn" id="failed-btn-more" value="Expand" #if not $queue_length['failed']# style="display:none" #end if#><input type="button" class="shows-less btn" id="failed-btn-less" value="Collapse" style="display:none"><br>
|
<input type="button" class="shows-more btn" id="failed-btn-more" value="Expand" #if not $queue_length['failed']# style="display:none" #end if#><input type="button" class="shows-less btn" id="failed-btn-less" value="Collapse" style="display:none"><br>
|
||||||
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
|
<table class="sickbeardTable manageTable" cellspacing="1" border="0" cellpadding="0" style="display:none">
|
||||||
<thead></thead>
|
<thead></thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
#set $row = 0
|
#set $row = 0
|
||||||
#for $cur_item in $queue_length['failed']:
|
#for $cur_item in $queue_length['failed']:
|
||||||
<tr class="#echo ('odd', 'even')[$row % 2]##set $row+=1#">
|
<tr class="#echo ('odd', 'even')[$row % 2]##set $row+=1#">
|
||||||
<td style="width:100%;text-align:left;color:white">
|
<td style="width:100%;text-align:left;color:white">
|
||||||
<a class="whitelink" href="$sbRoot/home/displayShow?show=$cur_item['indexerid']">$cur_item['name']</a> - $sickbeard.helpers.make_search_segment_html_string($cur_item['segment'])
|
<a class="whitelink" href="$sbRoot/home/displayShow?show=$cur_item['indexerid']">$cur_item['name']</a> - $sickbeard.helpers.make_search_segment_html_string($cur_item['segment'])
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
#end for
|
#end for
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
#else
|
|
||||||
<br>
|
|
||||||
#end if
|
#end if
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')
|
#include $os.path.join($sickbeard.PROG_DIR, 'gui/slick/interfaces/default/inc_bottom.tmpl')
|
||||||
|
|
|
||||||
|
|
@ -1,33 +1,66 @@
|
||||||
$(document).ready(function() {
|
$(function(){
|
||||||
$('#recentsearch,#propersearch').click(function(){
|
$('#recentsearch,#propersearch').click(function(){
|
||||||
$(this).addClass('disabled');
|
$(this).addClass('disabled');
|
||||||
})
|
});
|
||||||
$('#forcebacklog,#forcefullbacklog').click(function(){
|
$('#forcebacklog,#forcefullbacklog').click(function(){
|
||||||
$('#forcebacklog,#forcefullbacklog').addClass('disabled');
|
$('#forcebacklog,#forcefullbacklog').addClass('disabled');
|
||||||
$('#pausebacklog').removeClass('disabled');
|
$('#pausebacklog').removeClass('disabled');
|
||||||
})
|
});
|
||||||
$('#pausebacklog').click(function(){
|
$('#pausebacklog').click(function(){
|
||||||
$(this).addClass('disabled');
|
$(this).addClass('disabled');
|
||||||
})
|
});
|
||||||
$('.show-all-less').click(function(){
|
$('.show-all-less').click(function(){
|
||||||
$(this).nextAll('table').hide();
|
$(this).nextAll('table').hide();
|
||||||
$(this).nextAll('input.shows-more').show();
|
$(this).nextAll('input.shows-more').show();
|
||||||
$(this).nextAll('input.shows-less').hide();
|
$(this).nextAll('input.shows-less').hide();
|
||||||
})
|
});
|
||||||
$('.show-all-more').click(function(){
|
$('.show-all-more').click(function(){
|
||||||
$(this).nextAll('table').show();
|
$(this).nextAll('table').show();
|
||||||
$(this).nextAll('input.shows-more').hide();
|
$(this).nextAll('input.shows-more').hide();
|
||||||
$(this).nextAll('input.shows-less').show();
|
$(this).nextAll('input.shows-less').show();
|
||||||
})
|
});
|
||||||
|
|
||||||
$('.shows-less').click(function(){
|
$('.shows-less').click(function(){
|
||||||
$(this).nextAll('table:first').hide();
|
$(this).nextAll('table:first').hide();
|
||||||
$(this).hide();
|
$(this).hide();
|
||||||
$(this).prevAll('input:first').show();
|
$(this).prevAll('input:first').show();
|
||||||
})
|
});
|
||||||
$('.shows-more').click(function(){
|
$('.shows-more').click(function(){
|
||||||
$(this).nextAll('table:first').show();
|
$(this).nextAll('table:first').show();
|
||||||
$(this).hide();
|
$(this).hide();
|
||||||
$(this).nextAll('input:first').show();
|
$(this).nextAll('input:first').show();
|
||||||
})
|
});
|
||||||
});
|
$('.provider-retry').click(function () {
|
||||||
|
$(this).addClass('disabled');
|
||||||
|
var match = $(this).attr('id').match(/^(.+)-btn-retry$/);
|
||||||
|
$.ajax({
|
||||||
|
url: sbRoot + '/manage/manageSearches/retryProvider?provider=' + match[1],
|
||||||
|
type: 'GET',
|
||||||
|
complete: function () {
|
||||||
|
window.location.reload(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
$('.provider-failures').tablesorter({widgets : ['zebra'],
|
||||||
|
headers : { 0:{sorter:!1}, 1:{sorter:!1}, 2:{sorter:!1}, 3:{sorter:!1}, 4:{sorter:!1}, 5:{sorter:!1} }
|
||||||
|
});
|
||||||
|
|
||||||
|
$('.provider-fail-parent-toggle').click(function(){
|
||||||
|
$(this).closest('tr').nextUntil('tr:not(.tablesorter-childRow)').find('td').toggle();
|
||||||
|
return !1;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Make table cell focusable
|
||||||
|
// http://css-tricks.com/simple-css-row-column-highlighting/
|
||||||
|
var focus$ = $('.focus-highlight');
|
||||||
|
if (focus$.length){
|
||||||
|
focus$.find('td, th')
|
||||||
|
.attr('tabindex', '1')
|
||||||
|
// add touch device support
|
||||||
|
.on('touchstart', function(){
|
||||||
|
$(this).focus();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
|
||||||
|
|
@ -17,91 +17,89 @@
|
||||||
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
|
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from sickbeard import db
|
from sickbeard import db
|
||||||
|
from collections import OrderedDict
|
||||||
|
import re
|
||||||
|
|
||||||
MIN_DB_VERSION = 1
|
MIN_DB_VERSION = 1
|
||||||
MAX_DB_VERSION = 3
|
MAX_DB_VERSION = 4
|
||||||
|
|
||||||
|
|
||||||
# Add new migrations at the bottom of the list; subclass the previous migration.
|
# Add new migrations at the bottom of the list; subclass the previous migration.
|
||||||
class InitialSchema(db.SchemaUpgrade):
|
class InitialSchema(db.SchemaUpgrade):
|
||||||
|
def __init__(self, connection):
|
||||||
|
super(InitialSchema, self).__init__(connection)
|
||||||
|
|
||||||
|
self.queries = OrderedDict([
|
||||||
|
('base', [
|
||||||
|
'CREATE TABLE lastUpdate(provider TEXT, time NUMERIC)',
|
||||||
|
'CREATE TABLE lastSearch(provider TEXT, time NUMERIC)',
|
||||||
|
'CREATE TABLE db_version(db_version INTEGER)',
|
||||||
|
'INSERT INTO db_version(db_version) VALUES (1)',
|
||||||
|
'CREATE TABLE network_timezones(network_name TEXT PRIMARY KEY, timezone TEXT)'
|
||||||
|
]),
|
||||||
|
('consolidate_providers', [
|
||||||
|
'CREATE TABLE provider_cache(provider TEXT, name TEXT, season NUMERIC, episodes TEXT,'
|
||||||
|
' indexerid NUMERIC, url TEXT UNIQUE, time NUMERIC, quality TEXT, release_group TEXT, version NUMERIC)',
|
||||||
|
'CREATE TABLE network_conversions('
|
||||||
|
'tvdb_network TEXT PRIMARY KEY, tvrage_network TEXT, tvrage_country TEXT)',
|
||||||
|
'CREATE INDEX tvrage_idx ON network_conversions(tvrage_network, tvrage_country)'
|
||||||
|
]),
|
||||||
|
('add_backlogparts', [
|
||||||
|
'CREATE TABLE backlogparts('
|
||||||
|
'part NUMERIC NOT NULL, indexer NUMERIC NOT NULL, indexerid NUMERIC NOT NULL)',
|
||||||
|
'CREATE TABLE lastrecentsearch(name TEXT PRIMARY KEY NOT NULL, datetime NUMERIC NOT NULL)'
|
||||||
|
]),
|
||||||
|
('add_provider_fails', [
|
||||||
|
'CREATE TABLE provider_fails(prov_name TEXT, fail_type INTEGER, fail_code INTEGER, fail_time NUMERIC)',
|
||||||
|
'CREATE INDEX idx_prov_name_error ON provider_fails (prov_name)',
|
||||||
|
'CREATE UNIQUE INDEX idx_prov_errors ON provider_fails (prov_name, fail_time)',
|
||||||
|
'CREATE TABLE provider_fails_count(prov_name TEXT PRIMARY KEY,'
|
||||||
|
' failure_count NUMERIC, failure_time NUMERIC,'
|
||||||
|
' tmr_limit_count NUMERIC, tmr_limit_time NUMERIC, tmr_limit_wait NUMERIC)'
|
||||||
|
])
|
||||||
|
])
|
||||||
|
|
||||||
def test(self):
|
def test(self):
|
||||||
return self.hasTable('lastUpdate')
|
return self.hasTable('lastUpdate')
|
||||||
|
|
||||||
def execute(self):
|
def execute(self):
|
||||||
queries = [
|
self.do_query(self.queries.values())
|
||||||
'CREATE TABLE lastUpdate (provider TEXT, time NUMERIC)',
|
self.setDBVersion(MAX_DB_VERSION)
|
||||||
'CREATE TABLE lastSearch (provider TEXT, time NUMERIC)',
|
|
||||||
'CREATE TABLE db_version (db_version INTEGER)',
|
def backup(self):
|
||||||
'INSERT INTO db_version (db_version) VALUES (1)',
|
db.backup_database('cache.db', self.checkDBVersion())
|
||||||
'CREATE TABLE network_timezones (network_name TEXT PRIMARY KEY, timezone TEXT)',
|
|
||||||
'CREATE TABLE network_conversions ('
|
|
||||||
'tvdb_network TEXT PRIMARY KEY, tvrage_network TEXT, tvrage_country TEXT)',
|
|
||||||
'CREATE INDEX tvrage_idx on network_conversions (tvrage_network, tvrage_country)',
|
|
||||||
'CREATE TABLE provider_cache (provider TEXT ,name TEXT, season NUMERIC, episodes TEXT,'
|
|
||||||
' indexerid NUMERIC, url TEXT UNIQUE, time NUMERIC, quality TEXT, release_group TEXT, '
|
|
||||||
'version NUMERIC)',
|
|
||||||
'CREATE TABLE IF NOT EXISTS "backlogparts" ("part" NUMERIC NOT NULL ,'
|
|
||||||
' "indexer" NUMERIC NOT NULL , "indexerid" NUMERIC NOT NULL )',
|
|
||||||
'CREATE TABLE IF NOT EXISTS "lastrecentsearch" ("name" TEXT PRIMARY KEY NOT NULL'
|
|
||||||
' , "datetime" NUMERIC NOT NULL )',
|
|
||||||
]
|
|
||||||
for query in queries:
|
|
||||||
self.connection.action(query)
|
|
||||||
self.setDBVersion(3)
|
|
||||||
|
|
||||||
|
|
||||||
class ConsolidateProviders(InitialSchema):
|
class ConsolidateProviders(InitialSchema):
|
||||||
def test(self):
|
def test(self):
|
||||||
return self.checkDBVersion() > 1
|
return 1 < self.checkDBVersion()
|
||||||
|
|
||||||
def execute(self):
|
def execute(self):
|
||||||
|
self.backup()
|
||||||
db.backup_database('cache.db', self.checkDBVersion())
|
keep_tables = {'lastUpdate', 'lastSearch', 'db_version',
|
||||||
if self.hasTable('provider_cache'):
|
'network_timezones', 'network_conversions', 'provider_cache'}
|
||||||
self.connection.action('DROP TABLE provider_cache')
|
# old provider_cache is dropped before re-creation
|
||||||
|
self.do_query(['DROP TABLE [provider_cache]'] + self.queries['consolidate_providers'] +
|
||||||
self.connection.action('CREATE TABLE provider_cache (provider TEXT, name TEXT, season NUMERIC, episodes TEXT, '
|
['DROP TABLE [%s]' % t for t in (set(self.listTables()) - keep_tables)])
|
||||||
'indexerid NUMERIC, url TEXT UNIQUE, time NUMERIC, quality TEXT, release_group TEXT, '
|
self.finish(True)
|
||||||
'version NUMERIC)')
|
|
||||||
|
|
||||||
if not self.hasTable('network_conversions'):
|
|
||||||
self.connection.action('CREATE TABLE network_conversions ' +
|
|
||||||
'(tvdb_network TEXT PRIMARY KEY, tvrage_network TEXT, tvrage_country TEXT)')
|
|
||||||
self.connection.action('CREATE INDEX tvrage_idx ' +
|
|
||||||
'on network_conversions (tvrage_network, tvrage_country)')
|
|
||||||
|
|
||||||
keep_tables = set(['lastUpdate', 'lastSearch', 'db_version',
|
|
||||||
'network_timezones', 'network_conversions', 'provider_cache'])
|
|
||||||
current_tables = set(self.listTables())
|
|
||||||
remove_tables = list(current_tables - keep_tables)
|
|
||||||
for table in remove_tables:
|
|
||||||
self.connection.action('DROP TABLE [%s]' % table)
|
|
||||||
|
|
||||||
self.incDBVersion()
|
|
||||||
|
|
||||||
|
|
||||||
class AddBacklogParts(ConsolidateProviders):
|
class AddBacklogParts(ConsolidateProviders):
|
||||||
def test(self):
|
def test(self):
|
||||||
return self.checkDBVersion() > 2
|
return 2 < self.checkDBVersion()
|
||||||
|
|
||||||
def execute(self):
|
def execute(self):
|
||||||
|
self.backup()
|
||||||
|
self.do_query(self.queries['add_backlogparts'] +
|
||||||
|
['DROP TABLE [%s]' % t for t in ('scene_names', 'scene_exceptions_refresh', 'scene_exceptions')])
|
||||||
|
self.finish(True)
|
||||||
|
|
||||||
db.backup_database('cache.db', self.checkDBVersion())
|
|
||||||
if self.hasTable('scene_names'):
|
|
||||||
self.connection.action('DROP TABLE scene_names')
|
|
||||||
|
|
||||||
if not self.hasTable('backlogparts'):
|
class AddProviderFailureHandling(AddBacklogParts):
|
||||||
self.connection.action('CREATE TABLE IF NOT EXISTS "backlogparts" ("part" NUMERIC NOT NULL ,'
|
def test(self):
|
||||||
' "indexer" NUMERIC NOT NULL , "indexerid" NUMERIC NOT NULL )')
|
return 3 < self.checkDBVersion()
|
||||||
|
|
||||||
if not self.hasTable('lastrecentsearch'):
|
def execute(self):
|
||||||
self.connection.action('CREATE TABLE IF NOT EXISTS "lastrecentsearch" ("name" TEXT PRIMARY KEY NOT NULL'
|
self.backup()
|
||||||
' , "datetime" NUMERIC NOT NULL )')
|
self.do_query(self.queries['add_provider_fails'])
|
||||||
|
self.finish()
|
||||||
if self.hasTable('scene_exceptions_refresh'):
|
|
||||||
self.connection.action('DROP TABLE scene_exceptions_refresh')
|
|
||||||
if self.hasTable('scene_exceptions'):
|
|
||||||
self.connection.action('DROP TABLE scene_exceptions')
|
|
||||||
self.connection.action('VACUUM')
|
|
||||||
|
|
||||||
self.incDBVersion()
|
|
||||||
|
|
|
||||||
|
|
@ -432,6 +432,26 @@ class SchemaUpgrade(object):
|
||||||
tables.append(table[0])
|
tables.append(table[0])
|
||||||
return tables
|
return tables
|
||||||
|
|
||||||
|
def do_query(self, queries):
|
||||||
|
if not isinstance(queries, list):
|
||||||
|
queries = list(queries)
|
||||||
|
elif isinstance(queries[0], list):
|
||||||
|
queries = [item for sublist in queries for item in sublist]
|
||||||
|
|
||||||
|
for query in queries:
|
||||||
|
tbl_name = re.findall('(?i)DROP.*?TABLE.*?\[?([^\s\]]+)', query)
|
||||||
|
if tbl_name and not self.hasTable(tbl_name[0]):
|
||||||
|
continue
|
||||||
|
tbl_name = re.findall('(?i)CREATE.*?TABLE.*?\s([^\s(]+)\s*\(', query)
|
||||||
|
if tbl_name and self.hasTable(tbl_name[0]):
|
||||||
|
continue
|
||||||
|
self.connection.action(query)
|
||||||
|
|
||||||
|
def finish(self, tbl_dropped=False):
|
||||||
|
if tbl_dropped:
|
||||||
|
self.connection.action('VACUUM')
|
||||||
|
self.incDBVersion()
|
||||||
|
|
||||||
|
|
||||||
def MigrationCode(myDB):
|
def MigrationCode(myDB):
|
||||||
schema = {
|
schema = {
|
||||||
|
|
|
||||||
|
|
@ -65,6 +65,8 @@ class AlphaRatioProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (search_string, ('&freetorrent=1', '')[not self.freeleech])
|
search_url = self.urls['search'] % (search_string, ('&freetorrent=1', '')[not self.freeleech])
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ class AnizbProvider(generic.NZBProvider):
|
||||||
for params in search_params[mode]:
|
for params in search_params[mode]:
|
||||||
|
|
||||||
search_url = '%sapi/%s' % (self.url, params and (('?q=%s', '?q=%(q)s')['q' in params] % params) or '')
|
search_url = '%sapi/%s' % (self.url, params and (('?q=%s', '?q=%(q)s')['q' in params] % params) or '')
|
||||||
data = self.cache.getRSSFeed(search_url)
|
data = self.cache.get_rss(search_url)
|
||||||
time.sleep(1.1)
|
time.sleep(1.1)
|
||||||
|
|
||||||
cnt = len(results)
|
cnt = len(results)
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,8 @@ class BeyondHDProvider(generic.TorrentProvider):
|
||||||
search_url += self.urls['search'] % re.sub('[.\s]+', ' ', search_string)
|
search_url += self.urls['search'] % re.sub('[.\s]+', ' ', search_string)
|
||||||
|
|
||||||
data_json = self.get_url(search_url, json=True)
|
data_json = self.get_url(search_url, json=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
if data_json and 'results' in data_json and self._check_auth_from_data(data_json):
|
if data_json and 'results' in data_json and self._check_auth_from_data(data_json):
|
||||||
|
|
|
||||||
|
|
@ -71,6 +71,8 @@ class BitHDTVProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (search_string, self._categories_string(mode))
|
search_url = self.urls['search'] % (search_string, self._categories_string(mode))
|
||||||
|
|
||||||
html = self.get_url(search_url, timeout=90)
|
html = self.get_url(search_url, timeout=90)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -64,6 +64,8 @@ class BitmetvProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (self._categories_string(mode, 'cat=%s'), search_string)
|
search_url = self.urls['search'] % (self._categories_string(mode, 'cat=%s'), search_string)
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,8 @@ class BlutopiaProvider(generic.TorrentProvider):
|
||||||
self.token, '+'.join(search_string.split()), self._categories_string(mode, ''), '', '', '')
|
self.token, '+'.join(search_string.split()), self._categories_string(mode, ''), '', '', '')
|
||||||
|
|
||||||
resp = self.get_url(search_url, json=True)
|
resp = self.get_url(search_url, json=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,7 @@ class BTNProvider(generic.TorrentProvider):
|
||||||
self.ua = self.session.headers['User-Agent']
|
self.ua = self.session.headers['User-Agent']
|
||||||
self.reject_m2ts = False
|
self.reject_m2ts = False
|
||||||
self.cache = BTNCache(self)
|
self.cache = BTNCache(self)
|
||||||
|
self.has_limit = True
|
||||||
|
|
||||||
def _authorised(self, **kwargs):
|
def _authorised(self, **kwargs):
|
||||||
|
|
||||||
|
|
@ -67,6 +68,15 @@ class BTNProvider(generic.TorrentProvider):
|
||||||
raise AuthException('Must set Api key or Username/Password for %s in config provider options' % self.name)
|
raise AuthException('Must set Api key or Username/Password for %s in config provider options' % self.name)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def _check_response(self, data, url, post_data=None, post_json=None):
|
||||||
|
if not self.should_skip(log_warning=False):
|
||||||
|
if data and 'Call Limit' in data:
|
||||||
|
self.tmr_limit_update('1', 'h', '150/hr %s' % data)
|
||||||
|
self.log_failure_url(url, post_data, post_json)
|
||||||
|
else:
|
||||||
|
logger.log(u'Action prematurely ended. %(prov)s server error response = %(desc)s' %
|
||||||
|
{'prov': self.name, 'desc': data}, logger.WARNING)
|
||||||
|
|
||||||
def _search_provider(self, search_params, age=0, **kwargs):
|
def _search_provider(self, search_params, age=0, **kwargs):
|
||||||
|
|
||||||
self._authorised()
|
self._authorised()
|
||||||
|
|
@ -93,21 +103,19 @@ class BTNProvider(generic.TorrentProvider):
|
||||||
self.api_key, json.dumps(param_dct), items_per_page, offset))
|
self.api_key, json.dumps(param_dct), items_per_page, offset))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = None
|
response, error_text = None, None
|
||||||
if api_up and self.api_key:
|
if api_up and self.api_key:
|
||||||
self.session.headers['Content-Type'] = 'application/json-rpc'
|
self.session.headers['Content-Type'] = 'application/json-rpc'
|
||||||
response = helpers.getURL(
|
response = self.get_url(self.url_api, post_data=json_rpc(params), json=True)
|
||||||
self.url_api, post_data=json_rpc(params), session=self.session, json=True)
|
# response = {'error': {'message': 'Call Limit Exceeded Test'}}
|
||||||
if not response:
|
error_text = response['error']['message']
|
||||||
api_up = False
|
api_up = False
|
||||||
results = self.html(mode, search_string, results)
|
if 'Propers' == mode:
|
||||||
error_text = response['error']['message']
|
return results
|
||||||
logger.log(
|
results = self.html(mode, search_string, results)
|
||||||
('Call Limit' in error_text
|
if not results:
|
||||||
and u'Action aborted because the %(prov)s 150 calls/hr limit was reached'
|
self._check_response(error_text, self.url_api, post_data=json_rpc(params))
|
||||||
or u'Action prematurely ended. %(prov)s server error response = %(desc)s') %
|
return results
|
||||||
{'prov': self.name, 'desc': error_text}, logger.WARNING)
|
|
||||||
return results
|
|
||||||
except AuthException:
|
except AuthException:
|
||||||
logger.log('API looks to be down, add un/pw config detail to be used as a fallback', logger.WARNING)
|
logger.log('API looks to be down, add un/pw config detail to be used as a fallback', logger.WARNING)
|
||||||
except (KeyError, Exception):
|
except (KeyError, Exception):
|
||||||
|
|
@ -115,7 +123,7 @@ class BTNProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
data_json = response and 'result' in response and response['result'] or {}
|
data_json = response and 'result' in response and response['result'] or {}
|
||||||
if data_json:
|
if data_json:
|
||||||
|
self.tmr_limit_count = 0
|
||||||
found_torrents = 'torrents' in data_json and data_json['torrents'] or {}
|
found_torrents = 'torrents' in data_json and data_json['torrents'] or {}
|
||||||
|
|
||||||
# We got something, we know the API sends max 1000 results at a time.
|
# We got something, we know the API sends max 1000 results at a time.
|
||||||
|
|
@ -134,15 +142,10 @@ class BTNProvider(generic.TorrentProvider):
|
||||||
for page in range(1, pages_needed + 1):
|
for page in range(1, pages_needed + 1):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = helpers.getURL(
|
post_data = json_rpc(params, results_per_page, page * results_per_page)
|
||||||
self.url_api, json=True, session=self.session,
|
response = self.get_url(self.url_api, json=True, post_data=post_data)
|
||||||
post_data=json_rpc(params, results_per_page, page * results_per_page))
|
|
||||||
error_text = response['error']['message']
|
error_text = response['error']['message']
|
||||||
logger.log(
|
self._check_response(error_text, self.url_api, post_data=post_data)
|
||||||
('Call Limit' in error_text
|
|
||||||
and u'Action prematurely ended because the %(prov)s 150 calls/hr limit was reached'
|
|
||||||
or u'Action prematurely ended. %(prov)s server error response = %(desc)s') %
|
|
||||||
{'prov': self.name, 'desc': error_text}, logger.WARNING)
|
|
||||||
return results
|
return results
|
||||||
except (KeyError, Exception):
|
except (KeyError, Exception):
|
||||||
data_json = response and 'result' in response and response['result'] or {}
|
data_json = response and 'result' in response and response['result'] or {}
|
||||||
|
|
@ -150,6 +153,7 @@ class BTNProvider(generic.TorrentProvider):
|
||||||
# Note that this these are individual requests and might time out individually.
|
# Note that this these are individual requests and might time out individually.
|
||||||
# This would result in 'gaps' in the results. There is no way to fix this though.
|
# This would result in 'gaps' in the results. There is no way to fix this though.
|
||||||
if 'torrents' in data_json:
|
if 'torrents' in data_json:
|
||||||
|
self.tmr_limit_count = 0
|
||||||
found_torrents.update(data_json['torrents'])
|
found_torrents.update(data_json['torrents'])
|
||||||
|
|
||||||
cnt = len(results)
|
cnt = len(results)
|
||||||
|
|
@ -176,7 +180,8 @@ class BTNProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
if self.username and self.password:
|
if self.username and self.password:
|
||||||
return super(BTNProvider, self)._authorised(
|
return super(BTNProvider, self)._authorised(
|
||||||
post_params={'login': 'Log In!'}, logged_in=(lambda y='': 'casThe' in y[0:4096]))
|
post_params={'login': 'Log In!'},
|
||||||
|
logged_in=(lambda y='': 'casThe' in y[0:512] and '<title>Index' in y[0:512]))
|
||||||
raise AuthException('Password or Username for %s is empty in config provider options' % self.name)
|
raise AuthException('Password or Username for %s is empty in config provider options' % self.name)
|
||||||
|
|
||||||
def html(self, mode, search_string, results):
|
def html(self, mode, search_string, results):
|
||||||
|
|
@ -197,7 +202,10 @@ class BTNProvider(generic.TorrentProvider):
|
||||||
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
|
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
|
||||||
search_url = self.urls['search'] % (search_string, self._categories_string(mode, 'filter_cat[%s]=1'))
|
search_url = self.urls['search'] % (search_string, self._categories_string(mode, 'filter_cat[%s]=1'))
|
||||||
|
|
||||||
html = helpers.getURL(search_url, session=self.session)
|
html = self.get_url(search_url, use_tmr_limit=False)
|
||||||
|
if self.should_skip(log_warning=False, use_tmr_limit=False):
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(results)
|
cnt = len(results)
|
||||||
try:
|
try:
|
||||||
if not html or self._has_no_results(html):
|
if not html or self._has_no_results(html):
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,7 @@ class BTSceneProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
url = self.url
|
url = self.url
|
||||||
response = self.get_url(url)
|
response = self.get_url(url)
|
||||||
if not response:
|
if self.should_skip():
|
||||||
return results
|
return results
|
||||||
|
|
||||||
form = re.findall('(?is)(<form[^>]+)', response)
|
form = re.findall('(?is)(<form[^>]+)', response)
|
||||||
|
|
@ -84,6 +84,8 @@ class BTSceneProvider(generic.TorrentProvider):
|
||||||
else url + self.urls['search'] % (urllib.quote_plus(search_string))
|
else url + self.urls['search'] % (urllib.quote_plus(search_string))
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -65,6 +65,8 @@ class DHProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
html = self.get_url(self.urls['search'] % (
|
html = self.get_url(self.urls['search'] % (
|
||||||
'+'.join(search_string.split()), self._categories_string(mode), ('3', '0')[not self.freeleech]))
|
'+'.join(search_string.split()), self._categories_string(mode), ('3', '0')[not self.freeleech]))
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -62,6 +62,8 @@ class ETTVProvider(generic.TorrentProvider):
|
||||||
self._categories_string(mode), ('%2B ', '')['Cache' == mode] + '.'.join(search_string.split()))
|
self._categories_string(mode), ('%2B ', '')['Cache' == mode] + '.'.join(search_string.split()))
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
@ -110,6 +112,9 @@ class ETTVProvider(generic.TorrentProvider):
|
||||||
def get_data(self, url):
|
def get_data(self, url):
|
||||||
result = None
|
result = None
|
||||||
html = self.get_url(url, timeout=90)
|
html = self.get_url(url, timeout=90)
|
||||||
|
if self.should_skip():
|
||||||
|
return result
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = re.findall('(?i)"(magnet:[^"]+?)">', html)[0]
|
result = re.findall('(?i)"(magnet:[^"]+?)">', html)[0]
|
||||||
except IndexError:
|
except IndexError:
|
||||||
|
|
|
||||||
|
|
@ -83,6 +83,8 @@ class FanoProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (search_string, self._categories_string(mode))
|
search_url = self.urls['search'] % (search_string, self._categories_string(mode))
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -62,6 +62,8 @@ class FLProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
html = self.get_url(self.urls['search'] % ('+'.join(search_string.split()),
|
html = self.get_url(self.urls['search'] % ('+'.join(search_string.split()),
|
||||||
self._categories_string(mode, template='cats[]=%s')))
|
self._categories_string(mode, template='cats[]=%s')))
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,8 @@ class FunFileProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (self._categories_string(mode), search_string)
|
search_url = self.urls['search'] % (self._categories_string(mode), search_string)
|
||||||
|
|
||||||
html = self.get_url(search_url, timeout=self.url_timeout)
|
html = self.get_url(search_url, timeout=self.url_timeout)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -27,6 +27,7 @@ import re
|
||||||
import time
|
import time
|
||||||
import urlparse
|
import urlparse
|
||||||
import threading
|
import threading
|
||||||
|
import socket
|
||||||
from urllib import quote_plus
|
from urllib import quote_plus
|
||||||
import zlib
|
import zlib
|
||||||
from base64 import b16encode, b32decode
|
from base64 import b16encode, b32decode
|
||||||
|
|
@ -45,13 +46,157 @@ from sickbeard.exceptions import SickBeardException, AuthException, ex
|
||||||
from sickbeard.helpers import maybe_plural, remove_file_failed
|
from sickbeard.helpers import maybe_plural, remove_file_failed
|
||||||
from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
|
from sickbeard.name_parser.parser import NameParser, InvalidNameException, InvalidShowException
|
||||||
from sickbeard.show_name_helpers import get_show_names_all_possible
|
from sickbeard.show_name_helpers import get_show_names_all_possible
|
||||||
|
from sickbeard.sbdatetime import sbdatetime
|
||||||
|
|
||||||
|
|
||||||
class HaltParseException(SickBeardException):
|
class HaltParseException(SickBeardException):
|
||||||
"""Something requires the current processing to abort"""
|
"""Something requires the current processing to abort"""
|
||||||
|
|
||||||
|
|
||||||
class GenericProvider:
|
class ProviderFailTypes:
|
||||||
|
http = 1
|
||||||
|
connection = 2
|
||||||
|
connection_timeout = 3
|
||||||
|
timeout = 4
|
||||||
|
other = 5
|
||||||
|
limit = 6
|
||||||
|
nodata = 7
|
||||||
|
|
||||||
|
names = {http: 'http', timeout: 'timeout',
|
||||||
|
connection: 'connection', connection_timeout: 'connection_timeout',
|
||||||
|
nodata: 'nodata', other: 'other', limit: 'limit'}
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ProviderFail(object):
|
||||||
|
def __init__(self, fail_type=ProviderFailTypes.other, code=None, fail_time=None):
|
||||||
|
self.code = code
|
||||||
|
self.fail_type = fail_type
|
||||||
|
self.fail_time = (datetime.datetime.now(), fail_time)[isinstance(fail_time, datetime.datetime)]
|
||||||
|
|
||||||
|
|
||||||
|
class ProviderFailList(object):
|
||||||
|
def __init__(self, provider_name):
|
||||||
|
self.provider_name = provider_name
|
||||||
|
self._fails = []
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
self.clear_old()
|
||||||
|
self.load_list()
|
||||||
|
self.last_save = datetime.datetime.now()
|
||||||
|
self.dirty = False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def fails(self):
|
||||||
|
return self._fails
|
||||||
|
|
||||||
|
@property
|
||||||
|
def fails_sorted(self):
|
||||||
|
fail_dict = {}
|
||||||
|
b_d = {'count': 0}
|
||||||
|
for e in self._fails:
|
||||||
|
fail_date = e.fail_time.date()
|
||||||
|
fail_hour = e.fail_time.time().hour
|
||||||
|
date_time = datetime.datetime.combine(fail_date, datetime.time(hour=fail_hour))
|
||||||
|
if ProviderFailTypes.names[e.fail_type] not in fail_dict.get(date_time, {}):
|
||||||
|
default = {'date': str(fail_date), 'date_time': date_time, 'multirow': False}
|
||||||
|
for et in ProviderFailTypes.names.itervalues():
|
||||||
|
default[et] = b_d.copy()
|
||||||
|
fail_dict.setdefault(date_time, default)[ProviderFailTypes.names[e.fail_type]]['count'] = 1
|
||||||
|
else:
|
||||||
|
fail_dict[date_time][ProviderFailTypes.names[e.fail_type]]['count'] += 1
|
||||||
|
if ProviderFailTypes.http == e.fail_type:
|
||||||
|
if e.code in fail_dict[date_time].get(ProviderFailTypes.names[e.fail_type],
|
||||||
|
{'code': {}}).get('code', {}):
|
||||||
|
fail_dict[date_time][ProviderFailTypes.names[e.fail_type]]['code'][e.code] += 1
|
||||||
|
else:
|
||||||
|
fail_dict[date_time][ProviderFailTypes.names[e.fail_type]].setdefault('code', {})[e.code] = 1
|
||||||
|
|
||||||
|
row_count = {}
|
||||||
|
for (k, v) in fail_dict.iteritems():
|
||||||
|
row_count.setdefault(v.get('date'), 0)
|
||||||
|
if v.get('date') in row_count:
|
||||||
|
row_count[v.get('date')] += 1
|
||||||
|
for (k, v) in fail_dict.iteritems():
|
||||||
|
if 1 < row_count.get(v.get('date')):
|
||||||
|
fail_dict[k]['multirow'] = True
|
||||||
|
|
||||||
|
fail_list = sorted([fail_dict[k] for k in fail_dict.iterkeys()], key=lambda y: y.get('date_time'), reverse=True)
|
||||||
|
|
||||||
|
totals = {}
|
||||||
|
for fail_date in set([fail.get('date') for fail in fail_list]):
|
||||||
|
daytotals = {}
|
||||||
|
for et in ProviderFailTypes.names.itervalues():
|
||||||
|
daytotals.update({et: sum([x.get(et).get('count') for x in fail_list if fail_date == x.get('date')])})
|
||||||
|
totals.update({fail_date: daytotals})
|
||||||
|
for (fail_date, total) in totals.iteritems():
|
||||||
|
for i, item in enumerate(fail_list):
|
||||||
|
if fail_date == item.get('date'):
|
||||||
|
if item.get('multirow'):
|
||||||
|
fail_list[i:i] = [item.copy()]
|
||||||
|
for et in ProviderFailTypes.names.itervalues():
|
||||||
|
fail_list[i][et] = {'count': total[et]}
|
||||||
|
if et == ProviderFailTypes.names[ProviderFailTypes.http]:
|
||||||
|
fail_list[i][et]['code'] = {}
|
||||||
|
break
|
||||||
|
|
||||||
|
return fail_list
|
||||||
|
|
||||||
|
def add_fail(self, fail):
|
||||||
|
if isinstance(fail, ProviderFail):
|
||||||
|
with self.lock:
|
||||||
|
self.dirty = True
|
||||||
|
self._fails.append(fail)
|
||||||
|
logger.log('Adding fail.%s for %s' % (ProviderFailTypes.names.get(
|
||||||
|
fail.fail_type, ProviderFailTypes.names[ProviderFailTypes.other]), self.provider_name()),
|
||||||
|
logger.DEBUG)
|
||||||
|
self.save_list()
|
||||||
|
|
||||||
|
def save_list(self):
|
||||||
|
if self.dirty:
|
||||||
|
self.clear_old()
|
||||||
|
with self.lock:
|
||||||
|
my_db = db.DBConnection('cache.db')
|
||||||
|
cl = []
|
||||||
|
for f in self._fails:
|
||||||
|
cl.append(['INSERT OR IGNORE INTO provider_fails (prov_name, fail_type, fail_code, fail_time) '
|
||||||
|
'VALUES (?,?,?,?)', [self.provider_name(), f.fail_type, f.code,
|
||||||
|
sbdatetime.totimestamp(f.fail_time)]])
|
||||||
|
self.dirty = False
|
||||||
|
if cl:
|
||||||
|
my_db.mass_action(cl)
|
||||||
|
self.last_save = datetime.datetime.now()
|
||||||
|
|
||||||
|
def load_list(self):
|
||||||
|
with self.lock:
|
||||||
|
try:
|
||||||
|
my_db = db.DBConnection('cache.db')
|
||||||
|
if my_db.hasTable('provider_fails'):
|
||||||
|
results = my_db.select('SELECT * FROM provider_fails WHERE prov_name = ?', [self.provider_name()])
|
||||||
|
self._fails = []
|
||||||
|
for r in results:
|
||||||
|
try:
|
||||||
|
self._fails.append(ProviderFail(
|
||||||
|
fail_type=helpers.tryInt(r['fail_type']), code=helpers.tryInt(r['fail_code']),
|
||||||
|
fail_time=datetime.datetime.fromtimestamp(helpers.tryInt(r['fail_time']))))
|
||||||
|
except (StandardError, Exception):
|
||||||
|
continue
|
||||||
|
except (StandardError, Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def clear_old(self):
|
||||||
|
with self.lock:
|
||||||
|
try:
|
||||||
|
my_db = db.DBConnection('cache.db')
|
||||||
|
if my_db.hasTable('provider_fails'):
|
||||||
|
time_limit = sbdatetime.totimestamp(datetime.datetime.now() - datetime.timedelta(days=28))
|
||||||
|
my_db.action('DELETE FROM provider_fails WHERE fail_time < ?', [time_limit])
|
||||||
|
except (StandardError, Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class GenericProvider(object):
|
||||||
NZB = 'nzb'
|
NZB = 'nzb'
|
||||||
TORRENT = 'torrent'
|
TORRENT = 'torrent'
|
||||||
|
|
||||||
|
|
@ -86,6 +231,321 @@ class GenericProvider:
|
||||||
# 'Chrome/32.0.1700.107 Safari/537.36'}
|
# 'Chrome/32.0.1700.107 Safari/537.36'}
|
||||||
'User-Agent': USER_AGENT}
|
'User-Agent': USER_AGENT}
|
||||||
|
|
||||||
|
self._failure_count = 0
|
||||||
|
self._failure_time = None
|
||||||
|
self.fails = ProviderFailList(self.get_id)
|
||||||
|
self._tmr_limit_count = 0
|
||||||
|
self._tmr_limit_time = None
|
||||||
|
self._tmr_limit_wait = None
|
||||||
|
self._last_fail_type = None
|
||||||
|
self.has_limit = False
|
||||||
|
self.fail_times = {1: (0, 15), 2: (0, 30), 3: (1, 0), 4: (2, 0), 5: (3, 0), 6: (6, 0), 7: (12, 0), 8: (24, 0)}
|
||||||
|
self._load_fail_values()
|
||||||
|
|
||||||
|
def _load_fail_values(self):
|
||||||
|
if hasattr(sickbeard, 'DATA_DIR'):
|
||||||
|
my_db = db.DBConnection('cache.db')
|
||||||
|
if my_db.hasTable('provider_fails_count'):
|
||||||
|
r = my_db.select('SELECT * FROM provider_fails_count WHERE prov_name = ?', [self.get_id()])
|
||||||
|
if r:
|
||||||
|
self._failure_count = helpers.tryInt(r[0]['failure_count'], 0)
|
||||||
|
if r[0]['failure_time']:
|
||||||
|
self._failure_time = datetime.datetime.fromtimestamp(r[0]['failure_time'])
|
||||||
|
else:
|
||||||
|
self._failure_time = None
|
||||||
|
self._tmr_limit_count = helpers.tryInt(r[0]['tmr_limit_count'], 0)
|
||||||
|
if r[0]['tmr_limit_time']:
|
||||||
|
self._tmr_limit_time = datetime.datetime.fromtimestamp(r[0]['tmr_limit_time'])
|
||||||
|
else:
|
||||||
|
self._tmr_limit_time = None
|
||||||
|
if r[0]['tmr_limit_wait']:
|
||||||
|
self._tmr_limit_wait = datetime.timedelta(seconds=helpers.tryInt(r[0]['tmr_limit_wait'], 0))
|
||||||
|
else:
|
||||||
|
self._tmr_limit_wait = None
|
||||||
|
self._last_fail_type = self.last_fail
|
||||||
|
|
||||||
|
def _save_fail_value(self, field, value):
|
||||||
|
my_db = db.DBConnection('cache.db')
|
||||||
|
if my_db.hasTable('provider_fails_count'):
|
||||||
|
r = my_db.action('UPDATE provider_fails_count SET %s = ? WHERE prov_name = ?' % field,
|
||||||
|
[value, self.get_id()])
|
||||||
|
if 0 == r.rowcount:
|
||||||
|
my_db.action('REPLACE INTO provider_fails_count (prov_name, %s) VALUES (?,?)' % field,
|
||||||
|
[self.get_id(), value])
|
||||||
|
|
||||||
|
@property
|
||||||
|
def last_fail(self):
|
||||||
|
try:
|
||||||
|
return sorted(self.fails.fails, key=lambda x: x.fail_time, reverse=True)[0].fail_type
|
||||||
|
except (StandardError, Exception):
|
||||||
|
return None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def failure_count(self):
|
||||||
|
return self._failure_count
|
||||||
|
|
||||||
|
@failure_count.setter
|
||||||
|
def failure_count(self, value):
|
||||||
|
changed_val = self._failure_count != value
|
||||||
|
self._failure_count = value
|
||||||
|
if changed_val:
|
||||||
|
self._save_fail_value('failure_count', value)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def failure_time(self):
|
||||||
|
return self._failure_time
|
||||||
|
|
||||||
|
@failure_time.setter
|
||||||
|
def failure_time(self, value):
|
||||||
|
if None is value or isinstance(value, datetime.datetime):
|
||||||
|
changed_val = self._failure_time != value
|
||||||
|
self._failure_time = value
|
||||||
|
if changed_val:
|
||||||
|
self._save_fail_value('failure_time', (sbdatetime.totimestamp(value), value)[None is value])
|
||||||
|
|
||||||
|
@property
|
||||||
|
def tmr_limit_count(self):
|
||||||
|
return self._tmr_limit_count
|
||||||
|
|
||||||
|
@tmr_limit_count.setter
|
||||||
|
def tmr_limit_count(self, value):
|
||||||
|
changed_val = self._tmr_limit_count != value
|
||||||
|
self._tmr_limit_count = value
|
||||||
|
if changed_val:
|
||||||
|
self._save_fail_value('tmr_limit_count', value)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def tmr_limit_time(self):
|
||||||
|
return self._tmr_limit_time
|
||||||
|
|
||||||
|
@tmr_limit_time.setter
|
||||||
|
def tmr_limit_time(self, value):
|
||||||
|
if None is value or isinstance(value, datetime.datetime):
|
||||||
|
changed_val = self._tmr_limit_time != value
|
||||||
|
self._tmr_limit_time = value
|
||||||
|
if changed_val:
|
||||||
|
self._save_fail_value('tmr_limit_time', (sbdatetime.totimestamp(value), value)[None is value])
|
||||||
|
|
||||||
|
@property
|
||||||
|
def max_index(self):
|
||||||
|
return len(self.fail_times)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def tmr_limit_wait(self):
|
||||||
|
return self._tmr_limit_wait
|
||||||
|
|
||||||
|
@tmr_limit_wait.setter
|
||||||
|
def tmr_limit_wait(self, value):
|
||||||
|
if isinstance(getattr(self, 'fails', None), ProviderFailList) and isinstance(value, datetime.timedelta):
|
||||||
|
self.fails.add_fail(ProviderFail(fail_type=ProviderFailTypes.limit))
|
||||||
|
changed_val = self._tmr_limit_wait != value
|
||||||
|
self._tmr_limit_wait = value
|
||||||
|
if changed_val:
|
||||||
|
if None is value:
|
||||||
|
self._save_fail_value('tmr_limit_wait', value)
|
||||||
|
elif isinstance(value, datetime.timedelta):
|
||||||
|
self._save_fail_value('tmr_limit_wait', value.total_seconds())
|
||||||
|
|
||||||
|
def fail_time_index(self, base_limit=2):
|
||||||
|
i = self.failure_count - base_limit
|
||||||
|
return (i, self.max_index)[i >= self.max_index]
|
||||||
|
|
||||||
|
def tmr_limit_update(self, period, unit, desc):
|
||||||
|
self.tmr_limit_time = datetime.datetime.now()
|
||||||
|
self.tmr_limit_count += 1
|
||||||
|
limit_set = False
|
||||||
|
if None not in (period, unit):
|
||||||
|
limit_set = True
|
||||||
|
if unit in ('s', 'sec', 'secs', 'seconds', 'second'):
|
||||||
|
self.tmr_limit_wait = datetime.timedelta(seconds=helpers.tryInt(period))
|
||||||
|
elif unit in ('m', 'min', 'mins', 'minutes', 'minute'):
|
||||||
|
self.tmr_limit_wait = datetime.timedelta(minutes=helpers.tryInt(period))
|
||||||
|
elif unit in ('h', 'hr', 'hrs', 'hours', 'hour'):
|
||||||
|
self.tmr_limit_wait = datetime.timedelta(hours=helpers.tryInt(period))
|
||||||
|
elif unit in ('d', 'days', 'day'):
|
||||||
|
self.tmr_limit_wait = datetime.timedelta(days=helpers.tryInt(period))
|
||||||
|
else:
|
||||||
|
limit_set = False
|
||||||
|
if not limit_set:
|
||||||
|
time_index = self.fail_time_index(base_limit=0)
|
||||||
|
self.tmr_limit_wait = self.wait_time(time_index)
|
||||||
|
logger.log('Request limit reached. Waiting for %s until next retry. Message: %s' %
|
||||||
|
(self.tmr_limit_wait, desc or 'none found'), logger.WARNING)
|
||||||
|
|
||||||
|
def wait_time(self, time_index=None):
|
||||||
|
"""
|
||||||
|
Return a suitable wait time, selected by parameter, or based on the current failure count
|
||||||
|
|
||||||
|
:param time_index: A key value index into the fail_times dict, or selects using failure count if None
|
||||||
|
:type time_index: Integer
|
||||||
|
:return: Time
|
||||||
|
:rtype: Timedelta
|
||||||
|
"""
|
||||||
|
if None is time_index:
|
||||||
|
time_index = self.fail_time_index()
|
||||||
|
return datetime.timedelta(hours=self.fail_times[time_index][0], minutes=self.fail_times[time_index][1])
|
||||||
|
|
||||||
|
def fail_newest_delta(self):
|
||||||
|
"""
|
||||||
|
Return how long since most recent failure
|
||||||
|
:return: Period since most recent failure on record
|
||||||
|
:rtype: timedelta
|
||||||
|
"""
|
||||||
|
return datetime.datetime.now() - self.failure_time
|
||||||
|
|
||||||
|
def is_waiting(self):
|
||||||
|
return self.fail_newest_delta() < self.wait_time()
|
||||||
|
|
||||||
|
def valid_tmr_time(self):
|
||||||
|
return isinstance(self.tmr_limit_wait, datetime.timedelta) and \
|
||||||
|
isinstance(self.tmr_limit_time, datetime.datetime)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def get_next_try_time(self):
|
||||||
|
n = None
|
||||||
|
h = datetime.timedelta(seconds=0)
|
||||||
|
f = datetime.timedelta(seconds=0)
|
||||||
|
if self.valid_tmr_time():
|
||||||
|
h = self.tmr_limit_time + self.tmr_limit_wait - datetime.datetime.now()
|
||||||
|
if 3 <= self.failure_count and isinstance(self.failure_time, datetime.datetime) and self.is_waiting():
|
||||||
|
h = self.failure_time + self.wait_time() - datetime.datetime.now()
|
||||||
|
if datetime.timedelta(seconds=0) < max((h, f)):
|
||||||
|
n = max((h, f))
|
||||||
|
return n
|
||||||
|
|
||||||
|
def retry_next(self):
|
||||||
|
if self.valid_tmr_time():
|
||||||
|
self.tmr_limit_time = datetime.datetime.now() - self.tmr_limit_wait
|
||||||
|
if 3 <= self.failure_count and isinstance(self.failure_time, datetime.datetime) and self.is_waiting():
|
||||||
|
self.failure_time = datetime.datetime.now() - self.wait_time()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def fmt_delta(delta):
|
||||||
|
return str(delta).rsplit('.')[0]
|
||||||
|
|
||||||
|
def should_skip(self, log_warning=True, use_tmr_limit=True):
|
||||||
|
"""
|
||||||
|
Determine if a subsequent server request should be skipped. The result of this logic is based on most recent
|
||||||
|
server connection activity including, exhausted request limits, and counting connect failures to determine a
|
||||||
|
"cool down" period before recommending reconnection attempts; by returning False.
|
||||||
|
:param log_warning: Output to log if True (default) otherwise set False for no output.
|
||||||
|
:type log_warning: Boolean
|
||||||
|
:param use_tmr_limit: Setting this to False will ignore a tmr limit being reached and will instead return False.
|
||||||
|
:type use_tmr_limit: Boolean
|
||||||
|
:return: True for any known issue that would prevent a subsequent server connection, otherwise False.
|
||||||
|
:rtype: Boolean
|
||||||
|
"""
|
||||||
|
if self.valid_tmr_time():
|
||||||
|
time_left = self.tmr_limit_time + self.tmr_limit_wait - datetime.datetime.now()
|
||||||
|
if time_left > datetime.timedelta(seconds=0):
|
||||||
|
if log_warning:
|
||||||
|
# Ensure provider name output (e.g. when displaying config/provs) instead of e.g. thread "Tornado"
|
||||||
|
prepend = ('[%s] :: ' % self.name, '')[any([x.name in threading.currentThread().getName()
|
||||||
|
for x in sickbeard.providers.sortedProviderList()])]
|
||||||
|
logger.log('%sToo many requests reached at %s, waiting for %s' % (
|
||||||
|
prepend, self.fmt_delta(self.tmr_limit_time), self.fmt_delta(time_left)), logger.WARNING)
|
||||||
|
return use_tmr_limit
|
||||||
|
else:
|
||||||
|
self.tmr_limit_time = None
|
||||||
|
self.tmr_limit_wait = None
|
||||||
|
if 3 <= self.failure_count:
|
||||||
|
if None is self.failure_time:
|
||||||
|
self.failure_time = datetime.datetime.now()
|
||||||
|
if self.is_waiting():
|
||||||
|
if log_warning:
|
||||||
|
time_left = self.wait_time() - self.fail_newest_delta()
|
||||||
|
logger.log('Failed %s times, skipping provider for %s, last failure at %s with fail type: %s' % (
|
||||||
|
self.failure_count, self.fmt_delta(time_left), self.fmt_delta(self.failure_time),
|
||||||
|
ProviderFailTypes.names.get(
|
||||||
|
self.last_fail, ProviderFailTypes.names[ProviderFailTypes.other])), logger.WARNING)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def inc_failure_count(self, *args, **kwargs):
|
||||||
|
fail_type = ('fail_type' in kwargs and kwargs['fail_type'].fail_type) or \
|
||||||
|
(isinstance(args, tuple) and isinstance(args[0], ProviderFail) and args[0].fail_type)
|
||||||
|
if not isinstance(self.failure_time, datetime.datetime) or \
|
||||||
|
fail_type != self._last_fail_type or \
|
||||||
|
self.fail_newest_delta() > datetime.timedelta(seconds=3):
|
||||||
|
self.failure_count += 1
|
||||||
|
self.failure_time = datetime.datetime.now()
|
||||||
|
self._last_fail_type = fail_type
|
||||||
|
self.fails.add_fail(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
logger.log('%s: Not logging same failure within 3 seconds' % self.name, logger.DEBUG)
|
||||||
|
|
||||||
|
def get_url(self, url, skip_auth=False, use_tmr_limit=True, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Return data from a URI with a possible check for authentication prior to the data fetch.
|
||||||
|
Raised errors and no data in responses are tracked for making future logic decisions.
|
||||||
|
|
||||||
|
:param url: Address where to fetch data from
|
||||||
|
:type url: String
|
||||||
|
:param skip_auth: Skip authentication check of provider if True
|
||||||
|
:type skip_auth: Boolean
|
||||||
|
:param use_tmr_limit: An API limit can be +ve before a fetch, but unwanted, set False to short should_skip
|
||||||
|
:type use_tmr_limit: Boolean
|
||||||
|
:param args: params to pass-through to getURL
|
||||||
|
:type args:
|
||||||
|
:param kwargs: keyword params to pass-through to getURL
|
||||||
|
:type kwargs:
|
||||||
|
:return: None or data fetched from URL
|
||||||
|
:rtype: String or Nonetype
|
||||||
|
"""
|
||||||
|
data = None
|
||||||
|
|
||||||
|
# check for auth
|
||||||
|
if (not skip_auth and not (self.is_public_access()
|
||||||
|
and type(self).__name__ not in ['TorrentRssProvider']) and not self._authorised()) \
|
||||||
|
or self.should_skip(use_tmr_limit=use_tmr_limit):
|
||||||
|
return
|
||||||
|
|
||||||
|
kwargs['raise_exceptions'] = True
|
||||||
|
kwargs['raise_status_code'] = True
|
||||||
|
for k, v in dict(headers=self.headers, hooks=dict(response=self.cb_response), session=self.session).items():
|
||||||
|
kwargs.setdefault(k, v)
|
||||||
|
|
||||||
|
post_data = kwargs.get('post_data')
|
||||||
|
post_json = kwargs.get('post_json')
|
||||||
|
|
||||||
|
# noinspection PyUnusedLocal
|
||||||
|
log_failure_url = False
|
||||||
|
try:
|
||||||
|
data = helpers.getURL(url, *args, **kwargs)
|
||||||
|
if data:
|
||||||
|
if 0 != self.failure_count:
|
||||||
|
logger.log('Unblocking provider: %s' % self.get_id(), logger.DEBUG)
|
||||||
|
self.failure_count = 0
|
||||||
|
self.failure_time = None
|
||||||
|
else:
|
||||||
|
self.inc_failure_count(ProviderFail(fail_type=ProviderFailTypes.nodata))
|
||||||
|
log_failure_url = True
|
||||||
|
except requests.exceptions.HTTPError as e:
|
||||||
|
self.inc_failure_count(ProviderFail(fail_type=ProviderFailTypes.http, code=e.response.status_code))
|
||||||
|
except requests.exceptions.ConnectionError:
|
||||||
|
self.inc_failure_count(ProviderFail(fail_type=ProviderFailTypes.connection))
|
||||||
|
except requests.exceptions.ReadTimeout:
|
||||||
|
self.inc_failure_count(ProviderFail(fail_type=ProviderFailTypes.timeout))
|
||||||
|
except (requests.exceptions.Timeout, socket.timeout):
|
||||||
|
self.inc_failure_count(ProviderFail(fail_type=ProviderFailTypes.connection_timeout))
|
||||||
|
except (StandardError, Exception) as e:
|
||||||
|
log_failure_url = True
|
||||||
|
self.inc_failure_count(ProviderFail(fail_type=ProviderFailTypes.other))
|
||||||
|
|
||||||
|
self.fails.save_list()
|
||||||
|
if log_failure_url:
|
||||||
|
self.log_failure_url(url, post_data, post_json)
|
||||||
|
return data
|
||||||
|
|
||||||
|
def log_failure_url(self, url, post_data=None, post_json=None):
|
||||||
|
if self.should_skip(log_warning=False):
|
||||||
|
post = []
|
||||||
|
if post_data:
|
||||||
|
post += [' .. Post params: [%s]' % '&'.join([post_data])]
|
||||||
|
if post_json:
|
||||||
|
post += [' .. Json params: [%s]' % '&'.join([post_json])]
|
||||||
|
logger.log('Failure URL: %s%s' % (url, ''.join(post)), logger.WARNING)
|
||||||
|
|
||||||
def get_id(self):
|
def get_id(self):
|
||||||
return GenericProvider.make_id(self.name)
|
return GenericProvider.make_id(self.name)
|
||||||
|
|
||||||
|
|
@ -152,19 +612,6 @@ class GenericProvider:
|
||||||
self.session.response = dict(url=r.url, status_code=r.status_code, elapsed=r.elapsed, from_cache=r.from_cache)
|
self.session.response = dict(url=r.url, status_code=r.status_code, elapsed=r.elapsed, from_cache=r.from_cache)
|
||||||
return r
|
return r
|
||||||
|
|
||||||
def get_url(self, url, post_data=None, params=None, timeout=30, json=False):
|
|
||||||
"""
|
|
||||||
By default this is just a simple urlopen call but this method should be overridden
|
|
||||||
for providers with special URL requirements (like cookies)
|
|
||||||
"""
|
|
||||||
|
|
||||||
# check for auth
|
|
||||||
if not self._authorised():
|
|
||||||
return
|
|
||||||
|
|
||||||
return helpers.getURL(url, post_data=post_data, params=params, headers=self.headers, timeout=timeout,
|
|
||||||
session=self.session, json=json, hooks=dict(response=self.cb_response))
|
|
||||||
|
|
||||||
def download_result(self, result):
|
def download_result(self, result):
|
||||||
"""
|
"""
|
||||||
Save the result to disk.
|
Save the result to disk.
|
||||||
|
|
@ -428,9 +875,13 @@ class GenericProvider:
|
||||||
|
|
||||||
results = {}
|
results = {}
|
||||||
item_list = []
|
item_list = []
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
searched_scene_season = None
|
searched_scene_season = None
|
||||||
for ep_obj in episodes:
|
for ep_obj in episodes:
|
||||||
|
if self.should_skip(log_warning=False):
|
||||||
|
break
|
||||||
# search cache for episode result
|
# search cache for episode result
|
||||||
cache_result = self.cache.searchCache(ep_obj, manual_search)
|
cache_result = self.cache.searchCache(ep_obj, manual_search)
|
||||||
if cache_result:
|
if cache_result:
|
||||||
|
|
@ -457,6 +908,8 @@ class GenericProvider:
|
||||||
|
|
||||||
for cur_param in search_params:
|
for cur_param in search_params:
|
||||||
item_list += self._search_provider(cur_param, search_mode=search_mode, epcount=len(episodes))
|
item_list += self._search_provider(cur_param, search_mode=search_mode, epcount=len(episodes))
|
||||||
|
if self.should_skip():
|
||||||
|
break
|
||||||
|
|
||||||
return self.finish_find_search_results(show, episodes, search_mode, manual_search, results, item_list)
|
return self.finish_find_search_results(show, episodes, search_mode, manual_search, results, item_list)
|
||||||
|
|
||||||
|
|
@ -649,10 +1102,11 @@ class GenericProvider:
|
||||||
:param count: count of successfully processed items
|
:param count: count of successfully processed items
|
||||||
:param url: source url of item(s)
|
:param url: source url of item(s)
|
||||||
"""
|
"""
|
||||||
str1, thing, str3 = (('', '%s item' % mode.lower(), ''), (' usable', 'proper', ' found'))['Propers' == mode]
|
if not self.should_skip():
|
||||||
logger.log(u'%s %s in response from %s' % (('No' + str1, count)[0 < count], (
|
str1, thing, str3 = (('', '%s item' % mode.lower(), ''), (' usable', 'proper', ' found'))['Propers' == mode]
|
||||||
'%s%s%s%s' % (('', 'freeleech ')[getattr(self, 'freeleech', False)], thing, maybe_plural(count), str3)),
|
logger.log(u'%s %s in response from %s' % (('No' + str1, count)[0 < count], (
|
||||||
re.sub('(\s)\s+', r'\1', url)))
|
'%s%s%s%s' % (('', 'freeleech ')[getattr(self, 'freeleech', False)], thing, maybe_plural(count), str3)),
|
||||||
|
re.sub('(\s)\s+', r'\1', url)))
|
||||||
|
|
||||||
def check_auth_cookie(self):
|
def check_auth_cookie(self):
|
||||||
|
|
||||||
|
|
@ -723,12 +1177,13 @@ class GenericProvider:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
class NZBProvider(object, GenericProvider):
|
class NZBProvider(GenericProvider):
|
||||||
|
|
||||||
def __init__(self, name, supports_backlog=True, anime_only=False):
|
def __init__(self, name, supports_backlog=True, anime_only=False):
|
||||||
GenericProvider.__init__(self, name, supports_backlog, anime_only)
|
GenericProvider.__init__(self, name, supports_backlog, anime_only)
|
||||||
|
|
||||||
self.providerType = GenericProvider.NZB
|
self.providerType = GenericProvider.NZB
|
||||||
|
self.has_limit = True
|
||||||
|
|
||||||
def image_name(self):
|
def image_name(self):
|
||||||
|
|
||||||
|
|
@ -757,6 +1212,9 @@ class NZBProvider(object, GenericProvider):
|
||||||
results = [classes.Proper(x['name'], x['url'], datetime.datetime.fromtimestamp(x['time']), self.show) for x in
|
results = [classes.Proper(x['name'], x['url'], datetime.datetime.fromtimestamp(x['time']), self.show) for x in
|
||||||
cache_results]
|
cache_results]
|
||||||
|
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
index = 0
|
index = 0
|
||||||
alt_search = ('nzbs_org' == self.get_id())
|
alt_search = ('nzbs_org' == self.get_id())
|
||||||
do_search_alt = False
|
do_search_alt = False
|
||||||
|
|
@ -775,6 +1233,9 @@ class NZBProvider(object, GenericProvider):
|
||||||
|
|
||||||
urls = []
|
urls = []
|
||||||
while index < len(search_terms):
|
while index < len(search_terms):
|
||||||
|
if self.should_skip(log_warning=False):
|
||||||
|
break
|
||||||
|
|
||||||
search_params = {'q': search_terms[index], 'maxage': sickbeard.BACKLOG_DAYS + 2}
|
search_params = {'q': search_terms[index], 'maxage': sickbeard.BACKLOG_DAYS + 2}
|
||||||
if alt_search:
|
if alt_search:
|
||||||
|
|
||||||
|
|
@ -817,7 +1278,7 @@ class NZBProvider(object, GenericProvider):
|
||||||
return self._search_provider(search_params=search_params, **kwargs)
|
return self._search_provider(search_params=search_params, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
class TorrentProvider(object, GenericProvider):
|
class TorrentProvider(GenericProvider):
|
||||||
|
|
||||||
def __init__(self, name, supports_backlog=True, anime_only=False, cache_update_freq=None, update_freq=None):
|
def __init__(self, name, supports_backlog=True, anime_only=False, cache_update_freq=None, update_freq=None):
|
||||||
GenericProvider.__init__(self, name, supports_backlog, anime_only)
|
GenericProvider.__init__(self, name, supports_backlog, anime_only)
|
||||||
|
|
@ -995,8 +1456,9 @@ class TorrentProvider(object, GenericProvider):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if 10 < len(cur_url) and ((expire and (expire > int(time.time()))) or
|
if 10 < len(cur_url) and ((expire and (expire > int(time.time()))) or
|
||||||
self._has_signature(helpers.getURL(cur_url, session=self.session))):
|
self._has_signature(self.get_url(cur_url, skip_auth=True))):
|
||||||
|
if self.should_skip():
|
||||||
|
return None
|
||||||
for k, v in getattr(self, 'url_tmpl', {}).items():
|
for k, v in getattr(self, 'url_tmpl', {}).items():
|
||||||
self.urls[k] = v % {'home': cur_url, 'vars': getattr(self, 'url_vars', {}).get(k, '')}
|
self.urls[k] = v % {'home': cur_url, 'vars': getattr(self, 'url_vars', {}).get(k, '')}
|
||||||
|
|
||||||
|
|
@ -1056,15 +1518,17 @@ class TorrentProvider(object, GenericProvider):
|
||||||
|
|
||||||
if isinstance(url, type([])):
|
if isinstance(url, type([])):
|
||||||
for i in range(0, len(url)):
|
for i in range(0, len(url)):
|
||||||
helpers.getURL(url.pop(), session=self.session)
|
self.get_url(url.pop(), skip_auth=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return False
|
||||||
|
|
||||||
passfield, userfield = None, None
|
passfield, userfield = None, None
|
||||||
if not url:
|
if not url:
|
||||||
if hasattr(self, 'urls'):
|
if hasattr(self, 'urls'):
|
||||||
url = self.urls.get('login_action')
|
url = self.urls.get('login_action')
|
||||||
if url:
|
if url:
|
||||||
response = helpers.getURL(url, session=self.session)
|
response = self.get_url(url, skip_auth=True)
|
||||||
if None is response:
|
if self.should_skip() or None is response:
|
||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
post_params = isinstance(post_params, type({})) and post_params or {}
|
post_params = isinstance(post_params, type({})) and post_params or {}
|
||||||
|
|
@ -1104,8 +1568,8 @@ class TorrentProvider(object, GenericProvider):
|
||||||
if self.password not in post_params.values():
|
if self.password not in post_params.values():
|
||||||
post_params[(passfield, 'password')[not passfield]] = self.password
|
post_params[(passfield, 'password')[not passfield]] = self.password
|
||||||
|
|
||||||
response = helpers.getURL(url, post_data=post_params, session=self.session, timeout=timeout)
|
response = self.get_url(url, skip_auth=True, post_data=post_params, timeout=timeout)
|
||||||
if response:
|
if not self.should_skip() and response:
|
||||||
if logged_in(response):
|
if logged_in(response):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
@ -1153,6 +1617,8 @@ class TorrentProvider(object, GenericProvider):
|
||||||
:return: list of Proper objects
|
:return: list of Proper objects
|
||||||
"""
|
"""
|
||||||
results = []
|
results = []
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
search_terms = getattr(self, 'proper_search_terms', ['proper', 'repack', 'real'])
|
search_terms = getattr(self, 'proper_search_terms', ['proper', 'repack', 'real'])
|
||||||
if not isinstance(search_terms, list):
|
if not isinstance(search_terms, list):
|
||||||
|
|
@ -1164,9 +1630,14 @@ class TorrentProvider(object, GenericProvider):
|
||||||
|
|
||||||
clean_term = re.compile(r'(?i)[^a-z1-9|.]+')
|
clean_term = re.compile(r'(?i)[^a-z1-9|.]+')
|
||||||
for proper_term in search_terms:
|
for proper_term in search_terms:
|
||||||
|
if self.should_skip(log_warning=False):
|
||||||
|
break
|
||||||
|
|
||||||
proper_check = re.compile(r'(?i)(?:%s)' % clean_term.sub('', proper_term))
|
proper_check = re.compile(r'(?i)(?:%s)' % clean_term.sub('', proper_term))
|
||||||
for item in items:
|
for item in items:
|
||||||
|
if self.should_skip(log_warning=False):
|
||||||
|
break
|
||||||
|
|
||||||
title, url = self._title_and_url(item)
|
title, url = self._title_and_url(item)
|
||||||
if proper_check.search(title):
|
if proper_check.search(title):
|
||||||
results.append(classes.Proper(title, url, datetime.datetime.today(),
|
results.append(classes.Proper(title, url, datetime.datetime.today(),
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,8 @@ class GFTrackerProvider(generic.TorrentProvider):
|
||||||
(self.urls['search'] % search_string, '')['Cache' == mode])
|
(self.urls['search'] % search_string, '')['Cache' == mode])
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -70,6 +70,8 @@ class GrabTheInfoProvider(generic.TorrentProvider):
|
||||||
(self.urls['search'] % search_string, '')['Cache' == mode])
|
(self.urls['search'] % search_string, '')['Cache' == mode])
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,8 @@ class HD4FreeProvider(generic.TorrentProvider):
|
||||||
self.token, '+'.join(search_string.split()), self._categories_string(mode, ''), '', '', '')
|
self.token, '+'.join(search_string.split()), self._categories_string(mode, ''), '', '', '')
|
||||||
|
|
||||||
resp = self.get_url(search_url, json=True)
|
resp = self.get_url(search_url, json=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ class HDBitsProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
self.username, self.passkey, self.freeleech, self.minseed, self.minleech = 5 * [None]
|
self.username, self.passkey, self.freeleech, self.minseed, self.minleech = 5 * [None]
|
||||||
|
|
||||||
def check_auth_from_data(self, parsed_json):
|
def _check_auth_from_data(self, parsed_json):
|
||||||
|
|
||||||
if 'status' in parsed_json and 5 == parsed_json.get('status') and 'message' in parsed_json:
|
if 'status' in parsed_json and 5 == parsed_json.get('status') and 'message' in parsed_json:
|
||||||
logger.log(u'Incorrect username or password for %s: %s' % (self.name, parsed_json['message']), logger.DEBUG)
|
logger.log(u'Incorrect username or password for %s: %s' % (self.name, parsed_json['message']), logger.DEBUG)
|
||||||
|
|
@ -112,9 +112,11 @@ class HDBitsProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search']
|
search_url = self.urls['search']
|
||||||
|
|
||||||
json_resp = self.get_url(search_url, post_data=post_data, json=True)
|
json_resp = self.get_url(search_url, post_data=post_data, json=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if not (json_resp and self.check_auth_from_data(json_resp) and 'data' in json_resp):
|
if not (json_resp and self._check_auth_from_data(json_resp) and 'data' in json_resp):
|
||||||
logger.log(u'Response from %s does not contain any json data, abort' % self.name, logger.ERROR)
|
logger.log(u'Response from %s does not contain any json data, abort' % self.name, logger.ERROR)
|
||||||
return results
|
return results
|
||||||
except AuthException as e:
|
except AuthException as e:
|
||||||
|
|
|
||||||
|
|
@ -83,6 +83,8 @@ class HDSpaceProvider(generic.TorrentProvider):
|
||||||
search_url += self.urls['search'] % rc['nodots'].sub(' ', search_string)
|
search_url += self.urls['search'] % rc['nodots'].sub(' ', search_string)
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -86,6 +86,8 @@ class HDTorrentsProvider(generic.TorrentProvider):
|
||||||
self._categories_string(mode, template='category[]=%s')
|
self._categories_string(mode, template='category[]=%s')
|
||||||
.replace('&category[]=Animation', ('&genre[]=Animation', '')[mode in ['Cache', 'Propers']]))
|
.replace('&category[]=Animation', ('&genre[]=Animation', '')[mode in ['Cache', 'Propers']]))
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -88,6 +88,8 @@ class IPTorrentsProvider(generic.TorrentProvider):
|
||||||
(';free', '')[not self.freeleech], (';o=seeders', '')['Cache' == mode])
|
(';free', '')[not self.freeleech], (';o=seeders', '')['Cache' == mode])
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -67,6 +67,8 @@ class LimeTorrentsProvider(generic.TorrentProvider):
|
||||||
else self.urls['search'] % (urllib.quote_plus(search_string))
|
else self.urls['search'] % (urllib.quote_plus(search_string))
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -54,6 +54,8 @@ class MagnetDLProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % re.sub('[.\s]+', ' ', search_string)
|
search_url = self.urls['search'] % re.sub('[.\s]+', ' ', search_string)
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -65,6 +65,9 @@ class MoreThanProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
# fetches 15 results by default, and up to 100 if allowed in user profile
|
# fetches 15 results by default, and up to 100 if allowed in user profile
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
if not html or self._has_no_results(html):
|
if not html or self._has_no_results(html):
|
||||||
|
|
|
||||||
|
|
@ -68,6 +68,8 @@ class NcoreProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
# fetches 15 results by default, and up to 100 if allowed in user profile
|
# fetches 15 results by default, and up to 100 if allowed in user profile
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -51,7 +51,9 @@ class NebulanceProvider(generic.TorrentProvider):
|
||||||
post_params={'keeplogged': '1', 'form_tmpl': True}):
|
post_params={'keeplogged': '1', 'form_tmpl': True}):
|
||||||
return False
|
return False
|
||||||
if not self.user_authkey:
|
if not self.user_authkey:
|
||||||
response = helpers.getURL(self.urls['user'], session=self.session, json=True)
|
response = self.get_url(self.urls['user'], skip_auth=True, json=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return False
|
||||||
if 'response' in response:
|
if 'response' in response:
|
||||||
self.user_authkey, self.user_passkey = [response['response'].get(v) for v in 'authkey', 'passkey']
|
self.user_authkey, self.user_passkey = [response['response'].get(v) for v in 'authkey', 'passkey']
|
||||||
return self.user_authkey
|
return self.user_authkey
|
||||||
|
|
@ -74,6 +76,8 @@ class NebulanceProvider(generic.TorrentProvider):
|
||||||
search_url += self.urls['search'] % rc['nodots'].sub('+', search_string)
|
search_url += self.urls['search'] % rc['nodots'].sub('+', search_string)
|
||||||
|
|
||||||
data_json = self.get_url(search_url, json=True)
|
data_json = self.get_url(search_url, json=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,7 @@ from math import ceil
|
||||||
from sickbeard.sbdatetime import sbdatetime
|
from sickbeard.sbdatetime import sbdatetime
|
||||||
from . import generic
|
from . import generic
|
||||||
from sickbeard import helpers, logger, tvcache, classes, db
|
from sickbeard import helpers, logger, tvcache, classes, db
|
||||||
from sickbeard.common import neededQualities, Quality
|
from sickbeard.common import neededQualities, Quality, SNATCHED, SNATCHED_PROPER, SNATCHED_BEST, DOWNLOADED
|
||||||
from sickbeard.exceptions import AuthException, MultipleShowObjectsException
|
from sickbeard.exceptions import AuthException, MultipleShowObjectsException
|
||||||
from sickbeard.indexers.indexer_config import *
|
from sickbeard.indexers.indexer_config import *
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
@ -291,7 +291,12 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
return [x for x in cats if x['id'] not in self.excludes]
|
return [x for x in cats if x['id'] not in self.excludes]
|
||||||
return ','.join(set(cats.split(',')) - self.excludes)
|
return ','.join(set(cats.split(',')) - self.excludes)
|
||||||
|
|
||||||
def check_auth_from_data(self, data):
|
def _check_auth(self, is_required=None):
|
||||||
|
if self.should_skip():
|
||||||
|
return False
|
||||||
|
return super(NewznabProvider, self)._check_auth(is_required)
|
||||||
|
|
||||||
|
def _check_auth_from_data(self, data, url):
|
||||||
|
|
||||||
if data is None or not hasattr(data, 'tag'):
|
if data is None or not hasattr(data, 'tag'):
|
||||||
return False
|
return False
|
||||||
|
|
@ -306,6 +311,13 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
raise AuthException('Your account on %s has been suspended, contact the admin.' % self.name)
|
raise AuthException('Your account on %s has been suspended, contact the admin.' % self.name)
|
||||||
elif '102' == code:
|
elif '102' == code:
|
||||||
raise AuthException('Your account isn\'t allowed to use the API on %s, contact the admin.' % self.name)
|
raise AuthException('Your account isn\'t allowed to use the API on %s, contact the admin.' % self.name)
|
||||||
|
elif '500' == code:
|
||||||
|
try:
|
||||||
|
retry_time, unit = re.findall(r'Retry in (\d+)\W+([a-z]+)', description, flags=re.I)[0]
|
||||||
|
except IndexError:
|
||||||
|
retry_time, unit = None, None
|
||||||
|
self.tmr_limit_update(retry_time, unit, description)
|
||||||
|
self.log_failure_url(url)
|
||||||
elif '910' == code:
|
elif '910' == code:
|
||||||
logger.log(
|
logger.log(
|
||||||
'%s %s, please check with provider.' %
|
'%s %s, please check with provider.' %
|
||||||
|
|
@ -316,6 +328,7 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
logger.WARNING)
|
logger.WARNING)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
self.tmr_limit_count = 0
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def config_str(self):
|
def config_str(self):
|
||||||
|
|
@ -530,15 +543,20 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
(hits_per_page * 100 // hits_per_page * 2, hits_per_page * int(ceil(rel_limit * 1.5)))[season_search])
|
(hits_per_page * 100 // hits_per_page * 2, hits_per_page * int(ceil(rel_limit * 1.5)))[season_search])
|
||||||
|
|
||||||
def find_search_results(self, show, episodes, search_mode, manual_search=False, try_other_searches=False, **kwargs):
|
def find_search_results(self, show, episodes, search_mode, manual_search=False, try_other_searches=False, **kwargs):
|
||||||
self._check_auth()
|
check = self._check_auth()
|
||||||
|
results = {}
|
||||||
|
if (isinstance(check, bool) and not check) or self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
self.show = show
|
self.show = show
|
||||||
|
|
||||||
results = {}
|
|
||||||
item_list = []
|
item_list = []
|
||||||
name_space = {}
|
name_space = {}
|
||||||
|
|
||||||
searched_scene_season = s_mode = None
|
searched_scene_season = s_mode = None
|
||||||
for ep_obj in episodes:
|
for ep_obj in episodes:
|
||||||
|
if self.should_skip(log_warning=False):
|
||||||
|
break
|
||||||
# skip if season already searched
|
# skip if season already searched
|
||||||
if (s_mode or 'sponly' == search_mode) and 1 < len(episodes) \
|
if (s_mode or 'sponly' == search_mode) and 1 < len(episodes) \
|
||||||
and searched_scene_season == ep_obj.scene_season:
|
and searched_scene_season == ep_obj.scene_season:
|
||||||
|
|
@ -577,6 +595,8 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
try_all_searches=try_other_searches)
|
try_all_searches=try_other_searches)
|
||||||
item_list += items
|
item_list += items
|
||||||
name_space.update(n_space)
|
name_space.update(n_space)
|
||||||
|
if self.should_skip():
|
||||||
|
break
|
||||||
|
|
||||||
return self.finish_find_search_results(
|
return self.finish_find_search_results(
|
||||||
show, episodes, search_mode, manual_search, results, item_list, name_space=name_space)
|
show, episodes, search_mode, manual_search, results, item_list, name_space=name_space)
|
||||||
|
|
@ -617,7 +637,13 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
def _search_provider(self, search_params, needed=neededQualities(need_all=True), max_items=400,
|
def _search_provider(self, search_params, needed=neededQualities(need_all=True), max_items=400,
|
||||||
try_all_searches=False, **kwargs):
|
try_all_searches=False, **kwargs):
|
||||||
|
|
||||||
|
results, n_spaces = [], {}
|
||||||
|
if self.should_skip():
|
||||||
|
return results, n_spaces
|
||||||
|
|
||||||
api_key = self._check_auth()
|
api_key = self._check_auth()
|
||||||
|
if isinstance(api_key, bool) and not api_key:
|
||||||
|
return results, n_spaces
|
||||||
|
|
||||||
base_params = {'t': 'tvsearch',
|
base_params = {'t': 'tvsearch',
|
||||||
'maxage': sickbeard.USENET_RETENTION or 0,
|
'maxage': sickbeard.USENET_RETENTION or 0,
|
||||||
|
|
@ -644,8 +670,13 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
cat_webdl = self.cats.get(NewznabConstants.CAT_WEBDL)
|
cat_webdl = self.cats.get(NewznabConstants.CAT_WEBDL)
|
||||||
|
|
||||||
for mode in search_params.keys():
|
for mode in search_params.keys():
|
||||||
|
if self.should_skip(log_warning=False):
|
||||||
|
break
|
||||||
for i, params in enumerate(search_params[mode]):
|
for i, params in enumerate(search_params[mode]):
|
||||||
|
|
||||||
|
if self.should_skip(log_warning=False):
|
||||||
|
break
|
||||||
|
|
||||||
# category ids
|
# category ids
|
||||||
cat = []
|
cat = []
|
||||||
if 'Episode' == mode or 'Season' == mode:
|
if 'Episode' == mode or 'Season' == mode:
|
||||||
|
|
@ -697,14 +728,13 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
search_url = '%sapi?%s' % (self.url, urllib.urlencode(request_params))
|
search_url = '%sapi?%s' % (self.url, urllib.urlencode(request_params))
|
||||||
i and time.sleep(2.1)
|
i and time.sleep(2.1)
|
||||||
|
|
||||||
data = helpers.getURL(search_url)
|
data = self.get_url(search_url)
|
||||||
|
|
||||||
if not data:
|
if self.should_skip() or not data:
|
||||||
logger.log('No Data returned from %s' % self.name, logger.WARNING)
|
|
||||||
break
|
break
|
||||||
|
|
||||||
# hack this in until it's fixed server side
|
# hack this in until it's fixed server side
|
||||||
if data and not data.startswith('<?xml'):
|
if not data.startswith('<?xml'):
|
||||||
data = '<?xml version="1.0" encoding="ISO-8859-1" ?>%s' % data
|
data = '<?xml version="1.0" encoding="ISO-8859-1" ?>%s' % data
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
@ -714,7 +744,7 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
logger.log('Error trying to load %s RSS feed' % self.name, logger.WARNING)
|
logger.log('Error trying to load %s RSS feed' % self.name, logger.WARNING)
|
||||||
break
|
break
|
||||||
|
|
||||||
if not self.check_auth_from_data(parsed_xml):
|
if not self._check_auth_from_data(parsed_xml, search_url):
|
||||||
break
|
break
|
||||||
|
|
||||||
if 'rss' != parsed_xml.tag:
|
if 'rss' != parsed_xml.tag:
|
||||||
|
|
@ -794,6 +824,10 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
results = [classes.Proper(x['name'], x['url'], datetime.datetime.fromtimestamp(x['time']), self.show) for x in
|
results = [classes.Proper(x['name'], x['url'], datetime.datetime.fromtimestamp(x['time']), self.show) for x in
|
||||||
cache_results]
|
cache_results]
|
||||||
|
|
||||||
|
check = self._check_auth()
|
||||||
|
if isinstance(check, bool) and not check:
|
||||||
|
return results
|
||||||
|
|
||||||
index = 0
|
index = 0
|
||||||
alt_search = ('nzbs_org' == self.get_id())
|
alt_search = ('nzbs_org' == self.get_id())
|
||||||
do_search_alt = False
|
do_search_alt = False
|
||||||
|
|
@ -812,6 +846,9 @@ class NewznabProvider(generic.NZBProvider):
|
||||||
|
|
||||||
urls = []
|
urls = []
|
||||||
while index < len(search_terms):
|
while index < len(search_terms):
|
||||||
|
if self.should_skip(log_warning=False):
|
||||||
|
break
|
||||||
|
|
||||||
search_params = {'q': search_terms[index], 'maxage': sickbeard.BACKLOG_DAYS + 2}
|
search_params = {'q': search_terms[index], 'maxage': sickbeard.BACKLOG_DAYS + 2}
|
||||||
if alt_search:
|
if alt_search:
|
||||||
|
|
||||||
|
|
@ -885,8 +922,11 @@ class NewznabCache(tvcache.TVCache):
|
||||||
if 4489 != sickbeard.RECENTSEARCH_FREQUENCY or self.should_update():
|
if 4489 != sickbeard.RECENTSEARCH_FREQUENCY or self.should_update():
|
||||||
n_spaces = {}
|
n_spaces = {}
|
||||||
try:
|
try:
|
||||||
self._checkAuth()
|
check = self._checkAuth()
|
||||||
(items, n_spaces) = self.provider.cache_data(needed=needed)
|
if isinstance(check, bool) and not check:
|
||||||
|
items = None
|
||||||
|
else:
|
||||||
|
(items, n_spaces) = self.provider.cache_data(needed=needed)
|
||||||
except (StandardError, Exception):
|
except (StandardError, Exception):
|
||||||
items = None
|
items = None
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -53,6 +53,8 @@ class NyaaProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % ((0, 2)[self.confirmed], search_string)
|
search_url = self.urls['search'] % ((0, 2)[self.confirmed], search_string)
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -100,9 +100,12 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
|
||||||
result = None
|
result = None
|
||||||
if url and False is self._init_api():
|
if url and False is self._init_api():
|
||||||
data = self.get_url(url, timeout=90)
|
data = self.get_url(url, timeout=90)
|
||||||
|
if self.should_skip():
|
||||||
|
return result
|
||||||
if data:
|
if data:
|
||||||
if re.search('(?i)limit.*?reached', data):
|
if re.search('(?i)limit.*?reached', data):
|
||||||
logger.log('Daily Nzb Download limit reached', logger.DEBUG)
|
self.tmr_limit_update('1', 'h', 'Your 24 hour limit of 10 NZBs has been reached')
|
||||||
|
self.log_failure_url(url)
|
||||||
elif '</nzb>' not in data or 'seem to be logged in' in data:
|
elif '</nzb>' not in data or 'seem to be logged in' in data:
|
||||||
logger.log('Failed nzb data response: %s' % data, logger.DEBUG)
|
logger.log('Failed nzb data response: %s' % data, logger.DEBUG)
|
||||||
else:
|
else:
|
||||||
|
|
@ -138,6 +141,9 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
|
||||||
|
|
||||||
def cache_data(self, needed=neededQualities(need_all=True), **kwargs):
|
def cache_data(self, needed=neededQualities(need_all=True), **kwargs):
|
||||||
|
|
||||||
|
if self.should_skip():
|
||||||
|
return []
|
||||||
|
|
||||||
api_key = self._init_api()
|
api_key = self._init_api()
|
||||||
if False is api_key:
|
if False is api_key:
|
||||||
return self.search_html(needed=needed, **kwargs)
|
return self.search_html(needed=needed, **kwargs)
|
||||||
|
|
@ -153,6 +159,8 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
|
||||||
url = self.urls['cache'] % urllib.urlencode(params)
|
url = self.urls['cache'] % urllib.urlencode(params)
|
||||||
|
|
||||||
response = self.get_url(url)
|
response = self.get_url(url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
data = feedparser.parse(response.replace('<xml', '<?xml').replace('>\n<info>', '?>\n<feed>\n<info>')
|
data = feedparser.parse(response.replace('<xml', '<?xml').replace('>\n<info>', '?>\n<feed>\n<info>')
|
||||||
.replace('<search_req>\n', '').replace('</search_req>\n', '')
|
.replace('<search_req>\n', '').replace('</search_req>\n', '')
|
||||||
|
|
@ -183,6 +191,8 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
|
||||||
search_url = self.urls['search'] % urllib.urlencode(params)
|
search_url = self.urls['search'] % urllib.urlencode(params)
|
||||||
|
|
||||||
data_json = self.get_url(search_url, json=True)
|
data_json = self.get_url(search_url, json=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
if data_json and self._check_auth_from_data(data_json, is_xml=False):
|
if data_json and self._check_auth_from_data(data_json, is_xml=False):
|
||||||
for item in data_json:
|
for item in data_json:
|
||||||
if 'release' in item and 'getnzb' in item:
|
if 'release' in item and 'getnzb' in item:
|
||||||
|
|
@ -211,6 +221,8 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
|
||||||
mode = ('search', 'cache')['' == search]
|
mode = ('search', 'cache')['' == search]
|
||||||
search_url = self.urls[mode + '_html'] % search
|
search_url = self.urls[mode + '_html'] % search
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
cnt = len(results)
|
cnt = len(results)
|
||||||
try:
|
try:
|
||||||
if not html:
|
if not html:
|
||||||
|
|
@ -254,6 +266,8 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
|
||||||
|
|
||||||
search_terms = ['.PROPER.', '.REPACK.', '.REAL.']
|
search_terms = ['.PROPER.', '.REPACK.', '.REAL.']
|
||||||
results = []
|
results = []
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
for term in search_terms:
|
for term in search_terms:
|
||||||
for item in self._search_provider(term, search_mode='Propers', retention=4):
|
for item in self._search_provider(term, search_mode='Propers', retention=4):
|
||||||
|
|
@ -272,6 +286,9 @@ class OmgwtfnzbsProvider(generic.NZBProvider):
|
||||||
|
|
||||||
def _init_api(self):
|
def _init_api(self):
|
||||||
|
|
||||||
|
if self.should_skip():
|
||||||
|
return None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
api_key = self._check_auth()
|
api_key = self._check_auth()
|
||||||
if not api_key.startswith('cookie:'):
|
if not api_key.startswith('cookie:'):
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,8 @@ class PiSexyProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % search_string
|
search_url = self.urls['search'] % search_string
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -94,6 +94,8 @@ class PotUKProvider(generic.TorrentProvider):
|
||||||
params.setdefault(name, value)
|
params.setdefault(name, value)
|
||||||
del params['doprefs']
|
del params['doprefs']
|
||||||
html = self.get_url(search_url, post_data=params)
|
html = self.get_url(search_url, post_data=params)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
@ -135,6 +137,9 @@ class PotUKProvider(generic.TorrentProvider):
|
||||||
def get_data(self, url):
|
def get_data(self, url):
|
||||||
result = None
|
result = None
|
||||||
html = self.get_url(url, timeout=90)
|
html = self.get_url(url, timeout=90)
|
||||||
|
if self.should_skip():
|
||||||
|
return result
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = self._link(re.findall('(?i)"(attachment\.php[^"]+?)"', html)[0])
|
result = self._link(re.findall('(?i)"(attachment\.php[^"]+?)"', html)[0])
|
||||||
except IndexError:
|
except IndexError:
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,6 @@
|
||||||
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
|
# along with SickGear. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from . import generic
|
from . import generic
|
||||||
from sickbeard.rssfeeds import RSSFeeds
|
|
||||||
from lib.unidecode import unidecode
|
from lib.unidecode import unidecode
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -52,7 +51,7 @@ class PreToMeProvider(generic.TorrentProvider):
|
||||||
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
|
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
|
||||||
search_url = url + (self.urls['search'] % search_string, '')['Cache' == mode]
|
search_url = url + (self.urls['search'] % search_string, '')['Cache' == mode]
|
||||||
|
|
||||||
xml_data = RSSFeeds(self).get_feed(search_url)
|
xml_data = self.cache.get_rss(search_url)
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
if xml_data and 'entries' in xml_data:
|
if xml_data and 'entries' in xml_data:
|
||||||
|
|
|
||||||
|
|
@ -97,6 +97,8 @@ class PrivateHDProvider(generic.TorrentProvider):
|
||||||
'+'.join(search_string.split()), self._categories_string(mode, ''))
|
'+'.join(search_string.split()), self._categories_string(mode, ''))
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -85,11 +85,16 @@ class PTFProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
search_url = self.urls['search'] % ('+'.join(search_string.split()), self._categories_string(mode))
|
search_url = self.urls['search'] % ('+'.join(search_string.split()), self._categories_string(mode))
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
if not self.has_all_cookies(['session_key']):
|
if not self.has_all_cookies(['session_key']):
|
||||||
if not self._authorised():
|
if not self._authorised():
|
||||||
return results
|
return results
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -58,8 +58,8 @@ class RarbgProvider(generic.TorrentProvider):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
for r in range(0, 3):
|
for r in range(0, 3):
|
||||||
response = helpers.getURL(self.urls['api_token'], session=self.session, json=True)
|
response = self.get_url(self.urls['api_token'], json=True)
|
||||||
if response and 'token' in response:
|
if not self.should_skip() and response and 'token' in response:
|
||||||
self.token = response['token']
|
self.token = response['token']
|
||||||
self.token_expiry = datetime.datetime.now() + datetime.timedelta(minutes=14)
|
self.token_expiry = datetime.datetime.now() + datetime.timedelta(minutes=14)
|
||||||
return True
|
return True
|
||||||
|
|
@ -125,6 +125,8 @@ class RarbgProvider(generic.TorrentProvider):
|
||||||
searched_url = search_url % {'r': int(self.confirmed), 't': self.token}
|
searched_url = search_url % {'r': int(self.confirmed), 't': self.token}
|
||||||
|
|
||||||
data_json = self.get_url(searched_url, json=True)
|
data_json = self.get_url(searched_url, json=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
self.token_expiry = datetime.datetime.now() + datetime.timedelta(minutes=14)
|
self.token_expiry = datetime.datetime.now() + datetime.timedelta(minutes=14)
|
||||||
self.request_throttle = datetime.datetime.now() + datetime.timedelta(seconds=3)
|
self.request_throttle = datetime.datetime.now() + datetime.timedelta(seconds=3)
|
||||||
|
|
|
||||||
|
|
@ -63,6 +63,8 @@ class RevTTProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
html = self.get_url(self.urls['search'] % ('+'.join(search_string.split()),
|
html = self.get_url(self.urls['search'] % ('+'.join(search_string.split()),
|
||||||
self._categories_string(mode)))
|
self._categories_string(mode)))
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,6 @@ from . import generic
|
||||||
from sickbeard import logger, tvcache
|
from sickbeard import logger, tvcache
|
||||||
from sickbeard.helpers import tryInt
|
from sickbeard.helpers import tryInt
|
||||||
from sickbeard.exceptions import ex
|
from sickbeard.exceptions import ex
|
||||||
from sickbeard.rssfeeds import RSSFeeds
|
|
||||||
from lib.bencode import bdecode
|
from lib.bencode import bdecode
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -41,8 +40,6 @@ class TorrentRssProvider(generic.TorrentProvider):
|
||||||
self.search_mode = search_mode
|
self.search_mode = search_mode
|
||||||
self.search_fallback = bool(tryInt(search_fallback))
|
self.search_fallback = bool(tryInt(search_fallback))
|
||||||
|
|
||||||
self.feeder = RSSFeeds(self)
|
|
||||||
|
|
||||||
def image_name(self):
|
def image_name(self):
|
||||||
|
|
||||||
return generic.GenericProvider.image_name(self, 'torrentrss')
|
return generic.GenericProvider.image_name(self, 'torrentrss')
|
||||||
|
|
@ -102,6 +99,9 @@ class TorrentRssProvider(generic.TorrentProvider):
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
torrent_file = self.get_url(url)
|
torrent_file = self.get_url(url)
|
||||||
|
if self.should_skip():
|
||||||
|
break
|
||||||
|
|
||||||
try:
|
try:
|
||||||
bdecode(torrent_file)
|
bdecode(torrent_file)
|
||||||
break
|
break
|
||||||
|
|
@ -120,7 +120,7 @@ class TorrentRssProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
result = []
|
result = []
|
||||||
for mode in search_params.keys():
|
for mode in search_params.keys():
|
||||||
data = self.feeder.get_feed(self.url)
|
data = self.cache.get_rss(self.url)
|
||||||
|
|
||||||
result += (data and 'entries' in data) and data.entries or []
|
result += (data and 'entries' in data) and data.entries or []
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -61,6 +61,8 @@ class SceneHDProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (search_string, self._categories_string(mode, '%s', ','))
|
search_url = self.urls['search'] % (search_string, self._categories_string(mode, '%s', ','))
|
||||||
|
|
||||||
html = self.get_url(search_url, timeout=90)
|
html = self.get_url(search_url, timeout=90)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -80,6 +80,8 @@ class SceneTimeProvider(generic.TorrentProvider):
|
||||||
|
|
||||||
self.session.headers.update({'Referer': self.url + 'browse.php', 'X-Requested-With': 'XMLHttpRequest'})
|
self.session.headers.update({'Referer': self.url + 'browse.php', 'X-Requested-With': 'XMLHttpRequest'})
|
||||||
html = self.get_url(self.urls['browse'], post_data=post_data)
|
html = self.get_url(self.urls['browse'], post_data=post_data)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -49,8 +49,8 @@ class ShazbatProvider(generic.TorrentProvider):
|
||||||
def _authorised(self, **kwargs):
|
def _authorised(self, **kwargs):
|
||||||
|
|
||||||
return super(ShazbatProvider, self)._authorised(
|
return super(ShazbatProvider, self)._authorised(
|
||||||
logged_in=(lambda y=None: '<input type="password"' not in helpers.getURL(
|
logged_in=(lambda y=None: '<input type="password"' not in self.get_url(self.urls['feeds'], skip_auth=True)),
|
||||||
self.urls['feeds'], session=self.session)), post_params={'tv_login': self.username, 'form_tmpl': True})
|
post_params={'tv_login': self.username, 'form_tmpl': True})
|
||||||
|
|
||||||
def _search_provider(self, search_params, **kwargs):
|
def _search_provider(self, search_params, **kwargs):
|
||||||
|
|
||||||
|
|
@ -70,11 +70,16 @@ class ShazbatProvider(generic.TorrentProvider):
|
||||||
if 'Cache' == mode:
|
if 'Cache' == mode:
|
||||||
search_url = self.urls['browse']
|
search_url = self.urls['browse']
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
else:
|
else:
|
||||||
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
|
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
|
||||||
search_string = search_string.replace(show_detail, '').strip()
|
search_string = search_string.replace(show_detail, '').strip()
|
||||||
search_url = self.urls['search'] % search_string
|
search_url = self.urls['search'] % search_string
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
shows = rc['show_id'].findall(html)
|
shows = rc['show_id'].findall(html)
|
||||||
if not any(shows):
|
if not any(shows):
|
||||||
continue
|
continue
|
||||||
|
|
@ -85,6 +90,8 @@ class ShazbatProvider(generic.TorrentProvider):
|
||||||
continue
|
continue
|
||||||
html and time.sleep(1.1)
|
html and time.sleep(1.1)
|
||||||
html += self.get_url(self.urls['show'] % sid)
|
html += self.get_url(self.urls['show'] % sid)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,8 @@ class SkytorrentsProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % search_string
|
search_url = self.urls['search'] % search_string
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -67,6 +67,8 @@ class SpeedCDProvider(generic.TorrentProvider):
|
||||||
jxt=2, jxw='b', freeleech=('on', None)[not self.freeleech])
|
jxt=2, jxw='b', freeleech=('on', None)[not self.freeleech])
|
||||||
|
|
||||||
data_json = self.get_url(self.urls['search'], post_data=post_data, json=True)
|
data_json = self.get_url(self.urls['search'], post_data=post_data, json=True)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -106,7 +106,7 @@ class ThePirateBayProvider(generic.TorrentProvider):
|
||||||
quality = Quality.UNKNOWN
|
quality = Quality.UNKNOWN
|
||||||
file_name = None
|
file_name = None
|
||||||
data = self.get_url('%sajax_details_filelist.php?id=%s' % (self.url, torrent_id))
|
data = self.get_url('%sajax_details_filelist.php?id=%s' % (self.url, torrent_id))
|
||||||
if not data:
|
if self.should_skip() or not data:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
files_list = re.findall('<td.+>(.*?)</td>', data)
|
files_list = re.findall('<td.+>(.*?)</td>', data)
|
||||||
|
|
@ -193,6 +193,8 @@ class ThePirateBayProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['browse'] if 'Cache' == mode \
|
search_url = self.urls['browse'] if 'Cache' == mode \
|
||||||
else self.urls['search'] % (urllib.quote(search_string))
|
else self.urls['search'] % (urllib.quote(search_string))
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -49,6 +49,9 @@ class TokyoToshokanProvider(generic.TorrentProvider):
|
||||||
'stats': 'S:\s*?(\d)+\s*L:\s*(\d+)', 'size': 'size:\s*(\d+[.,]\d+\w+)'}.iteritems())
|
'stats': 'S:\s*?(\d)+\s*L:\s*(\d+)', 'size': 'size:\s*(\d+[.,]\d+\w+)'}.iteritems())
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return self._sort_seeding(mode, results)
|
||||||
|
|
||||||
if html:
|
if html:
|
||||||
try:
|
try:
|
||||||
with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
|
with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
|
||||||
|
|
@ -103,7 +106,7 @@ class TokyoToshokanCache(tvcache.TVCache):
|
||||||
|
|
||||||
mode = 'Cache'
|
mode = 'Cache'
|
||||||
search_url = '%srss.php?%s' % (self.provider.url, urllib.urlencode({'filter': '1'}))
|
search_url = '%srss.php?%s' % (self.provider.url, urllib.urlencode({'filter': '1'}))
|
||||||
data = self.getRSSFeed(search_url)
|
data = self.get_rss(search_url)
|
||||||
|
|
||||||
results = []
|
results = []
|
||||||
if data and 'entries' in data:
|
if data and 'entries' in data:
|
||||||
|
|
|
||||||
|
|
@ -74,6 +74,8 @@ class TorLockProvider(generic.TorrentProvider):
|
||||||
else self.urls['search'] % (urllib.quote_plus(search_string).replace('+', '-'))
|
else self.urls['search'] % (urllib.quote_plus(search_string).replace('+', '-'))
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -61,6 +61,8 @@ class TorrentBytesProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (search_string, self._categories_string(mode))
|
search_url = self.urls['search'] % (search_string, self._categories_string(mode))
|
||||||
|
|
||||||
html = self.get_url(search_url, timeout=90)
|
html = self.get_url(search_url, timeout=90)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -86,6 +86,8 @@ class TorrentDayProvider(generic.TorrentProvider):
|
||||||
search_string, ('&sort=7&type=desc', '')['Cache' == mode])
|
search_string, ('&sort=7&type=desc', '')['Cache' == mode])
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -69,6 +69,8 @@ class TorrentingProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (self._categories_string(), search_string)
|
search_url = self.urls['search'] % (self._categories_string(), search_string)
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -62,6 +62,8 @@ class TorrentLeechProvider(generic.TorrentProvider):
|
||||||
'query': isinstance(search_string, unicode) and unidecode(search_string) or search_string}
|
'query': isinstance(search_string, unicode) and unidecode(search_string) or search_string}
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -93,6 +93,8 @@ class Torrentz2Provider(generic.TorrentProvider):
|
||||||
'tv%s' % ('+' + quote_plus(search_string), '')['Cache' == mode])
|
'tv%s' % ('+' + quote_plus(search_string), '')['Cache' == mode])
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,8 @@ class TVChaosUKProvider(generic.TorrentProvider):
|
||||||
'order': 'desc', 'daysprune': '-1'})
|
'order': 'desc', 'daysprune': '-1'})
|
||||||
|
|
||||||
html = self.get_url(self.urls['search'], **kwargs)
|
html = self.get_url(self.urls['search'], **kwargs)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -70,6 +70,8 @@ class WOPProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (search_string, self._categories_string(mode, 'cats2[]=%s'))
|
search_url = self.urls['search'] % (search_string, self._categories_string(mode, 'cats2[]=%s'))
|
||||||
|
|
||||||
html = self.get_url(search_url, timeout=90)
|
html = self.get_url(search_url, timeout=90)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -58,6 +58,8 @@ class ZooqleProvider(generic.TorrentProvider):
|
||||||
search_url = self.urls['search'] % (search_string, self._categories_string(mode, '', ','))
|
search_url = self.urls['search'] % (search_string, self._categories_string(mode, '', ','))
|
||||||
|
|
||||||
html = self.get_url(search_url)
|
html = self.get_url(search_url)
|
||||||
|
if self.should_skip():
|
||||||
|
return results
|
||||||
|
|
||||||
cnt = len(items[mode])
|
cnt = len(items[mode])
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -5,54 +5,32 @@
|
||||||
|
|
||||||
import feedparser
|
import feedparser
|
||||||
|
|
||||||
from sickbeard import helpers, logger
|
from sickbeard import logger
|
||||||
from sickbeard.exceptions import ex
|
from sickbeard.exceptions import ex
|
||||||
|
|
||||||
|
|
||||||
class RSSFeeds:
|
class RSSFeeds:
|
||||||
|
|
||||||
def __init__(self, provider=None):
|
def __init__(self, provider=None):
|
||||||
|
|
||||||
self.provider = provider
|
self.provider = provider
|
||||||
self.response = None
|
|
||||||
|
|
||||||
def _check_auth_cookie(self):
|
def get_feed(self, url, **kwargs):
|
||||||
|
|
||||||
if self.provider:
|
if self.provider and self.provider.check_auth_cookie():
|
||||||
return self.provider.check_auth_cookie()
|
response = self.provider.get_url(url, **kwargs)
|
||||||
return True
|
if not self.provider.should_skip() and response:
|
||||||
|
try:
|
||||||
|
data = feedparser.parse(response)
|
||||||
|
data['rq_response'] = self.provider.session.response
|
||||||
|
if data and 'entries' in data:
|
||||||
|
return data
|
||||||
|
|
||||||
# noinspection PyUnusedLocal
|
if data and 'error' in data.feed:
|
||||||
def cb_response(self, r, *args, **kwargs):
|
err_code = data.feed['error']['code']
|
||||||
self.response = dict(url=r.url, elapsed=r.elapsed, from_cache=r.from_cache)
|
err_desc = data.feed['error']['description']
|
||||||
return r
|
logger.log(u'RSS error:[%s] code:[%s]' % (err_desc, err_code), logger.DEBUG)
|
||||||
|
else:
|
||||||
|
logger.log(u'RSS error loading url: ' + url, logger.DEBUG)
|
||||||
|
|
||||||
def get_feed(self, url, request_headers=None, **kwargs):
|
except Exception as e:
|
||||||
|
logger.log(u'RSS error: ' + ex(e), logger.DEBUG)
|
||||||
if not self._check_auth_cookie():
|
|
||||||
return
|
|
||||||
|
|
||||||
session = None
|
|
||||||
if self.provider and hasattr(self.provider, 'session'):
|
|
||||||
session = self.provider.session
|
|
||||||
|
|
||||||
response = helpers.getURL(url, headers=request_headers, session=session,
|
|
||||||
hooks=dict(response=self.cb_response), **kwargs)
|
|
||||||
if not response:
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
feed = feedparser.parse(response)
|
|
||||||
feed['rq_response'] = self.response
|
|
||||||
if feed and 'entries' in feed:
|
|
||||||
return feed
|
|
||||||
|
|
||||||
if feed and 'error' in feed.feed:
|
|
||||||
err_code = feed.feed['error']['code']
|
|
||||||
err_desc = feed.feed['error']['description']
|
|
||||||
logger.log(u'RSS ERROR:[%s] CODE:[%s]' % (err_desc, err_code), logger.DEBUG)
|
|
||||||
else:
|
|
||||||
logger.log(u'RSS error loading url: ' + url, logger.DEBUG)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.log(u'RSS error: ' + ex(e), logger.DEBUG)
|
|
||||||
|
|
|
||||||
|
|
@ -143,7 +143,7 @@ def snatch_episode(result, end_status=SNATCHED):
|
||||||
# make sure we have the torrent file content
|
# make sure we have the torrent file content
|
||||||
if not result.content and not result.url.startswith('magnet'):
|
if not result.content and not result.url.startswith('magnet'):
|
||||||
result.content = result.provider.get_url(result.url)
|
result.content = result.provider.get_url(result.url)
|
||||||
if not result.content:
|
if result.provider.should_skip() or not result.content:
|
||||||
logger.log(u'Torrent content failed to download from %s' % result.url, logger.ERROR)
|
logger.log(u'Torrent content failed to download from %s' % result.url, logger.ERROR)
|
||||||
return False
|
return False
|
||||||
# Snatches torrent with client
|
# Snatches torrent with client
|
||||||
|
|
@ -465,11 +465,18 @@ def search_for_needed_episodes(episodes):
|
||||||
best_result.content = None
|
best_result.content = None
|
||||||
if not best_result.url.startswith('magnet'):
|
if not best_result.url.startswith('magnet'):
|
||||||
best_result.content = best_result.provider.get_url(best_result.url)
|
best_result.content = best_result.provider.get_url(best_result.url)
|
||||||
|
if best_result.provider.should_skip():
|
||||||
|
break
|
||||||
if not best_result.content:
|
if not best_result.content:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
found_results[cur_ep] = best_result
|
found_results[cur_ep] = best_result
|
||||||
|
|
||||||
|
try:
|
||||||
|
cur_provider.save_list()
|
||||||
|
except (StandardError, Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
threading.currentThread().name = orig_thread_name
|
threading.currentThread().name = orig_thread_name
|
||||||
|
|
||||||
if not len(providers):
|
if not len(providers):
|
||||||
|
|
|
||||||
|
|
@ -107,7 +107,7 @@ class TVCache:
|
||||||
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def getRSSFeed(self, url, **kwargs):
|
def get_rss(self, url, **kwargs):
|
||||||
return RSSFeeds(self.provider).get_feed(url, **kwargs)
|
return RSSFeeds(self.provider).get_feed(url, **kwargs)
|
||||||
|
|
||||||
def _translateTitle(self, title):
|
def _translateTitle(self, title):
|
||||||
|
|
|
||||||
|
|
@ -4531,11 +4531,27 @@ class ManageSearches(Manage):
|
||||||
t.recent_search_status = sickbeard.searchQueueScheduler.action.is_recentsearch_in_progress()
|
t.recent_search_status = sickbeard.searchQueueScheduler.action.is_recentsearch_in_progress()
|
||||||
t.find_propers_status = sickbeard.searchQueueScheduler.action.is_propersearch_in_progress()
|
t.find_propers_status = sickbeard.searchQueueScheduler.action.is_propersearch_in_progress()
|
||||||
t.queue_length = sickbeard.searchQueueScheduler.action.queue_length()
|
t.queue_length = sickbeard.searchQueueScheduler.action.queue_length()
|
||||||
|
t.provider_fail_stats = filter(lambda stat: len(stat['fails']), [{
|
||||||
|
'active': p.is_active(), 'name': p.name, 'prov_id': p.get_id(), 'prov_img': p.image_name(),
|
||||||
|
'fails': p.fails.fails_sorted, 'tmr_limit_time': p.tmr_limit_time,
|
||||||
|
'next_try': p.get_next_try_time, 'has_limit': getattr(p, 'has_limit', False)}
|
||||||
|
for p in sickbeard.providerList + sickbeard.newznabProviderList])
|
||||||
|
t.provider_fails = 0 < len([p for p in t.provider_fail_stats if len(p['fails'])])
|
||||||
|
|
||||||
t.submenu = self.ManageMenu('Search')
|
t.submenu = self.ManageMenu('Search')
|
||||||
|
|
||||||
return t.respond()
|
return t.respond()
|
||||||
|
|
||||||
|
def retryProvider(self, provider=None, *args, **kwargs):
|
||||||
|
if not provider:
|
||||||
|
return
|
||||||
|
prov = [p for p in sickbeard.providerList + sickbeard.newznabProviderList if p.get_id() == provider]
|
||||||
|
if not prov:
|
||||||
|
return
|
||||||
|
prov[0].retry_next()
|
||||||
|
time.sleep(3)
|
||||||
|
return
|
||||||
|
|
||||||
def forceVersionCheck(self, *args, **kwargs):
|
def forceVersionCheck(self, *args, **kwargs):
|
||||||
# force a check to see if there is a new version
|
# force a check to see if there is a new version
|
||||||
if sickbeard.versionCheckScheduler.action.check_for_new_version(force=True):
|
if sickbeard.versionCheckScheduler.action.check_for_new_version(force=True):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue