monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2024-12-03 18:03:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #477 from JackDandy/feature/AddCrawlerExclusions

Browse source 
Add search crawler exclusions.
This commit is contained in:
JackDandy 2015-08-10 22:10:42 +01:00
commit 627179f3d7
5 changed files with 54 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGES.md
View file

@ -2,6 +2,7 @@
* Change to only refresh scene exception data for shows that need it * Change to only refresh scene exception data for shows that need it
* Fix small cosmetic issue to correctly display "full backlog" date * Fix small cosmetic issue to correctly display "full backlog" date
* Add search crawler exclusions
### 0.10.0 (2015-08-06 11:05:00 UTC) ### 0.10.0 (2015-08-06 11:05:00 UTC)

59
gui/slick/interfaces/default/inc_top.tmpl
View file

@ -5,8 +5,9 @@
<html> <html>
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="robots" content="noindex, nofollow, noarchive, nocache, noodp, noydir, noimageindex, nosnippet">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>SickGear - BRANCH:[$sickbeard.BRANCH] - $title</title> <title>SickGear - BRANCH:[$sickbeard.BRANCH] - $title</title>
@ -34,35 +35,35 @@
<meta name="msapplication-TileImage" content="$sbRoot/images/ico/mstile-144x144.png"> <meta name="msapplication-TileImage" content="$sbRoot/images/ico/mstile-144x144.png">
<meta name="msapplication-config" content="$sbRoot/css/browserconfig.xml"> <meta name="msapplication-config" content="$sbRoot/css/browserconfig.xml">
<link rel="stylesheet" type="text/css" href="$sbRoot/css/lib/bootstrap.css?$sbPID"/> <link rel="stylesheet" type="text/css" href="$sbRoot/css/lib/bootstrap.css?v=$sbPID"/>
<link rel="stylesheet" type="text/css" href="$sbRoot/css/browser.css?$sbPID" /> <link rel="stylesheet" type="text/css" href="$sbRoot/css/browser.css?v=$sbPID" />
<link rel="stylesheet" type="text/css" href="$sbRoot/css/lib/jquery-ui-1.10.4.custom.css?$sbPID" /> <link rel="stylesheet" type="text/css" href="$sbRoot/css/lib/jquery-ui-1.10.4.custom.css?v=$sbPID" />
<link rel="stylesheet" type="text/css" href="$sbRoot/css/lib/jquery.qtip-2.2.1.min.css?$sbPID"/> <link rel="stylesheet" type="text/css" href="$sbRoot/css/lib/jquery.qtip-2.2.1.min.css?v=$sbPID"/>
<link rel="stylesheet" type="text/css" href="$sbRoot/css/lib/pnotify.custom.min.css?$sbPID" /> <link rel="stylesheet" type="text/css" href="$sbRoot/css/lib/pnotify.custom.min.css?v=$sbPID" />
<link rel="stylesheet" type="text/css" href="$sbRoot/css/style.css?$sbPID"/> <link rel="stylesheet" type="text/css" href="$sbRoot/css/style.css?v=$sbPID"/>
<link rel="stylesheet" type="text/css" href="$sbRoot/css/${sickbeard.THEME_NAME}.css?$sbPID" /> <link rel="stylesheet" type="text/css" href="$sbRoot/css/${sickbeard.THEME_NAME}.css?v=$sbPID" />
<script type="text/javascript" src="$sbRoot/js/lib/jquery-1.8.3.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery-1.8.3.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/bootstrap.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/bootstrap.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/bootstrap-hover-dropdown.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/bootstrap-hover-dropdown.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery-ui-1.10.4.custom.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery-ui-1.10.4.custom.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.cookie.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.cookie.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.cookiejar.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.cookiejar.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.json-2.2.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.json-2.2.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.selectboxes.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.selectboxes.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.tablesorter-2.17.7.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.tablesorter-2.17.7.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.tablesorter.widgets-2.17.7.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.tablesorter.widgets-2.17.7.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.qtip-2.2.1.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.qtip-2.2.1.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/pnotify.custom.min.js"></script> <script type="text/javascript" src="$sbRoot/js/lib/pnotify.custom.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.form-3.35.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.form-3.35.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.ui.touch-punch-0.2.2.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.ui.touch-punch-0.2.2.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/isotope.pkgd.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/isotope.pkgd.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/imagesloaded.pkgd.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/imagesloaded.pkgd.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/lib/jquery.confirm.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/lib/jquery.confirm.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/script.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/script.js?v=$sbPID"></script>
#if $sickbeard.FUZZY_DATING #if $sickbeard.FUZZY_DATING
<script type="text/javascript" src="$sbRoot/js/moment/moment.min.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/moment/moment.min.js?v=$sbPID"></script>
<script type="text/javascript" src="$sbRoot/js/fuzzyMoment.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/fuzzyMoment.js?v=$sbPID"></script>
#end if #end if
<script type="text/javascript" charset="utf-8"> <script type="text/javascript" charset="utf-8">
<!-- <!--
@ -119,7 +120,7 @@
}); });
//--> //-->
</script> </script>
<script type="text/javascript" src="$sbRoot/js/confirmations.js?$sbPID"></script> <script type="text/javascript" src="$sbRoot/js/confirmations.js?v=$sbPID"></script>
</head> </head>
#set $tab = 4 #set $tab = 4
#set $body_attr = '' #set $body_attr = ''

1
sickbeard/webapi.py
View file

@ -75,6 +75,7 @@ class Api(webserve.BaseHandler):
def set_default_headers(self): def set_default_headers(self):
self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0')
self.set_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nocache, noodp, noydir, noimageindex, nosnippet')
def get(self, route, *args, **kwargs): def get(self, route, *args, **kwargs):
route = route.strip('/') or 'index' route = route.strip('/') or 'index'

8
sickbeard/webserve.py
View file

@ -49,7 +49,7 @@ from sickbeard.browser import foldersAtPath
from sickbeard.blackandwhitelist import BlackAndWhiteList, short_group_names from sickbeard.blackandwhitelist import BlackAndWhiteList, short_group_names
from sickbeard.searchBacklog import FULL_BACKLOG, LIMITED_BACKLOG from sickbeard.searchBacklog import FULL_BACKLOG, LIMITED_BACKLOG
from tornado import gen from tornado import gen
from tornado.web import RequestHandler, authenticated from tornado.web import RequestHandler, StaticFileHandler, authenticated
from lib import adba from lib import adba
from lib import subliminal from lib import subliminal
from lib.dateutil import tz from lib.dateutil import tz
@ -111,9 +111,15 @@ class PageTemplate(Template):
return super(PageTemplate, self).compile(*args, **kwargs) return super(PageTemplate, self).compile(*args, **kwargs)
class BaseStaticFileHandler(StaticFileHandler):
def set_extra_headers(self, path):
self.set_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nocache, noodp, noydir, noimageindex, nosnippet')
class BaseHandler(RequestHandler): class BaseHandler(RequestHandler):
def set_default_headers(self): def set_default_headers(self):
self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0')
self.set_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nocache, noodp, noydir, noimageindex, nosnippet')
def redirect(self, url, permanent=False, status=None): def redirect(self, url, permanent=False, status=None):
if not url.startswith(sickbeard.WEB_ROOT): if not url.startswith(sickbeard.WEB_ROOT):

31
sickbeard/webserveInit.py
View file

@ -7,7 +7,7 @@ import webapi
from sickbeard import logger from sickbeard import logger
from sickbeard.helpers import create_https_certificates from sickbeard.helpers import create_https_certificates
from tornado.web import Application, StaticFileHandler from tornado.web import Application
from tornado.httpserver import HTTPServer from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop from tornado.ioloop import IOLoop
@ -41,8 +41,8 @@ class WebServer(threading.Thread):
if self.enable_https: if self.enable_https:
# If either the HTTPS certificate or key do not exist, make some self-signed ones. # If either the HTTPS certificate or key do not exist, make some self-signed ones.
if not (self.https_cert and os.path.exists(self.https_cert)) or not ( if not (self.https_cert and os.path.exists(self.https_cert))\
self.https_key and os.path.exists(self.https_key)): or not (self.https_key and os.path.exists(self.https_key)):
if not create_https_certificates(self.https_cert, self.https_key): if not create_https_certificates(self.https_cert, self.https_key):
logger.log(u'Unable to create CERT/KEY files, disabling HTTPS') logger.log(u'Unable to create CERT/KEY files, disabling HTTPS')
sickbeard.ENABLE_HTTPS = False sickbeard.ENABLE_HTTPS = False
@ -55,13 +55,13 @@ class WebServer(threading.Thread):
# Load the app # Load the app
self.app = Application([], self.app = Application([],
debug=True, debug=True,
autoreload=False, autoreload=False,
gzip=True, gzip=True,
xheaders=sickbeard.HANDLE_REVERSE_PROXY, xheaders=sickbeard.HANDLE_REVERSE_PROXY,
cookie_secret=sickbeard.COOKIE_SECRET, cookie_secret=sickbeard.COOKIE_SECRET,
login_url='%s/login/' % self.options['web_root'], login_url='%s/login/' % self.options['web_root']
) )
# Main Handler # Main Handler
self.app.add_handlers('.*$', [ self.app.add_handlers('.*$', [
@ -104,27 +104,26 @@ class WebServer(threading.Thread):
# Static File Handlers # Static File Handlers
self.app.add_handlers('.*$', [ self.app.add_handlers('.*$', [
# favicon # favicon
(r'%s/(favicon\.ico)' % self.options['web_root'], StaticFileHandler, (r'%s/(favicon\.ico)' % self.options['web_root'], webserve.BaseStaticFileHandler,
{'path': os.path.join(self.options['data_root'], 'images/ico/favicon.ico')}), {'path': os.path.join(self.options['data_root'], 'images/ico/favicon.ico')}),
# images # images
(r'%s/images/(.*)' % self.options['web_root'], StaticFileHandler, (r'%s/images/(.*)' % self.options['web_root'], webserve.BaseStaticFileHandler,
{'path': os.path.join(self.options['data_root'], 'images')}), {'path': os.path.join(self.options['data_root'], 'images')}),
# cached images # cached images
(r'%s/cache/images/(.*)' % self.options['web_root'], StaticFileHandler, (r'%s/cache/images/(.*)' % self.options['web_root'], webserve.BaseStaticFileHandler,
{'path': os.path.join(sickbeard.CACHE_DIR, 'images')}), {'path': os.path.join(sickbeard.CACHE_DIR, 'images')}),
# css # css
(r'%s/css/(.*)' % self.options['web_root'], StaticFileHandler, (r'%s/css/(.*)' % self.options['web_root'], webserve.BaseStaticFileHandler,
{'path': os.path.join(self.options['data_root'], 'css')}), {'path': os.path.join(self.options['data_root'], 'css')}),
# javascript # javascript
(r'%s/js/(.*)' % self.options['web_root'], StaticFileHandler, (r'%s/js/(.*)' % self.options['web_root'], webserve.BaseStaticFileHandler,
{'path': os.path.join(self.options['data_root'], 'js')}), {'path': os.path.join(self.options['data_root'], 'js')}),
]) ])
def run(self): def run(self):
if self.enable_https: if self.enable_https:
protocol = 'https' protocol = 'https'