From b124b453fe4796a4c5576143c27973379505594f Mon Sep 17 00:00:00 2001 From: JackDandy Date: Sat, 8 Aug 2015 00:53:30 +0100 Subject: [PATCH] Add search crawler exclusions. --- CHANGES.md | 1 + gui/slick/interfaces/default/inc_top.tmpl | 59 ++++++++++++----------- sickbeard/webapi.py | 1 + sickbeard/webserve.py | 8 ++- sickbeard/webserveInit.py | 31 ++++++------ 5 files changed, 54 insertions(+), 46 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 10fe667a..9d97ec53 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ * Change to only refresh scene exception data for shows that need it * Fix small cosmetic issue to correctly display "full backlog" date +* Add search crawler exclusions ### 0.10.0 (2015-08-06 11:05:00 UTC) diff --git a/gui/slick/interfaces/default/inc_top.tmpl b/gui/slick/interfaces/default/inc_top.tmpl index 9d522803..9b152cbf 100644 --- a/gui/slick/interfaces/default/inc_top.tmpl +++ b/gui/slick/interfaces/default/inc_top.tmpl @@ -5,8 +5,9 @@ - + + SickGear - BRANCH:[$sickbeard.BRANCH] - $title @@ -34,35 +35,35 @@ - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + #if $sickbeard.FUZZY_DATING - - + + #end if - + #set $tab = 4 #set $body_attr = '' diff --git a/sickbeard/webapi.py b/sickbeard/webapi.py index 3733a45f..8d3f2d05 100644 --- a/sickbeard/webapi.py +++ b/sickbeard/webapi.py @@ -75,6 +75,7 @@ class Api(webserve.BaseHandler): def set_default_headers(self): self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') + self.set_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nocache, noodp, noydir, noimageindex, nosnippet') def get(self, route, *args, **kwargs): route = route.strip('/') or 'index' diff --git a/sickbeard/webserve.py b/sickbeard/webserve.py index 5212f034..8fe99512 100644 --- a/sickbeard/webserve.py +++ b/sickbeard/webserve.py @@ -49,7 +49,7 @@ from sickbeard.browser import foldersAtPath from sickbeard.blackandwhitelist import BlackAndWhiteList, short_group_names from sickbeard.searchBacklog import FULL_BACKLOG, LIMITED_BACKLOG from tornado import gen -from tornado.web import RequestHandler, authenticated +from tornado.web import RequestHandler, StaticFileHandler, authenticated from lib import adba from lib import subliminal from lib.dateutil import tz @@ -111,9 +111,15 @@ class PageTemplate(Template): return super(PageTemplate, self).compile(*args, **kwargs) +class BaseStaticFileHandler(StaticFileHandler): + def set_extra_headers(self, path): + self.set_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nocache, noodp, noydir, noimageindex, nosnippet') + + class BaseHandler(RequestHandler): def set_default_headers(self): self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') + self.set_header('X-Robots-Tag', 'noindex, nofollow, noarchive, nocache, noodp, noydir, noimageindex, nosnippet') def redirect(self, url, permanent=False, status=None): if not url.startswith(sickbeard.WEB_ROOT): diff --git a/sickbeard/webserveInit.py b/sickbeard/webserveInit.py index a881af61..19720e19 100644 --- a/sickbeard/webserveInit.py +++ b/sickbeard/webserveInit.py @@ -7,7 +7,7 @@ import webapi from sickbeard import logger from sickbeard.helpers import create_https_certificates -from tornado.web import Application, StaticFileHandler +from tornado.web import Application from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop @@ -41,8 +41,8 @@ class WebServer(threading.Thread): if self.enable_https: # If either the HTTPS certificate or key do not exist, make some self-signed ones. - if not (self.https_cert and os.path.exists(self.https_cert)) or not ( - self.https_key and os.path.exists(self.https_key)): + if not (self.https_cert and os.path.exists(self.https_cert))\ + or not (self.https_key and os.path.exists(self.https_key)): if not create_https_certificates(self.https_cert, self.https_key): logger.log(u'Unable to create CERT/KEY files, disabling HTTPS') sickbeard.ENABLE_HTTPS = False @@ -55,13 +55,13 @@ class WebServer(threading.Thread): # Load the app self.app = Application([], - debug=True, - autoreload=False, - gzip=True, - xheaders=sickbeard.HANDLE_REVERSE_PROXY, - cookie_secret=sickbeard.COOKIE_SECRET, - login_url='%s/login/' % self.options['web_root'], - ) + debug=True, + autoreload=False, + gzip=True, + xheaders=sickbeard.HANDLE_REVERSE_PROXY, + cookie_secret=sickbeard.COOKIE_SECRET, + login_url='%s/login/' % self.options['web_root'] + ) # Main Handler self.app.add_handlers('.*$', [ @@ -104,27 +104,26 @@ class WebServer(threading.Thread): # Static File Handlers self.app.add_handlers('.*$', [ # favicon - (r'%s/(favicon\.ico)' % self.options['web_root'], StaticFileHandler, + (r'%s/(favicon\.ico)' % self.options['web_root'], webserve.BaseStaticFileHandler, {'path': os.path.join(self.options['data_root'], 'images/ico/favicon.ico')}), # images - (r'%s/images/(.*)' % self.options['web_root'], StaticFileHandler, + (r'%s/images/(.*)' % self.options['web_root'], webserve.BaseStaticFileHandler, {'path': os.path.join(self.options['data_root'], 'images')}), # cached images - (r'%s/cache/images/(.*)' % self.options['web_root'], StaticFileHandler, + (r'%s/cache/images/(.*)' % self.options['web_root'], webserve.BaseStaticFileHandler, {'path': os.path.join(sickbeard.CACHE_DIR, 'images')}), # css - (r'%s/css/(.*)' % self.options['web_root'], StaticFileHandler, + (r'%s/css/(.*)' % self.options['web_root'], webserve.BaseStaticFileHandler, {'path': os.path.join(self.options['data_root'], 'css')}), # javascript - (r'%s/js/(.*)' % self.options['web_root'], StaticFileHandler, + (r'%s/js/(.*)' % self.options['web_root'], webserve.BaseStaticFileHandler, {'path': os.path.join(self.options['data_root'], 'js')}), ]) - def run(self): if self.enable_https: protocol = 'https'