User-agent: * # match all bots

#Disallow: /   # keep them out. Remove on go live!
Disallow: /make-a-payment-iframe

# The following directories are all linked to in the frontend, but we probably don't want them indexed.
Disallow: /css
Disallow: /Css
Disallow: /less
Disallow: /Less
Disallow: /media
Disallow: /Media
Disallow: /scripts
Disallow: /Scripts
Disallow: /umbraco
Disallow: /Umbraco

#
# Following robots.txt might be of use:
# http://www.hotelopia.com/robots.txt


# Causing '/Umbraco/RenderMvc' errors with the user agent: 
# Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)
User-agent: dotbot
Disallow: /

# Causing '/Umbraco/RenderMvc' and 'page not found' errors with the user agent: 
# Mozilla/5.0 (compatible; DomainAppender /1.0; +http://www.profound.net/domainappender)
User-agent: DomainAppender 
Disallow: /

# Causing '/Umbraco/RenderMvc' and 'page not found' errors with the user agent: 
# Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)
User-agent: AhrefsBot
Disallow: /

# Causing '/Umbraco/RenderMvc' and 'page not found' errors with the user agent: 
# Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
User-agent: SiteCheck-sitecrawl by Siteimprove.com
Disallow: /news


# Causing '/Umbraco/RenderMvc' and 'page not found' errors with the user agent: 
# Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)
# spider@seoscanners.net
# spider@seoscanners.net
User-Agent: seoscanners.net
Disallow: /

# Causing '/Umbraco/RenderMvc' and 'page not found' errors with the user agent: 
# Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)
User-Agent: MegaIndex.ru
Disallow: /

# Causing '/Umbraco/RenderMvc' and 'page not found' errors with the user agent: 
# AA_Crawler/Nutch-1.12
User-Agent: AA_Crawler
Disallow: /

User-agent: DeuSu
Disallow: /

User-agent: meds-online24.com
Disallow: /

User-agent: seoscanners
Disallow: /

User-agent: Pingdom.com
Disallow: /

User-agent: PRTGCloudBot
Disallow: /