# Capture bots or misbehaved bots, download lots of pages with no public benefit
user-agent: AhrefsBot
disallow: /

user-agent: BUbiNG
disallow: /

user-agent: dotbot
disallow: /

user-agent: mj12bot
disallow: /

user-agent: MJ12bot
disallow: /

User-agent: deepcrawl
disallow: /

user-agent: Scrapy
disallow: /

user-agent: grub-client
disallow: /

user-agent: k2spider
disallow: /

# http://www.webreaper.net/
user-agent: WebReaper
disallow: /

# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
user-agent: sitecheck.internetseer.com
disallow: /

user-agent: Zealbot
disallow: /

user-agent: MSIECrawler
disallow: /

user-agent: SiteSnagger
disallow: /

user-agent: WebStripper
disallow: /

user-agent: WebCopier
disallow: /

user-agent: Fetch
disallow: /

user-agent: Offline Explorer
disallow: /

user-agent: Teleport
disallow: /

user-agent: TeleportPro
disallow: /

user-agent: WebZIP
disallow: /

user-agent: linko
disallow: /

user-agent: HTTrack
disallow: /

user-agent: Microsoft.URL.Control
disallow: /

user-agent: Xenu
disallow: /

user-agent: larbin
disallow: /

user-agent: libwww
disallow: /

user-agent: ZyBORG
disallow: /

user-agent: Download Ninja
disallow: /

# Well behaved bots
user-agent: *
crawl-delay: 1
allow: /images/*.jpg$
allow: /images/*.jpg?_v=*
allow: /images/*.jpeg$
allow: /images/*.jpeg?_v=*
allow: /images/*.png$
allow: /images/*.png?_v=*
allow: /images/*.svg$
allow: /images/*.svg?_v=*
allow: /images/*.gif$
allow: /images/*.gif?_v=*
disallow: /images
disallow: /support/download/
disallow: /api/print/to-pdf/*
disallow: /api/print/to-txt/*
disallow: /*?rendering-to-pdf=true
disallow: /*ecommerce/
disallow: /*edelivery/
disallow: /*checkout/